pi-smart-router 0.12.0 → 0.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.pi/extensions/smart-router/command-formatters.ts +93 -0
- package/.pi/extensions/smart-router/commands.ts +24 -2
- package/.pi/extensions/smart-router/index.ts +2 -0
- package/.pi/extensions/smart-router/types.ts +1 -0
- package/README.md +3 -2
- package/config/benchmark-profiles.json +2 -2
- package/dist/infrastructure/telemetry/session-stats.d.ts +78 -0
- package/dist/infrastructure/telemetry/session-stats.d.ts.map +1 -0
- package/dist/infrastructure/telemetry/session-stats.js +187 -0
- package/dist/infrastructure/telemetry/session-stats.js.map +1 -0
- package/package.json +1 -1
- package/src/infrastructure/telemetry/session-stats.ts +290 -0
|
@@ -4,9 +4,14 @@ import {
|
|
|
4
4
|
} from '../../../src/infrastructure/telemetry/telemetry-limits.js';
|
|
5
5
|
import type {
|
|
6
6
|
ModelProfile,
|
|
7
|
+
PriceCatalog,
|
|
7
8
|
RoutingDecision,
|
|
8
9
|
RoutingTelemetry,
|
|
9
10
|
} from '../../../src/domain/types/index.js';
|
|
11
|
+
import {
|
|
12
|
+
aggregateSessionStatsFromFleet,
|
|
13
|
+
type SessionStatsSnapshot,
|
|
14
|
+
} from '../../../src/infrastructure/telemetry/session-stats.js';
|
|
10
15
|
import { SMART_ROUTER_USAGE } from './commands.js';
|
|
11
16
|
import {
|
|
12
17
|
DEFAULT_TELEMETRY_CONTRIB_EXPORT_LIMIT,
|
|
@@ -19,6 +24,8 @@ import {
|
|
|
19
24
|
import { formatPricingStalenessLine } from './pricing-lifecycle.js';
|
|
20
25
|
import type { FleetMode, SmartRouterCommand, SmartRouterRuntime } from './types.js';
|
|
21
26
|
|
|
27
|
+
export type { SessionStatsSnapshot };
|
|
28
|
+
|
|
22
29
|
/** Opaque / virtual auto ids that hide the concrete delegated fleet model (SP-178). */
|
|
23
30
|
function isBareOrSmartRouterAuto(modelId: string): boolean {
|
|
24
31
|
return modelId === 'auto' || modelId === 'smart-router/auto';
|
|
@@ -149,6 +156,10 @@ export function parseSmartRouterArgs(args: string): SmartRouterCommand {
|
|
|
149
156
|
return { command: 'history', limit: parseHistoryLimit(tokens[1]) };
|
|
150
157
|
}
|
|
151
158
|
|
|
159
|
+
if (tokens[0] === 'stats') {
|
|
160
|
+
return { command: 'stats', limit: parseHistoryLimit(tokens[1]) };
|
|
161
|
+
}
|
|
162
|
+
|
|
152
163
|
if (tokens[0] === 'mode' && (tokens[1] === 'scoped' || tokens[1] === 'all')) {
|
|
153
164
|
return { command: 'mode', mode: tokens[1] };
|
|
154
165
|
}
|
|
@@ -245,4 +256,86 @@ export function formatHistoryMessage(
|
|
|
245
256
|
.join('\n');
|
|
246
257
|
}
|
|
247
258
|
|
|
259
|
+
function formatUsd(value: number): string {
|
|
260
|
+
if (!Number.isFinite(value)) {
|
|
261
|
+
return 'n/a';
|
|
262
|
+
}
|
|
263
|
+
if (Math.abs(value) >= 1) {
|
|
264
|
+
return `$${value.toFixed(4)}`;
|
|
265
|
+
}
|
|
266
|
+
return `$${value.toFixed(6)}`;
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
function formatShare(value: number | null): string {
|
|
270
|
+
if (value === null) {
|
|
271
|
+
return 'n/a';
|
|
272
|
+
}
|
|
273
|
+
return `${(value * 100).toFixed(1)}%`;
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
function formatMean(value: number | null, suffix: string): string {
|
|
277
|
+
if (value === null) {
|
|
278
|
+
return 'n/a';
|
|
279
|
+
}
|
|
280
|
+
if (suffix === 'ms') {
|
|
281
|
+
return `${value.toFixed(1)}${suffix}`;
|
|
282
|
+
}
|
|
283
|
+
return `${formatUsd(value)}${suffix}`;
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
/**
|
|
287
|
+
* Operator-facing text for `/smart-router stats` (privacy-safe aggregates only).
|
|
288
|
+
*/
|
|
289
|
+
export function formatStatsMessage(
|
|
290
|
+
entries: readonly RoutingTelemetry[],
|
|
291
|
+
options?: {
|
|
292
|
+
fleet?: readonly ModelProfile[];
|
|
293
|
+
priceCatalog?: PriceCatalog | null;
|
|
294
|
+
},
|
|
295
|
+
): string {
|
|
296
|
+
const snapshot = aggregateSessionStatsFromFleet(
|
|
297
|
+
entries,
|
|
298
|
+
options?.fleet,
|
|
299
|
+
options?.priceCatalog,
|
|
300
|
+
);
|
|
301
|
+
|
|
302
|
+
if (snapshot.entry_count === 0) {
|
|
303
|
+
return 'No routing stats yet (empty telemetry window).';
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
const lines = [
|
|
307
|
+
`Entries: ${snapshot.entry_count}`,
|
|
308
|
+
`Cost: total ${formatUsd(snapshot.total_cost_usd)} | mean ${formatMean(snapshot.mean_cost_usd, '')}`,
|
|
309
|
+
`Latency: total ${snapshot.total_latency_ms.toFixed(0)}ms | mean ${formatMean(snapshot.mean_latency_ms, 'ms')}`,
|
|
310
|
+
`Planning delegate share: ${formatShare(snapshot.planning_delegate_share)} (direct ${formatShare(snapshot.direct_share)})`,
|
|
311
|
+
`Local vs cloud (when known): local ${formatShare(snapshot.local_share)} | cloud ${formatShare(snapshot.cloud_share)}`,
|
|
312
|
+
'Role cost breakdown:',
|
|
313
|
+
` primary (pin path): ${snapshot.role_cost.primary.count} | ${formatUsd(snapshot.role_cost.primary.total_cost_usd)}`,
|
|
314
|
+
` planning_delegate: ${snapshot.role_cost.planning_delegate.count} | ${formatUsd(snapshot.role_cost.planning_delegate.total_cost_usd)}`,
|
|
315
|
+
` other: ${snapshot.role_cost.other.count} | ${formatUsd(snapshot.role_cost.other.total_cost_usd)}`,
|
|
316
|
+
];
|
|
317
|
+
|
|
318
|
+
if (snapshot.frontier_savings_usd !== undefined) {
|
|
319
|
+
lines.push(
|
|
320
|
+
`Vs always-frontier savings (est.): ${formatUsd(snapshot.frontier_savings_usd)}`,
|
|
321
|
+
' formula: sum max(0, tokens/1e6 * frontier_cost_per_1m - estimated_cost_usd); omitted when prices missing',
|
|
322
|
+
);
|
|
323
|
+
} else {
|
|
324
|
+
lines.push('Vs always-frontier savings: (omitted — frontier prices unavailable)');
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
return lines.join('\n');
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
/** JSON snapshot helper for automation (same aggregate as formatStatsMessage). */
|
|
331
|
+
export function buildStatsSnapshot(
|
|
332
|
+
entries: readonly RoutingTelemetry[],
|
|
333
|
+
options?: {
|
|
334
|
+
fleet?: readonly ModelProfile[];
|
|
335
|
+
priceCatalog?: PriceCatalog | null;
|
|
336
|
+
},
|
|
337
|
+
): SessionStatsSnapshot {
|
|
338
|
+
return aggregateSessionStatsFromFleet(entries, options?.fleet, options?.priceCatalog);
|
|
339
|
+
}
|
|
340
|
+
|
|
248
341
|
export type { FleetMode };
|
|
@@ -2,6 +2,7 @@ import type { ExtensionAPI } from '@earendil-works/pi-coding-agent';
|
|
|
2
2
|
|
|
3
3
|
import {
|
|
4
4
|
formatHistoryMessage,
|
|
5
|
+
formatStatsMessage,
|
|
5
6
|
formatStatusMessage,
|
|
6
7
|
parseSmartRouterArgs,
|
|
7
8
|
} from './command-formatters.js';
|
|
@@ -13,13 +14,14 @@ import { FLEET_MODE_ENTRY_TYPE } from './session-lifecycle.js';
|
|
|
13
14
|
import type { SmartRouterRuntime } from './types.js';
|
|
14
15
|
|
|
15
16
|
export const SMART_ROUTER_USAGE =
|
|
16
|
-
'/smart-router [status] | history [limit] | mode scoped|all | pricing refresh | export dataset [--limit N] | export telemetry-contrib [--limit N] | feedback good|bad | unpin';
|
|
17
|
+
'/smart-router [status] | history [limit] | stats [limit] | mode scoped|all | pricing refresh | export dataset [--limit N] | export telemetry-contrib [--limit N] | feedback good|bad | unpin';
|
|
17
18
|
|
|
18
19
|
type CompletionItem = { value: string; label: string };
|
|
19
20
|
|
|
20
21
|
const TOP_LEVEL: CompletionItem[] = [
|
|
21
22
|
{ value: 'status', label: 'Show last routing decision' },
|
|
22
23
|
{ value: 'history', label: 'Show recent routing history' },
|
|
24
|
+
{ value: 'stats', label: 'Show session stats + role cost breakdown' },
|
|
23
25
|
{ value: 'mode', label: 'Switch fleet mode (scoped or all)' },
|
|
24
26
|
{ value: 'pricing', label: 'Manage pricing catalog' },
|
|
25
27
|
{ value: 'export', label: 'Export opt-in routing dataset' },
|
|
@@ -52,6 +54,8 @@ export const SMART_ROUTER_FULL_INVOCATIONS = [
|
|
|
52
54
|
'status',
|
|
53
55
|
'history',
|
|
54
56
|
'history 10',
|
|
57
|
+
'stats',
|
|
58
|
+
'stats 50',
|
|
55
59
|
'mode scoped',
|
|
56
60
|
'mode all',
|
|
57
61
|
'pricing refresh',
|
|
@@ -100,6 +104,10 @@ export function getSmartRouterArgumentCompletions(prefix: string): CompletionIte
|
|
|
100
104
|
return [{ value: 'history', label: 'Show recent routing history' }];
|
|
101
105
|
}
|
|
102
106
|
|
|
107
|
+
if (tokens[0] === 'stats') {
|
|
108
|
+
return [{ value: 'stats', label: 'Show session stats + role cost breakdown' }];
|
|
109
|
+
}
|
|
110
|
+
|
|
103
111
|
const firstToken = tokens[0] ?? '';
|
|
104
112
|
const filtered = filterByPrefix(TOP_LEVEL, firstToken);
|
|
105
113
|
return filtered.length > 0 ? filtered : null;
|
|
@@ -143,7 +151,7 @@ export function registerSmartRouterCommand(
|
|
|
143
151
|
): void {
|
|
144
152
|
pi.registerCommand('smart-router', {
|
|
145
153
|
description:
|
|
146
|
-
'Show routing status/history, switch fleet mode (scoped|all), refresh pricing, or export dataset',
|
|
154
|
+
'Show routing status/history/stats, switch fleet mode (scoped|all), refresh pricing, or export dataset',
|
|
147
155
|
getArgumentCompletions: getSmartRouterArgumentCompletions,
|
|
148
156
|
handler: async (args, ctx) => {
|
|
149
157
|
try {
|
|
@@ -167,6 +175,20 @@ export function registerSmartRouterCommand(
|
|
|
167
175
|
return;
|
|
168
176
|
}
|
|
169
177
|
|
|
178
|
+
if (parsed.command === 'stats') {
|
|
179
|
+
throwIfCommandAborted(signal);
|
|
180
|
+
const rows = await runtime.store.listTelemetry({ limit: parsed.limit });
|
|
181
|
+
throwIfCommandAborted(signal);
|
|
182
|
+
ctx.ui.notify(
|
|
183
|
+
formatStatsMessage(rows, {
|
|
184
|
+
fleet: runtime.streamDeps.fleet,
|
|
185
|
+
priceCatalog: runtime.priceCatalog,
|
|
186
|
+
}),
|
|
187
|
+
'info',
|
|
188
|
+
);
|
|
189
|
+
return;
|
|
190
|
+
}
|
|
191
|
+
|
|
170
192
|
if (parsed.command === 'pricing') {
|
|
171
193
|
throwIfCommandAborted(signal);
|
|
172
194
|
const { modelCount, lastUpdated } = await refreshPricingCatalog(
|
|
@@ -36,6 +36,7 @@ import {
|
|
|
36
36
|
} from './fleet-bootstrap.js';
|
|
37
37
|
import {
|
|
38
38
|
formatHistoryMessage,
|
|
39
|
+
formatStatsMessage,
|
|
39
40
|
formatStatusMessage,
|
|
40
41
|
parseSmartRouterArgs,
|
|
41
42
|
resolveHistoryModelId,
|
|
@@ -82,6 +83,7 @@ export {
|
|
|
82
83
|
formatLmuStatus,
|
|
83
84
|
formatPricingStalenessLine,
|
|
84
85
|
formatHistoryMessage,
|
|
86
|
+
formatStatsMessage,
|
|
85
87
|
formatStatusMessage,
|
|
86
88
|
getDatasetExportPath,
|
|
87
89
|
getRouterStateDbPath,
|
|
@@ -34,6 +34,7 @@ export type FleetMode = 'scoped' | 'all';
|
|
|
34
34
|
export type SmartRouterCommand =
|
|
35
35
|
| { command: 'status' }
|
|
36
36
|
| { command: 'history'; limit: number }
|
|
37
|
+
| { command: 'stats'; limit: number }
|
|
37
38
|
| { command: 'mode'; mode: FleetMode }
|
|
38
39
|
| { command: 'pricing'; subcommand: 'refresh' }
|
|
39
40
|
| { command: 'export'; subcommand: 'dataset'; limit: number }
|
package/README.md
CHANGED
|
@@ -111,7 +111,7 @@ After installing via `pi install npm:pi-smart-router` (or from clone — see bel
|
|
|
111
111
|
|
|
112
112
|
1. Authenticate providers (`/login`) and enable models in your scoped list if you use one (`/scoped-models`)
|
|
113
113
|
2. `/model smart-router/auto` — every turn runs through the routing pipeline
|
|
114
|
-
3. `/smart-router status
|
|
114
|
+
3. `/smart-router status`, `/smart-router history`, or `/smart-router stats` — inspect routing decisions and window aggregates
|
|
115
115
|
|
|
116
116
|
Set `SMART_ROUTER_LOG_ROUTING=1` before starting pi to print each routing decision to stderr (see [Environment variables](#environment-variables)).
|
|
117
117
|
|
|
@@ -180,7 +180,7 @@ pi exposes two different **auto** models. They are easy to confuse but play diff
|
|
|
180
180
|
**When to use `smart-router/auto`:**
|
|
181
181
|
|
|
182
182
|
- You want cost/capability-aware model selection across your full authenticated fleet
|
|
183
|
-
- You rely on session pinning, failover, or `/smart-router status` / `history` telemetry
|
|
183
|
+
- You rely on session pinning, failover, or `/smart-router status` / `history` / `stats` telemetry
|
|
184
184
|
- Tool-heavy sessions with Gemini economical models work via in-repo replay repair; add `cursor/auto` for unrepairable Google replay edge cases (see [pi-smart-router#85](https://github.com/beettlle/pi-smart-router/issues/85))
|
|
185
185
|
|
|
186
186
|
Cursor models (`cursor/*`, `composer-*`, and the opaque fleet id `default`) map to **frontier-cloud** tier in `pi-model-mapper.ts` so HyDRA can score them against Gemini and Claude instead of treating them as unknown economical models ([pi-smart-router#40](https://github.com/beettlle/pi-smart-router/issues/40), [pi-smart-router#70](https://github.com/beettlle/pi-smart-router/issues/70)). Related: [pi-smart-router#23](https://github.com/beettlle/pi-smart-router/issues/23) (turn envelope / pin order), [pi-smart-router#37](https://github.com/beettlle/pi-smart-router/issues/37) (Gemini `thought_signature` errors).
|
|
@@ -198,6 +198,7 @@ Cursor models bill against your **Cursor Pro subscription quota**, not per-token
|
|
|
198
198
|
| `/smart-router` | Same as `status` (default when no subcommand is given) |
|
|
199
199
|
| `/smart-router status` | Show fleet mode, fleet size, pricing freshness/staleness, and the last routing decision (stage, tier, selected model, latency) |
|
|
200
200
|
| `/smart-router history` | Show recent routing telemetry from SQLite (default limit; optional numeric limit, e.g. `/smart-router history 20`). Displays the concrete delegated model id (never bare virtual `auto`) |
|
|
201
|
+
| `/smart-router stats` | Privacy-safe session/window aggregates from routing telemetry: count, mean cost/latency, planning_delegate vs direct share, local vs cloud when distinguishable, and role cost breakdown (primary pin path / planning_delegate / other). Optional vs-always-frontier savings when frontier fleet prices exist (omitted otherwise). Optional numeric limit, e.g. `/smart-router stats 50` |
|
|
201
202
|
| `/smart-router mode scoped` | Route only among pi's **enabled model patterns** (default) |
|
|
202
203
|
| `/smart-router mode all` | Route among **all authenticated models** in the registry |
|
|
203
204
|
| `/smart-router pricing refresh` | Manually fetch LiteLLM pricing from `LITELLM_PRICING_URL`, persist to SQLite, and rebuild the fleet with updated rates |
|
|
@@ -7,8 +7,8 @@
|
|
|
7
7
|
"livecodebench": "https://livecodebench.github.io/leaderboard.html",
|
|
8
8
|
"bfcl": "https://gorilla.cs.berkeley.edu/leaderboard.html"
|
|
9
9
|
},
|
|
10
|
-
"scrape_date": "2026-07-
|
|
11
|
-
"catalog_freeze_date": "2026-07-
|
|
10
|
+
"scrape_date": "2026-07-14",
|
|
11
|
+
"catalog_freeze_date": "2026-07-14"
|
|
12
12
|
},
|
|
13
13
|
"aliases": {
|
|
14
14
|
"claude-3-5-sonnet": "claude-sonnet-4-6",
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Privacy-safe session / window stats over RoutingTelemetry (SP-207 / #118).
|
|
3
|
+
*
|
|
4
|
+
* Aggregates only numeric and categorical telemetry fields — never prompt,
|
|
5
|
+
* message, or tool-argument bodies.
|
|
6
|
+
*/
|
|
7
|
+
import type { ModelProfile, PriceCatalog, RoutingTelemetry, Tier } from '../../domain/types/index.js';
|
|
8
|
+
export type RoleCostBucket = 'primary' | 'planning_delegate' | 'other';
|
|
9
|
+
export type DeploymentClass = 'local' | 'cloud' | 'unknown';
|
|
10
|
+
export interface RoleBucketStats {
|
|
11
|
+
readonly count: number;
|
|
12
|
+
readonly total_cost_usd: number;
|
|
13
|
+
}
|
|
14
|
+
export interface RoleCostBreakdown {
|
|
15
|
+
readonly primary: RoleBucketStats;
|
|
16
|
+
readonly planning_delegate: RoleBucketStats;
|
|
17
|
+
readonly other: RoleBucketStats;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Compact JSON snapshot for automation / MCP (llm-use `stats_snapshot` analog).
|
|
21
|
+
* Optional `frontier_savings_usd` is omitted when prices are unavailable (fail closed).
|
|
22
|
+
*/
|
|
23
|
+
export interface SessionStatsSnapshot {
|
|
24
|
+
readonly entry_count: number;
|
|
25
|
+
readonly total_cost_usd: number;
|
|
26
|
+
readonly mean_cost_usd: number | null;
|
|
27
|
+
readonly total_latency_ms: number;
|
|
28
|
+
readonly mean_latency_ms: number | null;
|
|
29
|
+
/** Share of entries with planning_delegate_path === 'delegate' (0–1), null if empty. */
|
|
30
|
+
readonly planning_delegate_share: number | null;
|
|
31
|
+
/** Share of entries with non-delegate path (direct / none / null). */
|
|
32
|
+
readonly direct_share: number | null;
|
|
33
|
+
/** Share classified as local (zero-tier) when distinguishable; null if none classified. */
|
|
34
|
+
readonly local_share: number | null;
|
|
35
|
+
/** Share classified as cloud when distinguishable; null if none classified. */
|
|
36
|
+
readonly cloud_share: number | null;
|
|
37
|
+
readonly role_cost: RoleCostBreakdown;
|
|
38
|
+
/**
|
|
39
|
+
* Estimated USD saved vs always-frontier baseline.
|
|
40
|
+
* Formula: sum over entries with token counts of
|
|
41
|
+
* max(0, tokens/1e6 * frontier_cost_per_1m - estimated_cost_usd).
|
|
42
|
+
* Omitted when frontier price inputs are missing (fail closed).
|
|
43
|
+
*/
|
|
44
|
+
readonly frontier_savings_usd?: number;
|
|
45
|
+
}
|
|
46
|
+
export interface AggregateSessionStatsOptions {
|
|
47
|
+
/**
|
|
48
|
+
* USD per 1M tokens for the always-frontier baseline.
|
|
49
|
+
* When absent / non-finite / ≤0, `frontier_savings_usd` is omitted.
|
|
50
|
+
*/
|
|
51
|
+
readonly frontier_cost_per_1m?: number;
|
|
52
|
+
/** Optional model_id → tier map (e.g. from fleet) for local vs cloud. */
|
|
53
|
+
readonly tier_by_model_id?: ReadonlyMap<string, Tier>;
|
|
54
|
+
}
|
|
55
|
+
/** Mutually exclusive role for cost bucketing. */
|
|
56
|
+
export declare function classifyRoleCostBucket(entry: RoutingTelemetry): RoleCostBucket;
|
|
57
|
+
export declare function classifyDeployment(entry: RoutingTelemetry, tierByModelId?: ReadonlyMap<string, Tier>): DeploymentClass;
|
|
58
|
+
/**
|
|
59
|
+
* Resolve always-frontier cost/1M from fleet + optional catalog.
|
|
60
|
+
* Returns undefined when no positive frontier price is available (fail closed).
|
|
61
|
+
*/
|
|
62
|
+
export declare function resolveFrontierCostPer1M(fleet?: readonly ModelProfile[], catalog?: PriceCatalog | null): number | undefined;
|
|
63
|
+
/**
|
|
64
|
+
* Pure aggregate over routing telemetry for operator stats.
|
|
65
|
+
* Does not read or emit prompt/message/tool bodies.
|
|
66
|
+
*/
|
|
67
|
+
export declare function aggregateSessionStats(entries: readonly RoutingTelemetry[], options?: AggregateSessionStatsOptions): SessionStatsSnapshot;
|
|
68
|
+
/**
|
|
69
|
+
* Optional vs-always-frontier savings. Returns undefined when price input is
|
|
70
|
+
* missing or non-positive (fail closed) or when no entry has token counts.
|
|
71
|
+
*/
|
|
72
|
+
export declare function estimateFrontierSavingsUsd(entries: readonly RoutingTelemetry[], frontierCostPer1M: number | undefined): number | undefined;
|
|
73
|
+
/** Convenience: aggregate with fleet/catalog-derived tier map + frontier price. */
|
|
74
|
+
export declare function aggregateSessionStatsFromFleet(entries: readonly RoutingTelemetry[], fleet?: readonly ModelProfile[], catalog?: PriceCatalog | null): SessionStatsSnapshot;
|
|
75
|
+
/** Keys that must never appear on a stats snapshot (privacy). */
|
|
76
|
+
export declare const SESSION_STATS_FORBIDDEN_KEYS: readonly ["prompt", "prompt_text", "messages", "content", "tool_calls", "tool_args", "pepper"];
|
|
77
|
+
export declare function assertSessionStatsPrivacySafe(snapshot: SessionStatsSnapshot): void;
|
|
78
|
+
//# sourceMappingURL=session-stats.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"session-stats.d.ts","sourceRoot":"","sources":["../../../src/infrastructure/telemetry/session-stats.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EACV,YAAY,EAEZ,YAAY,EACZ,gBAAgB,EAChB,IAAI,EACL,MAAM,6BAA6B,CAAC;AAErC,MAAM,MAAM,cAAc,GAAG,SAAS,GAAG,mBAAmB,GAAG,OAAO,CAAC;AAEvE,MAAM,MAAM,eAAe,GAAG,OAAO,GAAG,OAAO,GAAG,SAAS,CAAC;AAE5D,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;CACjC;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,OAAO,EAAE,eAAe,CAAC;IAClC,QAAQ,CAAC,iBAAiB,EAAE,eAAe,CAAC;IAC5C,QAAQ,CAAC,KAAK,EAAE,eAAe,CAAC;CACjC;AAED;;;GAGG;AACH,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,wFAAwF;IACxF,QAAQ,CAAC,uBAAuB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChD,sEAAsE;IACtE,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,2FAA2F;IAC3F,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,+EAA+E;IAC/E,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,SAAS,EAAE,iBAAiB,CAAC;IACtC;;;;;OAKG;IACH,QAAQ,CAAC,oBAAoB,CAAC,EAAE,MAAM,CAAC;CACxC;AAED,MAAM,WAAW,4BAA4B;IAC3C;;;OAGG;IACH,QAAQ,CAAC,oBAAoB,CAAC,EAAE,MAAM,CAAC;IACvC,yEAAyE;IACzE,QAAQ,CAAC,gBAAgB,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;CACvD;AAED,kDAAkD;AAClD,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,gBAAgB,GAAG,cAAc,CAQ9E;AAED,wBAAgB,kBAAkB,CAChC,KAAK,EAAE,gBAAgB,EACvB,aAAa,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,GACxC,eAAe,CAUjB;AAMD;;;GAGG;AACH,wBAAgB,wBAAwB,CACtC,KAAK,CAAC,EAAE,SAAS,YAAY,EAAE,EAC/B,OAAO,CAAC,EAAE,YAAY,GAAG,IAAI,GAC5B,MAAM,GAAG,SAAS,CA6BpB;AAcD;;;GAGG;AACH,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,SAAS,gBAAgB,EAAE,EACpC,OAAO,GAAE,4BAAiC,GACzC,oBAAoB,CA+DtB;AAED;;;GAGG;AACH,wBAAgB,0BAA0B,CACxC,OAAO,EAAE,SAAS,gBAAgB,EAAE,EACpC,iBAAiB,EAAE,MAAM,GAAG,SAAS,GACpC,MAAM,GAAG,SAAS,CA4BpB;AAED,mFAAmF;AACnF,wBAAgB,8BAA8B,CAC5C,OAAO,EAAE,SAAS,gBAAgB,EAAE,EACpC,KAAK,CAAC,EAAE,SAAS,YAAY,EAAE,EAC/B,OAAO,CAAC,EAAE,YAAY,GAAG,IAAI,GAC5B,oBAAoB,CAOtB;AAED,iEAAiE;AACjE,eAAO,MAAM,4BAA4B,gGAQ/B,CAAC;AAEX,wBAAgB,6BAA6B,CAAC,QAAQ,EAAE,oBAAoB,GAAG,IAAI,CAOlF"}
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Privacy-safe session / window stats over RoutingTelemetry (SP-207 / #118).
|
|
3
|
+
*
|
|
4
|
+
* Aggregates only numeric and categorical telemetry fields — never prompt,
|
|
5
|
+
* message, or tool-argument bodies.
|
|
6
|
+
*/
|
|
7
|
+
/** Mutually exclusive role for cost bucketing. */
|
|
8
|
+
export function classifyRoleCostBucket(entry) {
|
|
9
|
+
if (entry.planning_delegate_path === 'delegate') {
|
|
10
|
+
return 'planning_delegate';
|
|
11
|
+
}
|
|
12
|
+
if (entry.pin_reason != null) {
|
|
13
|
+
return 'primary';
|
|
14
|
+
}
|
|
15
|
+
return 'other';
|
|
16
|
+
}
|
|
17
|
+
export function classifyDeployment(entry, tierByModelId) {
|
|
18
|
+
const fromFleet = tierByModelId?.get(entry.selected_model_id);
|
|
19
|
+
const tier = fromFleet ?? entry.tier_hint;
|
|
20
|
+
if (tier === 'zero-tier') {
|
|
21
|
+
return 'local';
|
|
22
|
+
}
|
|
23
|
+
if (tier === 'economical-cloud' || tier === 'frontier-cloud') {
|
|
24
|
+
return 'cloud';
|
|
25
|
+
}
|
|
26
|
+
return 'unknown';
|
|
27
|
+
}
|
|
28
|
+
function isPlanningDelegate(path) {
|
|
29
|
+
return path === 'delegate';
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Resolve always-frontier cost/1M from fleet + optional catalog.
|
|
33
|
+
* Returns undefined when no positive frontier price is available (fail closed).
|
|
34
|
+
*/
|
|
35
|
+
export function resolveFrontierCostPer1M(fleet, catalog) {
|
|
36
|
+
const candidates = [];
|
|
37
|
+
if (fleet) {
|
|
38
|
+
for (const model of fleet) {
|
|
39
|
+
if (model.tier !== 'frontier-cloud') {
|
|
40
|
+
continue;
|
|
41
|
+
}
|
|
42
|
+
const cost = model.pricing.fallback_cost_per_1m;
|
|
43
|
+
if (Number.isFinite(cost) && cost > 0) {
|
|
44
|
+
candidates.push(cost);
|
|
45
|
+
}
|
|
46
|
+
if (catalog) {
|
|
47
|
+
const key = model.pricing.registry_key ?? model.id;
|
|
48
|
+
const fromCatalog = catalog.user_overrides[key] ?? catalog.registry_snapshot[key];
|
|
49
|
+
if (fromCatalog !== undefined && Number.isFinite(fromCatalog) && fromCatalog > 0) {
|
|
50
|
+
candidates.push(fromCatalog);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
if (candidates.length === 0) {
|
|
56
|
+
return undefined;
|
|
57
|
+
}
|
|
58
|
+
// Upper-bound “always frontier” baseline: most expensive known frontier rate.
|
|
59
|
+
return Math.max(...candidates);
|
|
60
|
+
}
|
|
61
|
+
function buildTierMap(fleet) {
|
|
62
|
+
if (!fleet || fleet.length === 0) {
|
|
63
|
+
return undefined;
|
|
64
|
+
}
|
|
65
|
+
const map = new Map();
|
|
66
|
+
for (const model of fleet) {
|
|
67
|
+
map.set(model.id, model.tier);
|
|
68
|
+
map.set(`${model.provider}/${model.id}`, model.tier);
|
|
69
|
+
}
|
|
70
|
+
return map;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Pure aggregate over routing telemetry for operator stats.
|
|
74
|
+
* Does not read or emit prompt/message/tool bodies.
|
|
75
|
+
*/
|
|
76
|
+
export function aggregateSessionStats(entries, options = {}) {
|
|
77
|
+
const tierByModelId = options.tier_by_model_id;
|
|
78
|
+
const roleAccum = {
|
|
79
|
+
primary: { count: 0, total_cost_usd: 0 },
|
|
80
|
+
planning_delegate: { count: 0, total_cost_usd: 0 },
|
|
81
|
+
other: { count: 0, total_cost_usd: 0 },
|
|
82
|
+
};
|
|
83
|
+
let totalCost = 0;
|
|
84
|
+
let totalLatency = 0;
|
|
85
|
+
let delegateCount = 0;
|
|
86
|
+
let localCount = 0;
|
|
87
|
+
let cloudCount = 0;
|
|
88
|
+
let classifiedDeployment = 0;
|
|
89
|
+
for (const entry of entries) {
|
|
90
|
+
const cost = Number.isFinite(entry.estimated_cost_usd) ? entry.estimated_cost_usd : 0;
|
|
91
|
+
const latency = Number.isFinite(entry.routing_latency_ms) ? entry.routing_latency_ms : 0;
|
|
92
|
+
totalCost += cost;
|
|
93
|
+
totalLatency += latency;
|
|
94
|
+
if (isPlanningDelegate(entry.planning_delegate_path)) {
|
|
95
|
+
delegateCount += 1;
|
|
96
|
+
}
|
|
97
|
+
const role = classifyRoleCostBucket(entry);
|
|
98
|
+
roleAccum[role].count += 1;
|
|
99
|
+
roleAccum[role].total_cost_usd += cost;
|
|
100
|
+
const deployment = classifyDeployment(entry, tierByModelId);
|
|
101
|
+
if (deployment === 'local') {
|
|
102
|
+
localCount += 1;
|
|
103
|
+
classifiedDeployment += 1;
|
|
104
|
+
}
|
|
105
|
+
else if (deployment === 'cloud') {
|
|
106
|
+
cloudCount += 1;
|
|
107
|
+
classifiedDeployment += 1;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
const n = entries.length;
|
|
111
|
+
const snapshot = {
|
|
112
|
+
entry_count: n,
|
|
113
|
+
total_cost_usd: totalCost,
|
|
114
|
+
mean_cost_usd: n > 0 ? totalCost / n : null,
|
|
115
|
+
total_latency_ms: totalLatency,
|
|
116
|
+
mean_latency_ms: n > 0 ? totalLatency / n : null,
|
|
117
|
+
planning_delegate_share: n > 0 ? delegateCount / n : null,
|
|
118
|
+
direct_share: n > 0 ? (n - delegateCount) / n : null,
|
|
119
|
+
local_share: classifiedDeployment > 0 ? localCount / classifiedDeployment : null,
|
|
120
|
+
cloud_share: classifiedDeployment > 0 ? cloudCount / classifiedDeployment : null,
|
|
121
|
+
role_cost: {
|
|
122
|
+
primary: { ...roleAccum.primary },
|
|
123
|
+
planning_delegate: { ...roleAccum.planning_delegate },
|
|
124
|
+
other: { ...roleAccum.other },
|
|
125
|
+
},
|
|
126
|
+
};
|
|
127
|
+
const frontierSavings = estimateFrontierSavingsUsd(entries, options.frontier_cost_per_1m);
|
|
128
|
+
if (frontierSavings !== undefined) {
|
|
129
|
+
return { ...snapshot, frontier_savings_usd: frontierSavings };
|
|
130
|
+
}
|
|
131
|
+
return snapshot;
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Optional vs-always-frontier savings. Returns undefined when price input is
|
|
135
|
+
* missing or non-positive (fail closed) or when no entry has token counts.
|
|
136
|
+
*/
|
|
137
|
+
export function estimateFrontierSavingsUsd(entries, frontierCostPer1M) {
|
|
138
|
+
if (frontierCostPer1M === undefined ||
|
|
139
|
+
!Number.isFinite(frontierCostPer1M) ||
|
|
140
|
+
frontierCostPer1M <= 0) {
|
|
141
|
+
return undefined;
|
|
142
|
+
}
|
|
143
|
+
let savings = 0;
|
|
144
|
+
let counted = 0;
|
|
145
|
+
for (const entry of entries) {
|
|
146
|
+
const tokens = entry.estimated_input_tokens;
|
|
147
|
+
if (tokens === null || !Number.isFinite(tokens) || tokens < 0) {
|
|
148
|
+
continue;
|
|
149
|
+
}
|
|
150
|
+
const actual = Number.isFinite(entry.estimated_cost_usd) ? entry.estimated_cost_usd : 0;
|
|
151
|
+
const frontierCost = (tokens / 1_000_000) * frontierCostPer1M;
|
|
152
|
+
savings += Math.max(0, frontierCost - actual);
|
|
153
|
+
counted += 1;
|
|
154
|
+
}
|
|
155
|
+
if (counted === 0) {
|
|
156
|
+
return undefined;
|
|
157
|
+
}
|
|
158
|
+
return savings;
|
|
159
|
+
}
|
|
160
|
+
/** Convenience: aggregate with fleet/catalog-derived tier map + frontier price. */
|
|
161
|
+
export function aggregateSessionStatsFromFleet(entries, fleet, catalog) {
|
|
162
|
+
const tierMap = buildTierMap(fleet);
|
|
163
|
+
const frontierCost = resolveFrontierCostPer1M(fleet, catalog);
|
|
164
|
+
return aggregateSessionStats(entries, {
|
|
165
|
+
...(tierMap ? { tier_by_model_id: tierMap } : {}),
|
|
166
|
+
...(frontierCost !== undefined ? { frontier_cost_per_1m: frontierCost } : {}),
|
|
167
|
+
});
|
|
168
|
+
}
|
|
169
|
+
/** Keys that must never appear on a stats snapshot (privacy). */
|
|
170
|
+
export const SESSION_STATS_FORBIDDEN_KEYS = [
|
|
171
|
+
'prompt',
|
|
172
|
+
'prompt_text',
|
|
173
|
+
'messages',
|
|
174
|
+
'content',
|
|
175
|
+
'tool_calls',
|
|
176
|
+
'tool_args',
|
|
177
|
+
'pepper',
|
|
178
|
+
];
|
|
179
|
+
export function assertSessionStatsPrivacySafe(snapshot) {
|
|
180
|
+
const json = JSON.stringify(snapshot);
|
|
181
|
+
for (const key of SESSION_STATS_FORBIDDEN_KEYS) {
|
|
182
|
+
if (json.includes(`"${key}"`)) {
|
|
183
|
+
throw new Error(`Stats snapshot contains forbidden privacy key: ${key}`);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
//# sourceMappingURL=session-stats.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"session-stats.js","sourceRoot":"","sources":["../../../src/infrastructure/telemetry/session-stats.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AA+DH,kDAAkD;AAClD,MAAM,UAAU,sBAAsB,CAAC,KAAuB;IAC5D,IAAI,KAAK,CAAC,sBAAsB,KAAK,UAAU,EAAE,CAAC;QAChD,OAAO,mBAAmB,CAAC;IAC7B,CAAC;IACD,IAAI,KAAK,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC;QAC7B,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,kBAAkB,CAChC,KAAuB,EACvB,aAAyC;IAEzC,MAAM,SAAS,GAAG,aAAa,EAAE,GAAG,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;IAC9D,MAAM,IAAI,GAAG,SAAS,IAAI,KAAK,CAAC,SAAS,CAAC;IAC1C,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;QACzB,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,IAAI,IAAI,KAAK,kBAAkB,IAAI,IAAI,KAAK,gBAAgB,EAAE,CAAC;QAC7D,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAiC;IAC3D,OAAO,IAAI,KAAK,UAAU,CAAC;AAC7B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,wBAAwB,CACtC,KAA+B,EAC/B,OAA6B;IAE7B,MAAM,UAAU,GAAa,EAAE,CAAC;IAEhC,IAAI,KAAK,EAAE,CAAC;QACV,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;YAC1B,IAAI,KAAK,CAAC,IAAI,KAAK,gBAAgB,EAAE,CAAC;gBACpC,SAAS;YACX,CAAC;YACD,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,oBAAoB,CAAC;YAChD,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;gBACtC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,CAAC;YACD,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,YAAY,IAAI,KAAK,CAAC,EAAE,CAAC;gBACnD,MAAM,WAAW,GACf,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC;gBAChE,IAAI,WAAW,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;oBACjF,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;gBAC/B,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,8EAA8E;IAC9E,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC,CAAC;AACjC,CAAC;AAED,SAAS,YAAY,CAAC,KAA+B;IACnD,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjC,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,GAAG,GAAG,IAAI,GAAG,EAAgB,CAAC;IACpC,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC1B,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAC9B,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,EAAE,EAAE,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;IACvD,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CACnC,OAAoC,EACpC,UAAwC,EAAE;IAE1C,MAAM,aAAa,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAC/C,MAAM,SAAS,GAAG;QAChB,OAAO,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE;QACxC,iBAAiB,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE;QAClD,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE;KACvC,CAAC;IAEF,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,oBAAoB,GAAG,CAAC,CAAC;IAE7B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC;QACtF,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC;QACzF,SAAS,IAAI,IAAI,CAAC;QAClB,YAAY,IAAI,OAAO,CAAC;QAExB,IAAI,kBAAkB,CAAC,KAAK,CAAC,sBAAsB,CAAC,EAAE,CAAC;YACrD,aAAa,IAAI,CAAC,CAAC;QACrB,CAAC;QAED,MAAM,IAAI,GAAG,sBAAsB,CAAC,KAAK,CAAC,CAAC;QAC3C,SAAS,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;QAC3B,SAAS,CAAC,IAAI,CAAC,CAAC,cAAc,IAAI,IAAI,CAAC;QAEvC,MAAM,UAAU,GAAG,kBAAkB,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;QAC5D,IAAI,UAAU,KAAK,OAAO,EAAE,CAAC;YAC3B,UAAU,IAAI,CAAC,CAAC;YAChB,oBAAoB,IAAI,CAAC,CAAC;QAC5B,CAAC;aAAM,IAAI,UAAU,KAAK,OAAO,EAAE,CAAC;YAClC,UAAU,IAAI,CAAC,CAAC;YAChB,oBAAoB,IAAI,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IACzB,MAAM,QAAQ,GAAyB;QACrC,WAAW,EAAE,CAAC;QACd,cAAc,EAAE,SAAS;QACzB,aAAa,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI;QAC3C,gBAAgB,EAAE,YAAY;QAC9B,eAAe,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI;QAChD,uBAAuB,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI;QACzD,YAAY,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI;QACpD,WAAW,EAAE,oBAAoB,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,oBAAoB,CAAC,CAAC,CAAC,IAAI;QAChF,WAAW,EAAE,oBAAoB,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,oBAAoB,CAAC,CAAC,CAAC,IAAI;QAChF,SAAS,EAAE;YACT,OAAO,EAAE,EAAE,GAAG,SAAS,CAAC,OAAO,EAAE;YACjC,iBAAiB,EAAE,EAAE,GAAG,SAAS,CAAC,iBAAiB,EAAE;YACrD,KAAK,EAAE,EAAE,GAAG,SAAS,CAAC,KAAK,EAAE;SAC9B;KACF,CAAC;IAEF,MAAM,eAAe,GAAG,0BAA0B,CAAC,OAAO,EAAE,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAC1F,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;QAClC,OAAO,EAAE,GAAG,QAAQ,EAAE,oBAAoB,EAAE,eAAe,EAAE,CAAC;IAChE,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,0BAA0B,CACxC,OAAoC,EACpC,iBAAqC;IAErC,IACE,iBAAiB,KAAK,SAAS;QAC/B,CAAC,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QACnC,iBAAiB,IAAI,CAAC,EACtB,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,OAAO,GAAG,CAAC,CAAC;IAEhB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,KAAK,CAAC,sBAAsB,CAAC;QAC5C,IAAI,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9D,SAAS;QACX,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC;QACxF,MAAM,YAAY,GAAG,CAAC,MAAM,GAAG,SAAS,CAAC,GAAG,iBAAiB,CAAC;QAC9D,OAAO,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,YAAY,GAAG,MAAM,CAAC,CAAC;QAC9C,OAAO,IAAI,CAAC,CAAC;IACf,CAAC;IAED,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;QAClB,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,8BAA8B,CAC5C,OAAoC,EACpC,KAA+B,EAC/B,OAA6B;IAE7B,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IACpC,MAAM,YAAY,GAAG,wBAAwB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAC9D,OAAO,qBAAqB,CAAC,OAAO,EAAE;QACpC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACjD,GAAG,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,oBAAoB,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC9E,CAAC,CAAC;AACL,CAAC;AAED,iEAAiE;AACjE,MAAM,CAAC,MAAM,4BAA4B,GAAG;IAC1C,QAAQ;IACR,aAAa;IACb,UAAU;IACV,SAAS;IACT,YAAY;IACZ,WAAW;IACX,QAAQ;CACA,CAAC;AAEX,MAAM,UAAU,6BAA6B,CAAC,QAA8B;IAC1E,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IACtC,KAAK,MAAM,GAAG,IAAI,4BAA4B,EAAE,CAAC;QAC/C,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,kDAAkD,GAAG,EAAE,CAAC,CAAC;QAC3E,CAAC;IACH,CAAC;AACH,CAAC"}
|
package/package.json
CHANGED
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Privacy-safe session / window stats over RoutingTelemetry (SP-207 / #118).
|
|
3
|
+
*
|
|
4
|
+
* Aggregates only numeric and categorical telemetry fields — never prompt,
|
|
5
|
+
* message, or tool-argument bodies.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import type {
|
|
9
|
+
ModelProfile,
|
|
10
|
+
PlanningDelegatePath,
|
|
11
|
+
PriceCatalog,
|
|
12
|
+
RoutingTelemetry,
|
|
13
|
+
Tier,
|
|
14
|
+
} from '../../domain/types/index.js';
|
|
15
|
+
|
|
16
|
+
export type RoleCostBucket = 'primary' | 'planning_delegate' | 'other';
|
|
17
|
+
|
|
18
|
+
export type DeploymentClass = 'local' | 'cloud' | 'unknown';
|
|
19
|
+
|
|
20
|
+
export interface RoleBucketStats {
|
|
21
|
+
readonly count: number;
|
|
22
|
+
readonly total_cost_usd: number;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export interface RoleCostBreakdown {
|
|
26
|
+
readonly primary: RoleBucketStats;
|
|
27
|
+
readonly planning_delegate: RoleBucketStats;
|
|
28
|
+
readonly other: RoleBucketStats;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Compact JSON snapshot for automation / MCP (llm-use `stats_snapshot` analog).
|
|
33
|
+
* Optional `frontier_savings_usd` is omitted when prices are unavailable (fail closed).
|
|
34
|
+
*/
|
|
35
|
+
export interface SessionStatsSnapshot {
|
|
36
|
+
readonly entry_count: number;
|
|
37
|
+
readonly total_cost_usd: number;
|
|
38
|
+
readonly mean_cost_usd: number | null;
|
|
39
|
+
readonly total_latency_ms: number;
|
|
40
|
+
readonly mean_latency_ms: number | null;
|
|
41
|
+
/** Share of entries with planning_delegate_path === 'delegate' (0–1), null if empty. */
|
|
42
|
+
readonly planning_delegate_share: number | null;
|
|
43
|
+
/** Share of entries with non-delegate path (direct / none / null). */
|
|
44
|
+
readonly direct_share: number | null;
|
|
45
|
+
/** Share classified as local (zero-tier) when distinguishable; null if none classified. */
|
|
46
|
+
readonly local_share: number | null;
|
|
47
|
+
/** Share classified as cloud when distinguishable; null if none classified. */
|
|
48
|
+
readonly cloud_share: number | null;
|
|
49
|
+
readonly role_cost: RoleCostBreakdown;
|
|
50
|
+
/**
|
|
51
|
+
* Estimated USD saved vs always-frontier baseline.
|
|
52
|
+
* Formula: sum over entries with token counts of
|
|
53
|
+
* max(0, tokens/1e6 * frontier_cost_per_1m - estimated_cost_usd).
|
|
54
|
+
* Omitted when frontier price inputs are missing (fail closed).
|
|
55
|
+
*/
|
|
56
|
+
readonly frontier_savings_usd?: number;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export interface AggregateSessionStatsOptions {
|
|
60
|
+
/**
|
|
61
|
+
* USD per 1M tokens for the always-frontier baseline.
|
|
62
|
+
* When absent / non-finite / ≤0, `frontier_savings_usd` is omitted.
|
|
63
|
+
*/
|
|
64
|
+
readonly frontier_cost_per_1m?: number;
|
|
65
|
+
/** Optional model_id → tier map (e.g. from fleet) for local vs cloud. */
|
|
66
|
+
readonly tier_by_model_id?: ReadonlyMap<string, Tier>;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** Mutually exclusive role for cost bucketing. */
|
|
70
|
+
export function classifyRoleCostBucket(entry: RoutingTelemetry): RoleCostBucket {
|
|
71
|
+
if (entry.planning_delegate_path === 'delegate') {
|
|
72
|
+
return 'planning_delegate';
|
|
73
|
+
}
|
|
74
|
+
if (entry.pin_reason != null) {
|
|
75
|
+
return 'primary';
|
|
76
|
+
}
|
|
77
|
+
return 'other';
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export function classifyDeployment(
|
|
81
|
+
entry: RoutingTelemetry,
|
|
82
|
+
tierByModelId?: ReadonlyMap<string, Tier>,
|
|
83
|
+
): DeploymentClass {
|
|
84
|
+
const fromFleet = tierByModelId?.get(entry.selected_model_id);
|
|
85
|
+
const tier = fromFleet ?? entry.tier_hint;
|
|
86
|
+
if (tier === 'zero-tier') {
|
|
87
|
+
return 'local';
|
|
88
|
+
}
|
|
89
|
+
if (tier === 'economical-cloud' || tier === 'frontier-cloud') {
|
|
90
|
+
return 'cloud';
|
|
91
|
+
}
|
|
92
|
+
return 'unknown';
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function isPlanningDelegate(path: PlanningDelegatePath | null): boolean {
|
|
96
|
+
return path === 'delegate';
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Resolve always-frontier cost/1M from fleet + optional catalog.
|
|
101
|
+
* Returns undefined when no positive frontier price is available (fail closed).
|
|
102
|
+
*/
|
|
103
|
+
export function resolveFrontierCostPer1M(
|
|
104
|
+
fleet?: readonly ModelProfile[],
|
|
105
|
+
catalog?: PriceCatalog | null,
|
|
106
|
+
): number | undefined {
|
|
107
|
+
const candidates: number[] = [];
|
|
108
|
+
|
|
109
|
+
if (fleet) {
|
|
110
|
+
for (const model of fleet) {
|
|
111
|
+
if (model.tier !== 'frontier-cloud') {
|
|
112
|
+
continue;
|
|
113
|
+
}
|
|
114
|
+
const cost = model.pricing.fallback_cost_per_1m;
|
|
115
|
+
if (Number.isFinite(cost) && cost > 0) {
|
|
116
|
+
candidates.push(cost);
|
|
117
|
+
}
|
|
118
|
+
if (catalog) {
|
|
119
|
+
const key = model.pricing.registry_key ?? model.id;
|
|
120
|
+
const fromCatalog =
|
|
121
|
+
catalog.user_overrides[key] ?? catalog.registry_snapshot[key];
|
|
122
|
+
if (fromCatalog !== undefined && Number.isFinite(fromCatalog) && fromCatalog > 0) {
|
|
123
|
+
candidates.push(fromCatalog);
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
if (candidates.length === 0) {
|
|
130
|
+
return undefined;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
// Upper-bound “always frontier” baseline: most expensive known frontier rate.
|
|
134
|
+
return Math.max(...candidates);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function buildTierMap(fleet?: readonly ModelProfile[]): ReadonlyMap<string, Tier> | undefined {
|
|
138
|
+
if (!fleet || fleet.length === 0) {
|
|
139
|
+
return undefined;
|
|
140
|
+
}
|
|
141
|
+
const map = new Map<string, Tier>();
|
|
142
|
+
for (const model of fleet) {
|
|
143
|
+
map.set(model.id, model.tier);
|
|
144
|
+
map.set(`${model.provider}/${model.id}`, model.tier);
|
|
145
|
+
}
|
|
146
|
+
return map;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* Pure aggregate over routing telemetry for operator stats.
|
|
151
|
+
* Does not read or emit prompt/message/tool bodies.
|
|
152
|
+
*/
|
|
153
|
+
export function aggregateSessionStats(
|
|
154
|
+
entries: readonly RoutingTelemetry[],
|
|
155
|
+
options: AggregateSessionStatsOptions = {},
|
|
156
|
+
): SessionStatsSnapshot {
|
|
157
|
+
const tierByModelId = options.tier_by_model_id;
|
|
158
|
+
const roleAccum = {
|
|
159
|
+
primary: { count: 0, total_cost_usd: 0 },
|
|
160
|
+
planning_delegate: { count: 0, total_cost_usd: 0 },
|
|
161
|
+
other: { count: 0, total_cost_usd: 0 },
|
|
162
|
+
};
|
|
163
|
+
|
|
164
|
+
let totalCost = 0;
|
|
165
|
+
let totalLatency = 0;
|
|
166
|
+
let delegateCount = 0;
|
|
167
|
+
let localCount = 0;
|
|
168
|
+
let cloudCount = 0;
|
|
169
|
+
let classifiedDeployment = 0;
|
|
170
|
+
|
|
171
|
+
for (const entry of entries) {
|
|
172
|
+
const cost = Number.isFinite(entry.estimated_cost_usd) ? entry.estimated_cost_usd : 0;
|
|
173
|
+
const latency = Number.isFinite(entry.routing_latency_ms) ? entry.routing_latency_ms : 0;
|
|
174
|
+
totalCost += cost;
|
|
175
|
+
totalLatency += latency;
|
|
176
|
+
|
|
177
|
+
if (isPlanningDelegate(entry.planning_delegate_path)) {
|
|
178
|
+
delegateCount += 1;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
const role = classifyRoleCostBucket(entry);
|
|
182
|
+
roleAccum[role].count += 1;
|
|
183
|
+
roleAccum[role].total_cost_usd += cost;
|
|
184
|
+
|
|
185
|
+
const deployment = classifyDeployment(entry, tierByModelId);
|
|
186
|
+
if (deployment === 'local') {
|
|
187
|
+
localCount += 1;
|
|
188
|
+
classifiedDeployment += 1;
|
|
189
|
+
} else if (deployment === 'cloud') {
|
|
190
|
+
cloudCount += 1;
|
|
191
|
+
classifiedDeployment += 1;
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
const n = entries.length;
|
|
196
|
+
const snapshot: SessionStatsSnapshot = {
|
|
197
|
+
entry_count: n,
|
|
198
|
+
total_cost_usd: totalCost,
|
|
199
|
+
mean_cost_usd: n > 0 ? totalCost / n : null,
|
|
200
|
+
total_latency_ms: totalLatency,
|
|
201
|
+
mean_latency_ms: n > 0 ? totalLatency / n : null,
|
|
202
|
+
planning_delegate_share: n > 0 ? delegateCount / n : null,
|
|
203
|
+
direct_share: n > 0 ? (n - delegateCount) / n : null,
|
|
204
|
+
local_share: classifiedDeployment > 0 ? localCount / classifiedDeployment : null,
|
|
205
|
+
cloud_share: classifiedDeployment > 0 ? cloudCount / classifiedDeployment : null,
|
|
206
|
+
role_cost: {
|
|
207
|
+
primary: { ...roleAccum.primary },
|
|
208
|
+
planning_delegate: { ...roleAccum.planning_delegate },
|
|
209
|
+
other: { ...roleAccum.other },
|
|
210
|
+
},
|
|
211
|
+
};
|
|
212
|
+
|
|
213
|
+
const frontierSavings = estimateFrontierSavingsUsd(entries, options.frontier_cost_per_1m);
|
|
214
|
+
if (frontierSavings !== undefined) {
|
|
215
|
+
return { ...snapshot, frontier_savings_usd: frontierSavings };
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
return snapshot;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
/**
|
|
222
|
+
* Optional vs-always-frontier savings. Returns undefined when price input is
|
|
223
|
+
* missing or non-positive (fail closed) or when no entry has token counts.
|
|
224
|
+
*/
|
|
225
|
+
export function estimateFrontierSavingsUsd(
|
|
226
|
+
entries: readonly RoutingTelemetry[],
|
|
227
|
+
frontierCostPer1M: number | undefined,
|
|
228
|
+
): number | undefined {
|
|
229
|
+
if (
|
|
230
|
+
frontierCostPer1M === undefined ||
|
|
231
|
+
!Number.isFinite(frontierCostPer1M) ||
|
|
232
|
+
frontierCostPer1M <= 0
|
|
233
|
+
) {
|
|
234
|
+
return undefined;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
let savings = 0;
|
|
238
|
+
let counted = 0;
|
|
239
|
+
|
|
240
|
+
for (const entry of entries) {
|
|
241
|
+
const tokens = entry.estimated_input_tokens;
|
|
242
|
+
if (tokens === null || !Number.isFinite(tokens) || tokens < 0) {
|
|
243
|
+
continue;
|
|
244
|
+
}
|
|
245
|
+
const actual = Number.isFinite(entry.estimated_cost_usd) ? entry.estimated_cost_usd : 0;
|
|
246
|
+
const frontierCost = (tokens / 1_000_000) * frontierCostPer1M;
|
|
247
|
+
savings += Math.max(0, frontierCost - actual);
|
|
248
|
+
counted += 1;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
if (counted === 0) {
|
|
252
|
+
return undefined;
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
return savings;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/** Convenience: aggregate with fleet/catalog-derived tier map + frontier price. */
|
|
259
|
+
export function aggregateSessionStatsFromFleet(
|
|
260
|
+
entries: readonly RoutingTelemetry[],
|
|
261
|
+
fleet?: readonly ModelProfile[],
|
|
262
|
+
catalog?: PriceCatalog | null,
|
|
263
|
+
): SessionStatsSnapshot {
|
|
264
|
+
const tierMap = buildTierMap(fleet);
|
|
265
|
+
const frontierCost = resolveFrontierCostPer1M(fleet, catalog);
|
|
266
|
+
return aggregateSessionStats(entries, {
|
|
267
|
+
...(tierMap ? { tier_by_model_id: tierMap } : {}),
|
|
268
|
+
...(frontierCost !== undefined ? { frontier_cost_per_1m: frontierCost } : {}),
|
|
269
|
+
});
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
/** Keys that must never appear on a stats snapshot (privacy). */
|
|
273
|
+
export const SESSION_STATS_FORBIDDEN_KEYS = [
|
|
274
|
+
'prompt',
|
|
275
|
+
'prompt_text',
|
|
276
|
+
'messages',
|
|
277
|
+
'content',
|
|
278
|
+
'tool_calls',
|
|
279
|
+
'tool_args',
|
|
280
|
+
'pepper',
|
|
281
|
+
] as const;
|
|
282
|
+
|
|
283
|
+
export function assertSessionStatsPrivacySafe(snapshot: SessionStatsSnapshot): void {
|
|
284
|
+
const json = JSON.stringify(snapshot);
|
|
285
|
+
for (const key of SESSION_STATS_FORBIDDEN_KEYS) {
|
|
286
|
+
if (json.includes(`"${key}"`)) {
|
|
287
|
+
throw new Error(`Stats snapshot contains forbidden privacy key: ${key}`);
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
}
|