pi-pignon 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +40 -0
- package/LICENSE +21 -0
- package/README.md +514 -0
- package/examples/pignon.json +35 -0
- package/package.json +68 -0
- package/schema/config.schema.json +446 -0
- package/src/compare.ts +109 -0
- package/src/config/defaults.ts +95 -0
- package/src/config/describe.ts +30 -0
- package/src/config/load.ts +445 -0
- package/src/config/migrate.ts +86 -0
- package/src/config/presets.ts +54 -0
- package/src/config/schema.ts +224 -0
- package/src/deciders/create.ts +102 -0
- package/src/deciders/jev.ts +226 -0
- package/src/deciders/laya-local.ts +718 -0
- package/src/deciders/laya-serve.ts +53 -0
- package/src/deciders/parse.ts +48 -0
- package/src/deciders/questions.ts +34 -0
- package/src/deciders/strategy.ts +225 -0
- package/src/deciders/types.ts +70 -0
- package/src/extension.ts +439 -0
- package/src/onboarding.ts +203 -0
- package/src/policy.ts +215 -0
- package/src/report.ts +171 -0
- package/src/router.ts +215 -0
- package/src/stats.ts +55 -0
- package/src/types.ts +341 -0
- package/src/ui.ts +153 -0
package/src/policy.ts
ADDED
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure routing policy.
|
|
3
|
+
*
|
|
4
|
+
* No I/O, no Pi dependencies — the only thing to unit-test.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { DEFAULT_CONFIG, DEFAULT_THRESHOLDS } from "./config/defaults.js";
|
|
8
|
+
import {
|
|
9
|
+
type Form,
|
|
10
|
+
type ModelSpec,
|
|
11
|
+
type PolicyInput,
|
|
12
|
+
type PolicyOutput,
|
|
13
|
+
type Price,
|
|
14
|
+
type Profile,
|
|
15
|
+
type RoutingDecision,
|
|
16
|
+
type RoutingTable,
|
|
17
|
+
FORMS,
|
|
18
|
+
} from "./types.js";
|
|
19
|
+
|
|
20
|
+
/** Derive the task form from a decision. */
|
|
21
|
+
export function formOf(
|
|
22
|
+
decision: RoutingDecision,
|
|
23
|
+
minConfidenceForm = DEFAULT_THRESHOLDS.minConfidenceForm,
|
|
24
|
+
): Form {
|
|
25
|
+
if (!decision.needsExploration && decision.explorationConfidence >= minConfidenceForm) {
|
|
26
|
+
return "direct";
|
|
27
|
+
}
|
|
28
|
+
return "exploration";
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Given a Laya decision, the current profile, and context metadata,
|
|
33
|
+
* decide whether to switch models and to which profile.
|
|
34
|
+
*
|
|
35
|
+
* Upgrades are quality-driven: only tier confidence gates them. Every other
|
|
36
|
+
* switch (downgrade, lateral, or moving in from a model outside the table)
|
|
37
|
+
* must also get past the cooldown and the switch-cost check, because it
|
|
38
|
+
* throws away the current prompt cache.
|
|
39
|
+
*
|
|
40
|
+
* Fail-open: any missing or ambiguous signal keeps the current model.
|
|
41
|
+
*/
|
|
42
|
+
export function decide(input: PolicyInput): PolicyOutput {
|
|
43
|
+
const { decision, current, contextTokens, promptsSinceSwitch, config = DEFAULT_CONFIG } = input;
|
|
44
|
+
const { table, thresholds } = config;
|
|
45
|
+
|
|
46
|
+
// Fail-open : no usable decision -> do nothing.
|
|
47
|
+
if (!decision || decision.tier === null) {
|
|
48
|
+
return { target: null, reason: "no decision" };
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const form = formOf(decision, thresholds.minConfidenceForm);
|
|
52
|
+
let rank = rankOf(table, decision.tier);
|
|
53
|
+
if (rank < 0) {
|
|
54
|
+
return { target: null, reason: `unknown tier ${decision.tier}` };
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// Some tiers (typically the cheapest) are too weak for the long tool loop
|
|
58
|
+
// of an exploration task, whatever its reasoning demand: move up to the
|
|
59
|
+
// first tier that takes it.
|
|
60
|
+
if (form === "exploration" && !table[rank]!.explorationAllowed) {
|
|
61
|
+
const next = table.findIndex((t, i) => i > rank && t.explorationAllowed);
|
|
62
|
+
if (next >= 0) rank = next;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const tier = table[rank]!.id;
|
|
66
|
+
const target: Profile = { tier, form };
|
|
67
|
+
const targetSpec = table[rank]!.models[form];
|
|
68
|
+
|
|
69
|
+
const currentRank = current ? rankOf(table, current.tier) : -1;
|
|
70
|
+
const currentSpec = currentRank >= 0 ? table[currentRank]!.models[current!.form] : undefined;
|
|
71
|
+
|
|
72
|
+
// Several cells may share a model: compare what would actually run.
|
|
73
|
+
if (currentSpec && sameSpec(currentSpec, targetSpec)) {
|
|
74
|
+
return { target: null, reason: "already on target model" };
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
const move = currentRank < 0
|
|
78
|
+
? "enter"
|
|
79
|
+
: rank > currentRank
|
|
80
|
+
? "upgrade"
|
|
81
|
+
: rank < currentRank
|
|
82
|
+
? "downgrade"
|
|
83
|
+
: "lateral";
|
|
84
|
+
const label = current ? `${current.tier} -> ${tier} (${form})` : `${tier}/${form}`;
|
|
85
|
+
|
|
86
|
+
if (move === "upgrade") {
|
|
87
|
+
if (decision.tierConfidence < thresholds.minConfidenceUpgrade) {
|
|
88
|
+
return {
|
|
89
|
+
target: null,
|
|
90
|
+
reason: `confidence ${decision.tierConfidence.toFixed(2)} < upgrade threshold`,
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
return { target, reason: `upgrade ${label}` };
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// Moving in from an unrouted model may be an upgrade or a downgrade: apply
|
|
97
|
+
// the stricter (downgrade) threshold.
|
|
98
|
+
if (move !== "lateral" && decision.tierConfidence < thresholds.minConfidenceDowngrade) {
|
|
99
|
+
const threshold = move === "enter" ? "entry" : "downgrade";
|
|
100
|
+
return {
|
|
101
|
+
target: null,
|
|
102
|
+
reason: `confidence ${decision.tierConfidence.toFixed(2)} < ${threshold} threshold`,
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// Hysteresis: do not flap between models on consecutive prompts.
|
|
107
|
+
if (promptsSinceSwitch !== undefined && promptsSinceSwitch < thresholds.minPromptsBetweenSwitches) {
|
|
108
|
+
return {
|
|
109
|
+
target: null,
|
|
110
|
+
reason: `cooldown: ${promptsSinceSwitch} prompt(s) since last switch`,
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
const contextK = `${Math.round(contextTokens / 1_000)}k`;
|
|
115
|
+
|
|
116
|
+
// Lateral switches are about fit, not price: only the flat guard applies.
|
|
117
|
+
if (move === "lateral") {
|
|
118
|
+
if (contextTokens > thresholds.cacheGuardTokens) {
|
|
119
|
+
return { target: null, reason: `lateral blocked: context ${contextK}` };
|
|
120
|
+
}
|
|
121
|
+
return { target, reason: `lateral ${current!.form} -> ${form}` };
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
const currentPrice = knownPrice(input.currentPrice);
|
|
125
|
+
const targetPrice = knownPrice(input.priceOf?.(targetSpec));
|
|
126
|
+
|
|
127
|
+
if (currentPrice && targetPrice) {
|
|
128
|
+
const payback = paybackRequests(
|
|
129
|
+
currentPrice,
|
|
130
|
+
targetPrice,
|
|
131
|
+
contextTokens,
|
|
132
|
+
thresholds.assumedOutputTokensPerRequest,
|
|
133
|
+
);
|
|
134
|
+
if (payback > thresholds.maxPaybackRequests) {
|
|
135
|
+
const detail = Number.isFinite(payback)
|
|
136
|
+
? `pays back in ${payback.toFixed(1)} requests > ${thresholds.maxPaybackRequests}`
|
|
137
|
+
: "target is not cheaper";
|
|
138
|
+
return { target: null, reason: `context ${contextK}: ${detail}` };
|
|
139
|
+
}
|
|
140
|
+
return { target, reason: `${move} ${label}, pays back in ${payback.toFixed(1)} requests` };
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
// Prices unknown: fall back to the flat context guard.
|
|
144
|
+
if (contextTokens > thresholds.cacheGuardTokens) {
|
|
145
|
+
return { target: null, reason: `context ${contextK}: cache protected` };
|
|
146
|
+
}
|
|
147
|
+
return {
|
|
148
|
+
target,
|
|
149
|
+
reason: move === "enter" ? `enter ${label} from unrouted model` : `downgrade ${label}`,
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* How many LLM requests a switch takes to pay for itself.
|
|
155
|
+
*
|
|
156
|
+
* The first request on the new model reads the whole context uncached (or
|
|
157
|
+
* writes it to cache, whichever is dearer) instead of at the cache-read rate.
|
|
158
|
+
* Each later request saves the difference in cache-read price on the context
|
|
159
|
+
* plus the difference in output price. Returns Infinity when nothing is saved.
|
|
160
|
+
*/
|
|
161
|
+
export function paybackRequests(
|
|
162
|
+
current: Price,
|
|
163
|
+
target: Price,
|
|
164
|
+
contextTokens: number,
|
|
165
|
+
outputTokensPerRequest: number,
|
|
166
|
+
): number {
|
|
167
|
+
const missPrice = Math.max(target.input, target.cacheWrite);
|
|
168
|
+
const premium = contextTokens * (missPrice - target.cacheRead);
|
|
169
|
+
const savingPerRequest =
|
|
170
|
+
contextTokens * (current.cacheRead - target.cacheRead) +
|
|
171
|
+
outputTokensPerRequest * (current.output - target.output);
|
|
172
|
+
if (savingPerRequest <= 0) return Infinity;
|
|
173
|
+
return premium / savingPerRequest;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/** Treat an all-zero price (common for unpriced registry entries) as unknown. */
|
|
177
|
+
function knownPrice(price: Price | undefined): Price | undefined {
|
|
178
|
+
if (!price) return undefined;
|
|
179
|
+
return price.input > 0 || price.output > 0 ? price : undefined;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
function sameSpec(a: ModelSpec, b: ModelSpec): boolean {
|
|
183
|
+
return a.provider === b.provider && a.modelId === b.modelId && a.thinking === b.thinking;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/** Position of a tier in the table (its rank), or -1 when it is not there. */
|
|
187
|
+
function rankOf(table: RoutingTable, tier: string): number {
|
|
188
|
+
return table.findIndex((t) => t.id === tier);
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/** The model a profile runs on, or undefined when its tier is not in the table. */
|
|
192
|
+
export function specOf(table: RoutingTable, profile: Profile): ModelSpec | undefined {
|
|
193
|
+
return table.find((t) => t.id === profile.tier)?.models[profile.form];
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/**
|
|
197
|
+
* Resolve a (provider, modelId) pair to a Profile by scanning the routing
|
|
198
|
+
* table. When several cells share the model, the first match is returned;
|
|
199
|
+
* `decide` compares specs, so which of those cells is picked does not matter.
|
|
200
|
+
*/
|
|
201
|
+
export function profileFromModel(
|
|
202
|
+
provider: string,
|
|
203
|
+
modelId: string,
|
|
204
|
+
table: RoutingTable,
|
|
205
|
+
): Profile | null {
|
|
206
|
+
for (const tier of table) {
|
|
207
|
+
for (const form of FORMS) {
|
|
208
|
+
const spec = tier.models[form];
|
|
209
|
+
if (spec.provider === provider && spec.modelId === modelId) {
|
|
210
|
+
return { tier: tier.id, form };
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
return null;
|
|
215
|
+
}
|
package/src/report.ts
ADDED
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One-shot reports (`/pignon doctor`, `config`, `log`, `/pignon-stats`).
|
|
3
|
+
*
|
|
4
|
+
* In the TUI a report opens in a dismissible overlay: no line cap (a
|
|
5
|
+
* string-array widget keeps only 10 lines) and nothing left above the editor
|
|
6
|
+
* once it is closed. RPC clients get a notification, print/JSON mode stderr.
|
|
7
|
+
*
|
|
8
|
+
* The overlay scrolls by itself: Pi composites overlays by calling
|
|
9
|
+
* `render(width)` and slicing to `maxHeight`, so a pi-tui `ScrollView` inside
|
|
10
|
+
* one never receives a viewport and would be cut off like the widget was.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
14
|
+
import { Key, matchesKey, truncateToWidth, visibleWidth } from "@earendil-works/pi-tui";
|
|
15
|
+
|
|
16
|
+
/** The subset of Pi's theme the overlay uses. */
|
|
17
|
+
export interface ReportTheme {
|
|
18
|
+
fg(color: "accent" | "border" | "muted" | "dim", text: string): string;
|
|
19
|
+
bold(text: string): string;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** Share of the terminal height the overlay may use. */
|
|
23
|
+
const HEIGHT_SHARE = 0.7;
|
|
24
|
+
/** Top and bottom border rows. */
|
|
25
|
+
const CHROME_ROWS = 2;
|
|
26
|
+
/** Border and padding columns around each line. */
|
|
27
|
+
const CHROME_COLUMNS = 4;
|
|
28
|
+
/** Overlay width while the lines are still computing, when they cannot be measured yet. */
|
|
29
|
+
const PENDING_WIDTH = "90%";
|
|
30
|
+
|
|
31
|
+
export class ReportOverlay {
|
|
32
|
+
private body: string[];
|
|
33
|
+
private offset = 0;
|
|
34
|
+
/** Body rows shown by the last render; paging moves by this much. */
|
|
35
|
+
private visibleRows = 1;
|
|
36
|
+
|
|
37
|
+
constructor(
|
|
38
|
+
readonly title: string,
|
|
39
|
+
lines: string[],
|
|
40
|
+
private readonly theme: ReportTheme,
|
|
41
|
+
/** Terminal height, read on every render so a resize is picked up. */
|
|
42
|
+
private readonly terminalRows: () => number,
|
|
43
|
+
private readonly requestRender: () => void,
|
|
44
|
+
private readonly close: () => void,
|
|
45
|
+
) {
|
|
46
|
+
this.body = lines;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
get lines(): readonly string[] {
|
|
50
|
+
return this.body;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
setLines(lines: string[]): void {
|
|
54
|
+
this.body = lines;
|
|
55
|
+
this.offset = 0;
|
|
56
|
+
this.requestRender();
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
handleInput(data: string): void {
|
|
60
|
+
const page = Math.max(1, this.visibleRows - 1);
|
|
61
|
+
if (matchesKey(data, Key.escape) || matchesKey(data, Key.enter) || matchesKey(data, "q")) {
|
|
62
|
+
this.close();
|
|
63
|
+
return;
|
|
64
|
+
}
|
|
65
|
+
if (matchesKey(data, Key.up) || matchesKey(data, "k")) this.scrollTo(this.offset - 1);
|
|
66
|
+
else if (matchesKey(data, Key.down) || matchesKey(data, "j")) this.scrollTo(this.offset + 1);
|
|
67
|
+
else if (matchesKey(data, Key.pageUp)) this.scrollTo(this.offset - page);
|
|
68
|
+
else if (matchesKey(data, Key.pageDown) || matchesKey(data, Key.space)) this.scrollTo(this.offset + page);
|
|
69
|
+
else if (matchesKey(data, Key.home) || matchesKey(data, "g")) this.scrollTo(0);
|
|
70
|
+
else if (matchesKey(data, Key.end) || matchesKey(data, "shift+g")) this.scrollTo(Number.MAX_SAFE_INTEGER);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
render(width: number): string[] {
|
|
74
|
+
const { theme } = this;
|
|
75
|
+
const border = (s: string) => theme.fg("border", s);
|
|
76
|
+
const inner = Math.max(1, width - CHROME_COLUMNS);
|
|
77
|
+
|
|
78
|
+
const maxRows = Math.max(1, Math.floor(this.terminalRows() * HEIGHT_SHARE) - CHROME_ROWS);
|
|
79
|
+
this.visibleRows = Math.min(this.body.length, maxRows);
|
|
80
|
+
this.offset = this.clamp(this.offset);
|
|
81
|
+
const shown = this.body.slice(this.offset, this.offset + this.visibleRows);
|
|
82
|
+
|
|
83
|
+
const scrolls = this.body.length > this.visibleRows;
|
|
84
|
+
const position = scrolls ? `${this.offset + 1}–${this.offset + shown.length}/${this.body.length}` : "";
|
|
85
|
+
const hint = scrolls ? "↑↓ PgUp/PgDn scroll · Esc close" : "Esc close";
|
|
86
|
+
|
|
87
|
+
return [
|
|
88
|
+
this.rule("╭", "╮", this.title && theme.fg("accent", theme.bold(this.title)), "", width),
|
|
89
|
+
...shown.map((line) => `${border("│")} ${truncateToWidth(line, inner, "…", true)} ${border("│")}`),
|
|
90
|
+
this.rule("╰", "╯", theme.fg("dim", hint), position && theme.fg("dim", position), width),
|
|
91
|
+
];
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
invalidate(): void {}
|
|
95
|
+
|
|
96
|
+
/** `╭─ left ───── right ─╮`, with the labels dropped if they do not fit. */
|
|
97
|
+
private rule(open: string, close: string, left: string, right: string, width: number): string {
|
|
98
|
+
const border = (s: string) => this.theme.fg("border", s);
|
|
99
|
+
const labelled = (s: string) => (s ? ` ${s} ` : "");
|
|
100
|
+
let l = labelled(left);
|
|
101
|
+
let r = labelled(right);
|
|
102
|
+
if (visibleWidth(l) + visibleWidth(r) + CHROME_COLUMNS > width) r = "";
|
|
103
|
+
if (visibleWidth(l) + CHROME_COLUMNS > width) l = "";
|
|
104
|
+
const fill = Math.max(0, width - CHROME_COLUMNS - visibleWidth(l) - visibleWidth(r));
|
|
105
|
+
return border(`${open}─`) + l + border("─".repeat(fill)) + r + border(`─${close}`);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
private scrollTo(offset: number): void {
|
|
109
|
+
const next = this.clamp(offset);
|
|
110
|
+
if (next === this.offset) return;
|
|
111
|
+
this.offset = next;
|
|
112
|
+
this.requestRender();
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
private clamp(offset: number): number {
|
|
116
|
+
return Math.max(0, Math.min(offset, this.body.length - this.visibleRows));
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Columns that show the widest line (or the title) whole. Pi resolves overlay
|
|
122
|
+
* options once, when the overlay opens, and clamps them to the terminal.
|
|
123
|
+
*/
|
|
124
|
+
export function reportWidth(title: string, lines: readonly string[]): number {
|
|
125
|
+
return Math.max(visibleWidth(title) + 6, ...lines.map(visibleWidth)) + CHROME_COLUMNS;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Show a report until the user closes it. `lines` may still be computing
|
|
130
|
+
* (doctor probes the deciders): the overlay opens at once and fills in.
|
|
131
|
+
*/
|
|
132
|
+
export async function showReport(
|
|
133
|
+
ctx: ExtensionContext,
|
|
134
|
+
title: string,
|
|
135
|
+
lines: string[] | Promise<string[]>,
|
|
136
|
+
): Promise<void> {
|
|
137
|
+
const settled = Promise.resolve(lines).catch((err: unknown) => [
|
|
138
|
+
` ✗ ${err instanceof Error ? err.message : String(err)}`,
|
|
139
|
+
]);
|
|
140
|
+
|
|
141
|
+
if (ctx.mode !== "tui") {
|
|
142
|
+
const text = [title, ...(await settled)].join("\n");
|
|
143
|
+
if (ctx.hasUI) ctx.ui.notify(text, "info"); // RPC: custom components are not forwarded
|
|
144
|
+
else process.stderr.write(`${text}\n`); // keeps stdout parseable in JSON mode
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
await ctx.ui.custom<void>(
|
|
149
|
+
(tui, theme, _keybindings, done) => {
|
|
150
|
+
const overlay = new ReportOverlay(
|
|
151
|
+
title,
|
|
152
|
+
Array.isArray(lines) ? lines : [" running checks…"],
|
|
153
|
+
theme,
|
|
154
|
+
() => tui.terminal.rows,
|
|
155
|
+
() => tui.requestRender(),
|
|
156
|
+
() => done(undefined),
|
|
157
|
+
);
|
|
158
|
+
if (!Array.isArray(lines)) void settled.then((result) => overlay.setLines(result));
|
|
159
|
+
return overlay;
|
|
160
|
+
},
|
|
161
|
+
{
|
|
162
|
+
overlay: true,
|
|
163
|
+
overlayOptions: {
|
|
164
|
+
width: Array.isArray(lines) ? reportWidth(title, lines) : PENDING_WIDTH,
|
|
165
|
+
minWidth: 40,
|
|
166
|
+
anchor: "center",
|
|
167
|
+
margin: 1,
|
|
168
|
+
},
|
|
169
|
+
},
|
|
170
|
+
);
|
|
171
|
+
}
|
package/src/router.ts
ADDED
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-prompt routing: ask the decider, run the policy, apply the verdict.
|
|
3
|
+
*
|
|
4
|
+
* Pi-free: everything it needs from the host (current model, registry, model
|
|
5
|
+
* switching, UI feedback) goes through `RouterHost`, so it can be tested with
|
|
6
|
+
* plain fakes.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { createHash } from "node:crypto";
|
|
10
|
+
|
|
11
|
+
import { parseDecision } from "./deciders/parse.js";
|
|
12
|
+
import { buildQuestions } from "./deciders/questions.js";
|
|
13
|
+
import type { Decider } from "./deciders/types.js";
|
|
14
|
+
import { decide, formOf, profileFromModel, specOf } from "./policy.js";
|
|
15
|
+
import type {
|
|
16
|
+
DeciderAttempt,
|
|
17
|
+
PolicyOutput,
|
|
18
|
+
Price,
|
|
19
|
+
Profile,
|
|
20
|
+
RouterConfig,
|
|
21
|
+
RouterLogEntry,
|
|
22
|
+
RouterMode,
|
|
23
|
+
RoutingDecision,
|
|
24
|
+
ThinkingLevel,
|
|
25
|
+
} from "./types.js";
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Laya reads at most ~320 tokens of the prompt (512-token window minus the
|
|
29
|
+
* question header) from the start and ignores the rest, so sending more only
|
|
30
|
+
* costs tokenization time (and, for remote deciders, money and privacy).
|
|
31
|
+
*/
|
|
32
|
+
export const MAX_PROMPT_CHARS = 4_000;
|
|
33
|
+
|
|
34
|
+
/** The part of a host model the router reads. */
|
|
35
|
+
export interface HostModel {
|
|
36
|
+
provider: string;
|
|
37
|
+
id: string;
|
|
38
|
+
cost?: Price;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** What the router needs from the agent it runs in (Pi, or a test fake). */
|
|
42
|
+
export interface RouterHost<M extends HostModel = HostModel> {
|
|
43
|
+
/** Model the prompt would run on now. */
|
|
44
|
+
readonly model: M | undefined;
|
|
45
|
+
readonly contextTokens: number;
|
|
46
|
+
/** Cancellation of the current agent run. */
|
|
47
|
+
readonly signal: AbortSignal | undefined;
|
|
48
|
+
findModel(provider: string, modelId: string): M | undefined;
|
|
49
|
+
/** Switch model without the switch being taken for a manual pin. Resolves false without auth. */
|
|
50
|
+
switchModel(model: M): Promise<boolean>;
|
|
51
|
+
setThinkingLevel(level: ThinkingLevel): void;
|
|
52
|
+
status(text: string): void;
|
|
53
|
+
notify(text: string, level: "info" | "warning" | "error"): void;
|
|
54
|
+
showDeciding(deciderModel: string): void;
|
|
55
|
+
hideDeciding(): void;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export interface RoutePromptOptions {
|
|
59
|
+
decider: Decider;
|
|
60
|
+
config: RouterConfig;
|
|
61
|
+
mode: Exclude<RouterMode, "off">;
|
|
62
|
+
/** Prompts since the router last switched models; undefined if it never has. */
|
|
63
|
+
promptsSinceSwitch: number | undefined;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export interface RouteResult {
|
|
67
|
+
/** Whether the model was switched. */
|
|
68
|
+
applied: boolean;
|
|
69
|
+
entry: RouterLogEntry;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** Ask the decider about one prompt and apply the verdict. Never throws. */
|
|
73
|
+
export async function routePrompt<M extends HostModel>(
|
|
74
|
+
host: RouterHost<M>,
|
|
75
|
+
options: RoutePromptOptions,
|
|
76
|
+
prompt: string,
|
|
77
|
+
): Promise<RouteResult> {
|
|
78
|
+
const { decider, config, mode, promptsSinceSwitch } = options;
|
|
79
|
+
const { table, thresholds } = config;
|
|
80
|
+
const model = host.model;
|
|
81
|
+
const current = model ? profileFromModel(model.provider, model.id, table) : null;
|
|
82
|
+
const contextTokens = host.contextTokens;
|
|
83
|
+
const currentModel = model ? `${model.provider}/${model.id}` : undefined;
|
|
84
|
+
const entryBase = { mode, config, decider, currentModel, prompt, current, contextTokens };
|
|
85
|
+
|
|
86
|
+
try {
|
|
87
|
+
host.status("pignon is deciding...");
|
|
88
|
+
host.showDeciding(decider.model ?? "unknown");
|
|
89
|
+
let result;
|
|
90
|
+
try {
|
|
91
|
+
result = await decider.decide(
|
|
92
|
+
{ text: prompt.slice(0, MAX_PROMPT_CHARS), questions: buildQuestions(config) },
|
|
93
|
+
host.signal,
|
|
94
|
+
);
|
|
95
|
+
} finally {
|
|
96
|
+
host.hideDeciding();
|
|
97
|
+
}
|
|
98
|
+
const decision = parseDecision(result.answers, result.latencyMs, config);
|
|
99
|
+
|
|
100
|
+
const verdict = decide({
|
|
101
|
+
decision,
|
|
102
|
+
current,
|
|
103
|
+
contextTokens,
|
|
104
|
+
promptsSinceSwitch,
|
|
105
|
+
currentPrice: model?.cost,
|
|
106
|
+
priceOf: (spec) => host.findModel(spec.provider, spec.modelId)?.cost,
|
|
107
|
+
config,
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
let applied = false;
|
|
111
|
+
if (mode === "live" && verdict.target) {
|
|
112
|
+
const spec = specOf(table, verdict.target)!;
|
|
113
|
+
const target = host.findModel(spec.provider, spec.modelId);
|
|
114
|
+
if (!target) {
|
|
115
|
+
host.notify(`pignon: ${spec.provider}/${spec.modelId} is not in the model registry`, "warning");
|
|
116
|
+
} else if (await host.switchModel(target)) {
|
|
117
|
+
host.setThinkingLevel(spec.thinking);
|
|
118
|
+
applied = true;
|
|
119
|
+
} else {
|
|
120
|
+
host.notify(`pignon: no auth for ${spec.provider}/${spec.modelId}`, "warning");
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
const entry = buildLogEntry({
|
|
125
|
+
...entryBase,
|
|
126
|
+
deciderId: result.deciderId,
|
|
127
|
+
remote: result.remote ?? decider.remote,
|
|
128
|
+
deciderModel: result.model,
|
|
129
|
+
...(result.costUsd !== undefined ? { costUsd: result.costUsd } : {}),
|
|
130
|
+
...(result.attempts ? { attempts: result.attempts } : {}),
|
|
131
|
+
decision,
|
|
132
|
+
verdict,
|
|
133
|
+
applied,
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
const badge = mode === "live" ? (applied ? "⚡" : "·") : "👁";
|
|
137
|
+
const label = `${decision.tier ?? "?"}/${formOf(decision, thresholds.minConfidenceForm)} p=${decision.tierConfidence.toFixed(2)}`;
|
|
138
|
+
host.status(`pignon ${badge} ${label} — ${verdict.reason}`);
|
|
139
|
+
return { applied, entry };
|
|
140
|
+
} catch (err) {
|
|
141
|
+
const error = err instanceof Error ? err.message : String(err);
|
|
142
|
+
host.status(`pignon ✗ ${error.slice(0, 80)}`);
|
|
143
|
+
const entry = buildLogEntry({
|
|
144
|
+
...entryBase,
|
|
145
|
+
deciderModel: decider.model ?? "unknown",
|
|
146
|
+
decision: null,
|
|
147
|
+
verdict: { target: null, reason: "error" },
|
|
148
|
+
applied: false,
|
|
149
|
+
error,
|
|
150
|
+
});
|
|
151
|
+
return { applied: false, entry };
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
// ---------------------------------------------------------------------------
|
|
156
|
+
// Log entries
|
|
157
|
+
// ---------------------------------------------------------------------------
|
|
158
|
+
|
|
159
|
+
/** Short, stable fingerprint of a prompt; the text itself is never stored. */
|
|
160
|
+
export function hashPrompt(prompt: string): string {
|
|
161
|
+
return createHash("sha256").update(prompt).digest("hex").slice(0, 16);
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
interface LogEntryInput {
|
|
165
|
+
mode: RouterMode;
|
|
166
|
+
config: RouterConfig;
|
|
167
|
+
decider: Decider;
|
|
168
|
+
/** Decider that answered, when it differs from `decider` (a strategy). */
|
|
169
|
+
deciderId?: string;
|
|
170
|
+
remote?: boolean;
|
|
171
|
+
costUsd?: number;
|
|
172
|
+
attempts?: DeciderAttempt[];
|
|
173
|
+
currentModel: string | undefined;
|
|
174
|
+
deciderModel: string;
|
|
175
|
+
prompt: string;
|
|
176
|
+
decision: RoutingDecision | null;
|
|
177
|
+
current: Profile | null;
|
|
178
|
+
contextTokens: number;
|
|
179
|
+
verdict: PolicyOutput;
|
|
180
|
+
applied: boolean;
|
|
181
|
+
error?: string;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
function buildLogEntry(input: LogEntryInput): RouterLogEntry {
|
|
185
|
+
const { decision, current, verdict } = input;
|
|
186
|
+
const spec = verdict.target ? specOf(input.config.table, verdict.target) : undefined;
|
|
187
|
+
return {
|
|
188
|
+
ts: Date.now(),
|
|
189
|
+
mode: input.mode,
|
|
190
|
+
decider: input.deciderId ?? input.decider.id,
|
|
191
|
+
remote: input.remote ?? input.decider.remote,
|
|
192
|
+
...(input.costUsd !== undefined ? { costUsd: input.costUsd } : {}),
|
|
193
|
+
...(input.attempts ? { attempts: input.attempts } : {}),
|
|
194
|
+
deciderModel: input.deciderModel,
|
|
195
|
+
questionsVersion: input.config.questions.version,
|
|
196
|
+
promptHash: hashPrompt(input.prompt),
|
|
197
|
+
promptLength: input.prompt.length,
|
|
198
|
+
tier: decision?.tier ?? null,
|
|
199
|
+
tierConfidence: decision?.tierConfidence ?? null,
|
|
200
|
+
needsExploration: decision?.needsExploration ?? null,
|
|
201
|
+
explorationConfidence: decision?.explorationConfidence ?? null,
|
|
202
|
+
form: decision ? formOf(decision, input.config.thresholds.minConfidenceForm) : null,
|
|
203
|
+
latencyMs: decision?.latencyMs ?? null,
|
|
204
|
+
currentTier: current?.tier ?? null,
|
|
205
|
+
currentForm: current?.form ?? null,
|
|
206
|
+
...(input.currentModel !== undefined ? { currentModel: input.currentModel } : {}),
|
|
207
|
+
contextTokens: input.contextTokens,
|
|
208
|
+
targetTier: verdict.target?.tier ?? null,
|
|
209
|
+
targetForm: verdict.target?.form ?? null,
|
|
210
|
+
...(spec ? { targetModel: `${spec.provider}/${spec.modelId}`, targetThinking: spec.thinking } : {}),
|
|
211
|
+
reason: verdict.reason,
|
|
212
|
+
applied: input.applied,
|
|
213
|
+
...(input.error !== undefined ? { error: input.error } : {}),
|
|
214
|
+
};
|
|
215
|
+
}
|
package/src/stats.ts
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `/pignon-stats`: tier x form x confidence histogram over a session's decisions.
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import { DEFAULT_CONFIG } from "./config/defaults.js";
|
|
6
|
+
import { type RouterLogEntry, type RoutingTable, FORMS } from "./types.js";
|
|
7
|
+
|
|
8
|
+
/** Rows follow the table; decisions for tiers no longer in it are not counted. */
|
|
9
|
+
export function buildStatsLines(rows: RouterLogEntry[], table: RoutingTable = DEFAULT_CONFIG.table): string[] {
|
|
10
|
+
const buckets = ["<0.5", "0.5-0.7", "0.7-0.85", "0.85-0.95", ">=0.95"];
|
|
11
|
+
const bucketOf = (c: number) =>
|
|
12
|
+
c < 0.5 ? 0 : c < 0.7 ? 1 : c < 0.85 ? 2 : c < 0.95 ? 3 : 4;
|
|
13
|
+
|
|
14
|
+
const grid: Record<string, number[]> = {};
|
|
15
|
+
for (const { id } of table) {
|
|
16
|
+
for (const form of FORMS) {
|
|
17
|
+
grid[`${id}/${form}`] = [0, 0, 0, 0, 0];
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
let applied = 0;
|
|
22
|
+
const latencies: number[] = [];
|
|
23
|
+
const byDecider = new Map<string, number>();
|
|
24
|
+
let cost = 0;
|
|
25
|
+
|
|
26
|
+
for (const row of rows) {
|
|
27
|
+
const cell = row.tier && row.form ? `${row.tier}/${row.form}` : null;
|
|
28
|
+
if (cell && grid[cell]) grid[cell][bucketOf(row.tierConfidence ?? 0)]++;
|
|
29
|
+
// Failed decisions have no latency; counting them as 0 ms skews the mean.
|
|
30
|
+
if (typeof row.latencyMs === "number") latencies.push(row.latencyMs);
|
|
31
|
+
if (row.applied) applied++;
|
|
32
|
+
if (row.decider) byDecider.set(row.decider, (byDecider.get(row.decider) ?? 0) + 1);
|
|
33
|
+
cost += row.costUsd ?? 0;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
const avgLatency = latencies.length
|
|
37
|
+
? `${Math.round(latencies.reduce((a, b) => a + b, 0) / latencies.length)} ms`
|
|
38
|
+
: "n/a";
|
|
39
|
+
|
|
40
|
+
return [
|
|
41
|
+
`pignon — ${rows.length} decisions, ${applied} applied`,
|
|
42
|
+
`${"profile".padEnd(22)}${buckets.map((b) => b.padStart(9)).join("")}`,
|
|
43
|
+
...Object.entries(grid).map(
|
|
44
|
+
([cell, counts]) =>
|
|
45
|
+
`${cell.padEnd(22)}${counts.map((n) => String(n).padStart(9)).join("")}`,
|
|
46
|
+
),
|
|
47
|
+
`avg latency ${avgLatency}`,
|
|
48
|
+
...(byDecider.size > 0
|
|
49
|
+
? [
|
|
50
|
+
`deciders ${[...byDecider].map(([id, n]) => `${id} ${n}`).join(" · ")}` +
|
|
51
|
+
(cost > 0 ? ` · cost $${cost.toFixed(5)}` : ""),
|
|
52
|
+
]
|
|
53
|
+
: []),
|
|
54
|
+
];
|
|
55
|
+
}
|