pi-pignon 0.1.1

package/package.json

@@ -0,0 +1,68 @@
+{
+  "name": "pi-pignon",
+  "version": "0.1.1",
+  "description": "Pi coding agent extension that shifts to the right LLM for each prompt, using a local (Laya) or remote (Jev) decision model to judge task difficulty",
+  "keywords": [
+    "pi-package",
+    "pi-extension",
+    "pi-coding-agent",
+    "llm-router",
+    "model-routing",
+    "llm",
+    "laya",
+    "jev"
+  ],
+  "homepage": "https://github.com/siiick/pi-pignon#readme",
+  "bugs": {
+    "url": "https://github.com/siiick/pi-pignon/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/siiick/pi-pignon.git"
+  },
+  "license": "MIT",
+  "author": "Nicolas Chaintron",
+  "type": "module",
+  "files": [
+    "src",
+    "schema",
+    "examples",
+    "CHANGELOG.md"
+  ],
+  "engines": {
+    "node": ">=22.19"
+  },
+  "scripts": {
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "typecheck": "tsc --noEmit",
+    "clean": "rm -rf dist node_modules/.vitest",
+    "build": "echo 'nothing to build (TypeScript runs via jiti)'",
+    "check": "tsc --noEmit && vitest run && npm run test:worker",
+    "test:worker": "cd worker && .venv/bin/python -m unittest",
+    "schema": "UPDATE_SCHEMA=1 vitest run tests/schema.test.ts",
+    "test:live": "PIGNON_LIVE=1 vitest run tests/live",
+    "prepublishOnly": "npm run check"
+  },
+  "pi": {
+    "extensions": [
+      "./src/extension.ts"
+    ]
+  },
+  "dependencies": {
+    "@typesafe-ai/sdk": "^0.6.0"
+  },
+  "peerDependencies": {
+    "@earendil-works/pi-coding-agent": "*",
+    "@earendil-works/pi-tui": "*",
+    "typebox": "*"
+  },
+  "devDependencies": {
+    "@earendil-works/pi-coding-agent": "^0.87.1",
+    "@earendil-works/pi-tui": "^0.87.1",
+    "@types/node": "^22.0.0",
+    "typescript": "^5.6.0",
+    "vitest": "^2.1.0",
+    "typebox": "^1.3.34"
+  }
+}

package/schema/config.schema.json

@@ -0,0 +1,446 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://raw.githubusercontent.com/siiick/pi-pignon/main/schema/config.schema.json",
+  "type": "object",
+  "properties": {
+    "$schema": {
+      "type": "string"
+    },
+    "version": {
+      "type": "number",
+      "const": 2,
+      "description": "Config format version."
+    },
+    "extends": {
+      "enum": [
+        "openrouter",
+        "anthropic",
+        "openai"
+      ],
+      "description": "Model preset for the built-in names, applied before `models`: `openrouter` (OpenRouter: DeepSeek flash models, GLM for reasoning, HY4 as agent (the built-in table)); `anthropic` (Anthropic: Haiku for small edits, Sonnet for the rest, Opus for hard reasoning); `openai` (OpenAI: GPT-6 Luna for small edits, GPT-5.6 Terra for the rest, GPT-6 Sol for reasoning, Codex as agent)."
+    },
+    "deciders": {
+      "type": "array",
+      "items": {
+        "anyOf": [
+          {
+            "type": "object",
+            "required": [
+              "type"
+            ],
+            "properties": {
+              "type": {
+                "type": "string",
+                "const": "laya-serve",
+                "description": "A Laya model served by the official laya-serve (`pip install \"laya[serve]\"`). Runs on your machine unless url points elsewhere."
+              },
+              "url": {
+                "type": "string",
+                "pattern": "^https?://",
+                "description": "Server root. Default: http://127.0.0.1:8000 (laya-serve's default port)."
+              },
+              "model": {
+                "type": "string",
+                "minLength": 1,
+                "description": "Laya checkpoint: `english`, `multilingual` or `typed-decisions`. Default: chosen by the server from the prompt's language."
+              },
+              "apiKeyEnv": {
+                "type": "string",
+                "minLength": 1,
+                "description": "Environment variable holding the server's key, when it was started with LAYA_API_KEY. Default: no key."
+              },
+              "timeoutMs": {
+                "type": "number",
+                "exclusiveMinimum": 0,
+                "description": "Timeout for one decision, in milliseconds. Default: 1500."
+              }
+            },
+            "additionalProperties": false
+          },
+          {
+            "type": "object",
+            "required": [
+              "type"
+            ],
+            "properties": {
+              "type": {
+                "type": "string",
+                "const": "laya-local",
+                "description": "Experimental: pignon's own Laya worker (Apple Silicon, laya-mlx), installed from the repository's worker/ directory."
+              },
+              "timeoutMs": {
+                "type": "number",
+                "exclusiveMinimum": 0,
+                "description": "Timeout for one decision, in milliseconds. Default: thresholds.layaTimeoutMs."
+              },
+              "command": {
+                "type": "array",
+                "items": {
+                  "type": "string",
+                  "minLength": 1
+                },
+                "minItems": 1,
+                "description": "Command that starts the worker, e.g. [\"uv\", \"run\", \"--project\", \"/path/to/pignon/worker\", \"pignon-laya\"]. Default: found automatically (LAYA_PYTHON, a source checkout, then pignon-laya on PATH)."
+              }
+            },
+            "additionalProperties": false
+          },
+          {
+            "type": "object",
+            "required": [
+              "type"
+            ],
+            "properties": {
+              "type": {
+                "type": "string",
+                "const": "jev",
+                "description": "TypeSafe's hosted Jev model. Sends each routed prompt (first 4000 characters) to the API."
+              },
+              "model": {
+                "type": "string",
+                "minLength": 1,
+                "description": "Jev model to pin, e.g. `jev-1.13.0`. Default: jev-latest."
+              },
+              "baseURL": {
+                "type": "string",
+                "minLength": 1,
+                "description": "API root. `https://openrouter.ai/api` goes through OpenRouter. Default: TypeSafe."
+              },
+              "apiKeyEnv": {
+                "type": "string",
+                "minLength": 1,
+                "description": "Environment variable holding the API key. Default: TYPESAFE_API_KEY."
+              },
+              "timeoutMs": {
+                "type": "number",
+                "exclusiveMinimum": 0,
+                "description": "Timeout for one decision, in milliseconds. Default: 1500."
+              },
+              "maxRetries": {
+                "type": "integer",
+                "minimum": 0,
+                "maximum": 3,
+                "description": "Retries after a failed attempt. Default: 0."
+              }
+            },
+            "additionalProperties": false
+          }
+        ]
+      },
+      "minItems": 1,
+      "maxItems": 4,
+      "description": "Decision models, in the order to try them. Default: laya-local when its experimental worker is installed, else jev when TYPESAFE_API_KEY is set. /pignon init adds laya-serve when it is running."
+    },
+    "strategy": {
+      "type": "object",
+      "properties": {
+        "mode": {
+          "enum": [
+            "sequential",
+            "parallel"
+          ],
+          "description": "`sequential` (default): ask the deciders in order until one is confident enough. `parallel`: ask them all at once, e.g. to compare them."
+        },
+        "escalateBelow": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1,
+          "description": "Sequential: ask the next decider when tier confidence is below this. Default 0.75."
+        },
+        "pick": {
+          "enum": [
+            "most-confident",
+            "first"
+          ],
+          "description": "Parallel: route on the most confident answer (default), or on the first decider in the list that answered (the others are only recorded)."
+        },
+        "budgetMs": {
+          "type": "number",
+          "exclusiveMinimum": 0,
+          "description": "Wall-time limit for one decision, all deciders included. Default 3000."
+        }
+      },
+      "additionalProperties": false,
+      "description": "How several deciders are combined."
+    },
+    "models": {
+      "type": "object",
+      "patternProperties": {
+        "^[A-Za-z0-9][A-Za-z0-9._-]*$": {
+          "type": "object",
+          "required": [
+            "provider",
+            "modelId",
+            "thinking"
+          ],
+          "properties": {
+            "provider": {
+              "type": "string",
+              "minLength": 1,
+              "description": "Pi provider id, e.g. `openrouter` or `anthropic`."
+            },
+            "modelId": {
+              "type": "string",
+              "minLength": 1,
+              "description": "Model id within the provider. Check with `pi --list-models`."
+            },
+            "thinking": {
+              "enum": [
+                "off",
+                "low",
+                "medium",
+                "high",
+                "xhigh"
+              ],
+              "description": "Pi thinking level to set with the model. Pi clamps it to what the model supports."
+            }
+          },
+          "additionalProperties": false
+        }
+      },
+      "description": "Named models, referenced from `tiers`. Merged over the built-in names (fast, balanced, reasoner, agent)."
+    },
+    "tiers": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": [
+          "id",
+          "criterion"
+        ],
+        "properties": {
+          "id": {
+            "type": "string",
+            "pattern": "^[a-z][a-z0-9_-]*$",
+            "description": "Tier name, shown on decision cards. Lowercase letters, digits, `-` and `_`."
+          },
+          "criterion": {
+            "type": "string",
+            "minLength": 1,
+            "description": "How to recognize a task of this tier. This text is what the decision model reads."
+          },
+          "model": {
+            "anyOf": [
+              {
+                "type": "string",
+                "minLength": 1,
+                "description": "Name of an entry in `models`."
+              },
+              {
+                "type": "object",
+                "required": [
+                  "provider",
+                  "modelId",
+                  "thinking"
+                ],
+                "properties": {
+                  "provider": {
+                    "type": "string",
+                    "minLength": 1,
+                    "description": "Pi provider id, e.g. `openrouter` or `anthropic`."
+                  },
+                  "modelId": {
+                    "type": "string",
+                    "minLength": 1,
+                    "description": "Model id within the provider. Check with `pi --list-models`."
+                  },
+                  "thinking": {
+                    "enum": [
+                      "off",
+                      "low",
+                      "medium",
+                      "high",
+                      "xhigh"
+                    ],
+                    "description": "Pi thinking level to set with the model. Pi clamps it to what the model supports."
+                  }
+                },
+                "additionalProperties": false
+              }
+            ],
+            "description": "Model for every task of this tier. Use it, or both `direct` and `exploration`."
+          },
+          "direct": {
+            "anyOf": [
+              {
+                "type": "string",
+                "minLength": 1,
+                "description": "Name of an entry in `models`."
+              },
+              {
+                "type": "object",
+                "required": [
+                  "provider",
+                  "modelId",
+                  "thinking"
+                ],
+                "properties": {
+                  "provider": {
+                    "type": "string",
+                    "minLength": 1,
+                    "description": "Pi provider id, e.g. `openrouter` or `anthropic`."
+                  },
+                  "modelId": {
+                    "type": "string",
+                    "minLength": 1,
+                    "description": "Model id within the provider. Check with `pi --list-models`."
+                  },
+                  "thinking": {
+                    "enum": [
+                      "off",
+                      "low",
+                      "medium",
+                      "high",
+                      "xhigh"
+                    ],
+                    "description": "Pi thinking level to set with the model. Pi clamps it to what the model supports."
+                  }
+                },
+                "additionalProperties": false
+              }
+            ],
+            "description": "Model for tasks that can be done without exploring the codebase."
+          },
+          "exploration": {
+            "anyOf": [
+              {
+                "type": "string",
+                "minLength": 1,
+                "description": "Name of an entry in `models`."
+              },
+              {
+                "type": "object",
+                "required": [
+                  "provider",
+                  "modelId",
+                  "thinking"
+                ],
+                "properties": {
+                  "provider": {
+                    "type": "string",
+                    "minLength": 1,
+                    "description": "Pi provider id, e.g. `openrouter` or `anthropic`."
+                  },
+                  "modelId": {
+                    "type": "string",
+                    "minLength": 1,
+                    "description": "Model id within the provider. Check with `pi --list-models`."
+                  },
+                  "thinking": {
+                    "enum": [
+                      "off",
+                      "low",
+                      "medium",
+                      "high",
+                      "xhigh"
+                    ],
+                    "description": "Pi thinking level to set with the model. Pi clamps it to what the model supports."
+                  }
+                },
+                "additionalProperties": false
+              }
+            ],
+            "description": "Model for tasks that need to explore the codebase first."
+          },
+          "explorationAllowed": {
+            "type": "boolean",
+            "description": "Set false to send tasks that need exploration to the next tier up. Default true."
+          }
+        },
+        "additionalProperties": false
+      },
+      "minItems": 2,
+      "maxItems": 8,
+      "description": "Difficulty tiers, easiest first. Replaces the default list as a whole."
+    },
+    "questions": {
+      "type": "object",
+      "properties": {
+        "version": {
+          "type": "string",
+          "minLength": 1,
+          "description": "Label stored with each decision. Change it whenever you edit the wording or the tier criteria."
+        },
+        "tierInstructions": {
+          "type": "string",
+          "minLength": 1
+        },
+        "explorationInstructions": {
+          "type": "string",
+          "minLength": 1
+        },
+        "explorationCriteria": {
+          "type": "object",
+          "properties": {
+            "yes": {
+              "type": "string",
+              "minLength": 1
+            },
+            "no": {
+              "type": "string",
+              "minLength": 1
+            }
+          },
+          "additionalProperties": false
+        }
+      },
+      "additionalProperties": false,
+      "description": "Wording of the questions sent to the decision model."
+    },
+    "confidenceSource": {
+      "enum": [
+        "reported",
+        "top-probability"
+      ],
+      "description": "`reported`: the decision model's confidence. `top-probability`: the probability of the chosen answer, for checkpoints whose confidence is uncalibrated."
+    },
+    "thresholds": {
+      "type": "object",
+      "properties": {
+        "minConfidenceDowngrade": {
+          "type": "number",
+          "minimum": 0,
+          "description": "Confidence needed to downgrade, or to move in from a model outside the table."
+        },
+        "minConfidenceUpgrade": {
+          "type": "number",
+          "minimum": 0,
+          "description": "Confidence needed to upgrade."
+        },
+        "minConfidenceForm": {
+          "type": "number",
+          "minimum": 0,
+          "description": "Confidence needed to call a task direct rather than exploration."
+        },
+        "cacheGuardTokens": {
+          "type": "number",
+          "minimum": 0,
+          "description": "Context size above which lateral switches (and downgrades, when prices are unknown) are refused."
+        },
+        "minPromptsBetweenSwitches": {
+          "type": "number",
+          "minimum": 0,
+          "description": "Prompts to wait after a switch before the next downgrade or lateral switch."
+        },
+        "maxPaybackRequests": {
+          "type": "number",
+          "minimum": 0,
+          "description": "A downgrade must recoup its cache-miss cost within this many LLM requests."
+        },
+        "assumedOutputTokensPerRequest": {
+          "type": "number",
+          "minimum": 0,
+          "description": "Output per request assumed when estimating what a downgrade saves."
+        },
+        "layaTimeoutMs": {
+          "type": "number",
+          "minimum": 0,
+          "description": "Timeout for one decision from the local Laya worker, in milliseconds."
+        }
+      },
+      "additionalProperties": false
+    }
+  },
+  "additionalProperties": false,
+  "title": "pignon configuration"
+}

package/src/compare.ts

@@ -0,0 +1,109 @@
+/**
+ * `/pignon-stats compare` and `/pignon-stats export`: how two deciders agree,
+ * over the decisions where both answered (parallel strategy, or sequential
+ * escalations).
+ */
+
+import { mkdirSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+
+import { agentDir } from "./config/load.js";
+
+import type { DeciderAttempt, RouterLogEntry, RoutingTable } from "./types.js";
+
+type Answered = DeciderAttempt & { outcome: "answered" };
+
+const answeredBy = (row: RouterLogEntry, decider: string): Answered | undefined =>
+  row.attempts?.find((a): a is Answered => a.decider === decider && a.outcome === "answered");
+
+/** The two deciders that answered most often, in order of first appearance. */
+export function comparedPair(rows: RouterLogEntry[]): [string, string] | null {
+  const counts = new Map<string, number>();
+  for (const row of rows) {
+    for (const a of row.attempts ?? []) {
+      if (a.outcome === "answered") counts.set(a.decider, (counts.get(a.decider) ?? 0) + 1);
+    }
+  }
+  const top = [...counts].sort((x, y) => y[1] - x[1]).slice(0, 2);
+  if (top.length < 2) return null;
+  const order = [...counts.keys()];
+  const [a, b] = top.map(([id]) => id).sort((x, y) => order.indexOf(x) - order.indexOf(y));
+  return [a!, b!];
+}
+
+export function buildCompareLines(rows: RouterLogEntry[], table: RoutingTable): string[] | null {
+  const pair = comparedPair(rows);
+  if (!pair) return null;
+  const [a, b] = pair;
+  const both = rows
+    .map((row) => ({ a: answeredBy(row, a), b: answeredBy(row, b) }))
+    .filter((p): p is { a: Answered; b: Answered } => p.a !== undefined && p.b !== undefined);
+  if (both.length === 0) return null;
+
+  const pct = (n: number) => `${Math.round((100 * n) / both.length)}%`;
+  const tierAgree = both.filter((p) => p.a.tier === p.b.tier).length;
+  const formAgree = both.filter((p) => p.a.needsExploration === p.b.needsExploration).length;
+
+  // Confusion matrix over the tiers seen, in table order first.
+  const seen = new Set(both.flatMap((p) => [p.a.tier ?? "?", p.b.tier ?? "?"]));
+  const tiers = [...table.map((t) => t.id).filter((id) => seen.has(id)), ...[...seen].filter((id) => !table.some((t) => t.id === id))];
+  const width = Math.max(10, ...tiers.map((t) => t.length + 2));
+  const corner = `${a} ↓  ${b} →`;
+  const matrix = [
+    `${corner.padEnd(Math.max(corner.length + 2, width))}${tiers.map((t) => t.padStart(width)).join("")}`,
+    ...tiers.map((row) => {
+      const cells = tiers.map((col) => both.filter((p) => (p.a.tier ?? "?") === row && (p.b.tier ?? "?") === col).length);
+      return `${row.padEnd(Math.max(corner.length + 2, width))}${cells.map((n) => String(n).padStart(width)).join("")}`;
+    }),
+  ];
+
+  const perDecider = (id: string) => {
+    const attempts = rows.flatMap((r) => r.attempts ?? []).filter((x) => x.decider === id);
+    const answered = attempts.filter((x): x is Answered => x.outcome === "answered");
+    const latencies = answered.map((x) => x.latencyMs ?? 0).sort((x, y) => x - y);
+    const confidences = answered.map((x) => x.tierConfidence ?? 0);
+    const cost = attempts.reduce((sum, x) => sum + (x.costUsd ?? 0), 0);
+    return {
+      confidence: confidences.length ? (confidences.reduce((x, y) => x + y, 0) / confidences.length).toFixed(2) : "—",
+      latency: latencies.length ? `${percentile(latencies, 0.5)}/${percentile(latencies, 0.95)} ms` : "—",
+      failures: String(attempts.filter((x) => x.outcome === "failed").length),
+      cost: cost > 0 ? `$${cost.toFixed(5)}` : "—",
+      used: String(attempts.filter((x) => x.used).length),
+    };
+  };
+  const stats = [perDecider(a), perDecider(b)];
+  const col = Math.max(12, a.length + 2, b.length + 2);
+  const row = (label: string, key: keyof ReturnType<typeof perDecider>) =>
+    `${label.padEnd(18)}${stats.map((s) => s[key].padStart(col)).join("")}`;
+
+  return [
+    `pignon compare — ${a} vs ${b} · ${both.length} decisions answered by both`,
+    `tier agreement ${pct(tierAgree)} · exploration agreement ${pct(formAgree)}`,
+    ...matrix,
+    `${"".padEnd(18)}${[a, b].map((id) => id.padStart(col)).join("")}`,
+    row("mean confidence", "confidence"),
+    row("latency p50/p95", "latency"),
+    row("failures", "failures"),
+    row("cost", "cost"),
+    row("routed on", "used"),
+  ];
+}
+
+function percentile(sorted: number[], p: number): number {
+  return Math.round(sorted[Math.min(sorted.length - 1, Math.floor(p * sorted.length))]!);
+}
+
+/** Default export location: outside any project, next to Pi's own files. */
+export function defaultExportPath(now = new Date(), env: NodeJS.ProcessEnv = process.env): string {
+  const stamp = now.toISOString().replace(/[:.]/g, "-");
+  return join(agentDir(env), "pignon-exports", `decisions-${stamp}.jsonl`);
+}
+
+/**
+ * Write one JSON line per decision. Entries hold a prompt hash and length,
+ * never the prompt, so the file is safe to share.
+ */
+export function exportDecisions(rows: RouterLogEntry[], path: string): void {
+  mkdirSync(dirname(path), { recursive: true });
+  writeFileSync(path, rows.map((row) => JSON.stringify(row)).join("\n") + "\n");
+}