pi-smart-router 0.9.2 → 0.10.0

package/README.md

@@ -590,6 +590,11 @@ npm run routing:ingest-fc-rewardbench -- \
   --output /tmp/fc-rewardbench-pack.jsonl
 
 # Optional weak TwinRouterBench tier proxy (exclude from holdout ECE)
+# Preferred input: checked-in CI subset (SP-199/SP-201); full-track after SP-200 is local-only.
+npm run routing:ingest-twinrouterbench-weak -- \
+  --input tests/eval/corpus/twinrouterbench/ci-subset.json \
+  --output /tmp/trb-weak-from-ci-subset.jsonl
+
 npm run routing:ingest-twinrouterbench-weak -- \
   --input tests/eval/corpus/label-packs/twinrouterbench-weak/ci-fixture.jsonl \
   --output /tmp/trb-weak-pack.jsonl
@@ -604,6 +609,11 @@ npm run routing:calibration-dry-run
 # Operator packs (schema-valid JSONL)
 npm run routing:calibration-dry-run -- --packs /tmp/swe-gym-pack.jsonl /tmp/fc-rewardbench-pack.jsonl
 
+# Warm-start: weak / exclude_from_holdout_ece rows join the **fit** pool only
+npm run routing:calibration-dry-run -- \
+  --packs /tmp/swe-gym-pack.jsonl /tmp/trb-weak-from-ci-subset.jsonl \
+  --include-excluded-in-fit
+
 # Soft ECE advisory fail (threshold 0.25 calibrated ECE; not a release-gate absolute)
 npm run routing:calibration-dry-run -- --packs /tmp/swe-gym-pack.jsonl --enforce-soft-ece
 ```
@@ -615,9 +625,10 @@ Dry-run behavior:
 | < 30 ECE-eligible rows | `SAMPLE_STARVED` report-only (exit 0); no soft pass/fail |
 | ≥ 30 ECE-eligible rows | Fit logistic + isotonic; report `holdout_ece_raw` / `holdout_ece_calibrated` |
 | Rows with `exclude_from_holdout_ece` | Counted separately; **never** enter holdout ECE metrics (weak TwinRouterBench) |
+| `--include-excluded-in-fit` | Weak rows may warm-start the fit pool; `ece_eligible` / holdout ECE stay verifier-grade |
 | Soft threshold | Advisory `0.25` calibrated ECE — **does not** change `config/release-gates.json` |
 
-**#96 / `modernbert_k4` advisory:** when deciding whether to enable ModernBERT K=4 heads, use **pack holdout ECE / Top-1 error on verifier-grade packs** (SWE-Gym + FC-RewardBench), not fixture-only QR. Weak TwinRouterBench rows are warm-start only. This task does **not** flip `modernbert_k4` defaults.
+**#96 / `modernbert_k4` advisory:** when deciding whether to enable ModernBERT K=4 heads, use **pack holdout ECE / Top-1 error on verifier-grade packs** (SWE-Gym + FC-RewardBench), not fixture-only QR and **not** weak-fit ECE. Weak TwinRouterBench rows are warm-start only. This task does **not** flip `modernbert_k4` defaults.
 
 ### Operator tuning (frugality slider)
 
@@ -842,10 +853,10 @@ Contributors must run `npm run build` before publishing or consuming the library
 | `npm run routing:train-calibration` | Train routing calibration artifact bundle |
 | `npm run routing:train-p-success` | Train standalone `config/p-success-weights.json` (synthetic fixture by default) |
 | `npm run routing:verify-calibration` | Verify calibration bundle against benchmark prompts |
-| `npm run routing:calibration-dry-run` | Pack-fed isotonic dry-run: holdout ECE on label packs (CI fixtures by default) |
+| `npm run routing:calibration-dry-run` | Pack-fed isotonic dry-run: holdout ECE on label packs (CI fixtures by default); optional `--include-excluded-in-fit` |
 | `npm run routing:ingest-swe-gym` | Convert SWE-Gym verifier-style JSONL → privacy-safe label pack |
 | `npm run routing:ingest-fc-rewardbench` | Convert FC-RewardBench JSONL → privacy-safe label pack |
-| `npm run routing:ingest-twinrouterbench-weak` | Convert TwinRouterBench weak tier labels → pack (exclude from ECE) |
+| `npm run routing:ingest-twinrouterbench-weak` | Convert TwinRouterBench weak tier labels → pack (prefer `ci-subset.json`; exclude from ECE) |
 | `npm run routing:ingest-benchmarks` | Regenerate `config/benchmark-profiles.json` from leaderboard fixtures |
 | `npm run routing:verify-benchmark-profiles` | CI smoke: assert checked-in profiles match fixture ingest |
 | `npm run routing:eval-replay` | Counterfactual replay on eval trace fixtures |
@@ -853,7 +864,13 @@ Contributors must run `npm run build` before publishing or consuming the library
 | `npm run routing:eval-harness:smoke` | Harness summary JSON only (CI smoke; no network) |
 | `npm run routing:eval-harness:corpus-smoke` | Harness summary on TwinRouterBench CI corpus subset (`tests/eval/corpus/twinrouterbench`) |
 | `npm run routing:assert-release-gates:corpus-report` | Soft-feed: assert corpus vs absolute gates with `--report-only` (exit 0; does not gate releases) |
+| `npm run routing:twinrouterbench:full-track` | Local/nightly: pin fetch → full convert (no `--limit`) → harness + gates `--report-only` (gitignored cache) |
+| `npm run routing:twinrouterbench:full-ingest` | Convert cached `question_bank.jsonl` → full static-track JSON (no `--limit`) |
+| `npm run routing:twinrouterbench:full-report` | Harness summary + gates `--report-only` on cached full track |
 | `npm run routing:ingest-twinrouterbench` | Convert TwinRouterBench `question_bank.jsonl` → CI subset / full corpus JSON |
+| `npm run routing:ingest-llmrouterbench` | Convert LLMRouterBench BaselineRecord JSONL → static-track subset JSON |
+| `npm run routing:llmrouterbench-regret` | Offline regret / CS report on vendored LLMRouterBench subset (optional; not PR CI) |
+| `npm run routing:community-bench` | Privacy-safe community bench report (Track A TwinRouterBench + optional Track B/C) |
 | `npm run benchmark:encoder` | Compare MiniLM vs Granite encoder latency on held-out agent turns |
 
 ### Offline eval harness (agent-native routing)
@@ -871,7 +888,7 @@ npm run routing:eval-harness
 # CI-style summary only (default fixtures under tests/eval/fixtures)
 npm run routing:eval-harness:smoke
 
-# TwinRouterBench CI corpus subset (≤50 code/tool records; offline)
+# TwinRouterBench CI corpus subset (≤150 code/tool records; offline)
 npm run routing:eval-harness:corpus-smoke
 
 # Custom fixture directory (includes TwinRouterBench static track subdirs)
@@ -879,27 +896,98 @@ npm run routing:eval-harness -- --fixtures tests/eval/fixtures
 
 # Counterfactual replay only (SP-151)
 npm run routing:eval-replay
+
+# Full ~970-row static track (local / nightly only — do not check JSON into git)
+npm run routing:twinrouterbench:full-track
 ```
 
-**CI smoke:** `.github/workflows/eval-harness-smoke.yml` runs on PRs that touch eval scripts, fixtures, or the workflow. It executes `routing:eval-harness:smoke`, `routing:eval-harness:corpus-smoke`, and eval unit tests — fast, offline, no provider network calls. Job timeout stays at 10 minutes.
+**CI smoke:** `.github/workflows/eval-harness-smoke.yml` runs on PRs that touch eval scripts, fixtures, or the workflow. It executes `routing:eval-harness:smoke`, `routing:eval-harness:corpus-smoke`, and eval unit tests — fast, offline, no provider network calls. Job timeout stays at 10 minutes. The optional full-track nightly (`.github/workflows/twinrouterbench-full-nightly.yml`, `schedule` + `workflow_dispatch` only) is **not** on `pull_request` and must not be configured as a required status check — failures there do not gate PR CI or `release:functional-smoke`.
 
 **TwinRouterBench static track:** import step-level router-visible prefixes with execution-verified target tiers (`track: "static"`). The adapter in `scripts/eval/twinrouterbench-adapter.ts` converts static track records into native eval fixtures for the three-track harness. See `docs/gemini-research.md` §9 for methodology context.
 
-#### TwinRouterBench CI corpus (SP-186 / SP-187 / SP-188)
+#### TwinRouterBench CI corpus (SP-186 / SP-187 / SP-188 / SP-199)
 
 | Item | Location / command |
 |------|--------------------|
 | **Pinned upstream** | CommonstackAI/TwinRouterBench `@430acecac71141de77afd8e5e13690d236d58e93` (Apache-2.0) |
-| **CI subset** | `tests/eval/corpus/twinrouterbench/ci-subset.json` (≤50 code/tool records) |
+| **CI subset** | `tests/eval/corpus/twinrouterbench/ci-subset.json` (≤150 code/tool records) |
 | **Provenance** | `tests/eval/corpus/twinrouterbench/PROVENANCE.md` |
-| **Regenerate** | `npm run routing:ingest-twinrouterbench -- --input <question_bank.jsonl> --output tests/eval/corpus/twinrouterbench/ci-subset.json --limit 50 --prefer-code-tool` |
+| **Regenerate** | `npm run routing:ingest-twinrouterbench -- --input <question_bank.jsonl> --output tests/eval/corpus/twinrouterbench/ci-subset.json --limit 150 --prefer-code-tool` |
 | **Harness smoke** | `npm run routing:eval-harness:corpus-smoke` |
 | **Gate soft-feed** | `npm run routing:assert-release-gates:corpus-report` |
+| **Human QA protocol** | [`docs/qa/shadow-dogfood-protocol.md`](docs/qa/shadow-dogfood-protocol.md) · `npm run qa:shadow-dogfood` |
 
-**Absolute release gates stay on default fixtures.** `npm run release:functional-smoke` continues to assert `tests/eval/fixtures` against `config/release-gates.json` — do not point it at the corpus without operator review. Today the corpus subset fails `mean_over_routing_rate_max` (≈0.85 vs absolute max 0.15); that gap is intentional soft signal for the [#95](https://github.com/beettlle/pi-smart-router/issues/95) public static-track acceptance criteria alongside live dogfood traces. Use `--fixtures tests/eval/corpus/twinrouterbench` (or the corpus-report script) for #95 public-track scoring; keep absolute threshold edits out of band until operators approve.
+**Absolute release gates stay on default fixtures.** `npm run release:functional-smoke` continues to assert `tests/eval/fixtures` against `config/release-gates.json` — do not point it at the corpus without operator review. Today the corpus subset fails `mean_over_routing_rate_max` (≈0.85 vs absolute max 0.15); that gap is intentional soft signal for the [#95](https://github.com/beettlle/pi-smart-router/issues/95) public static-track acceptance criteria alongside live dogfood traces. Use `--fixtures tests/eval/corpus/twinrouterbench` (or the corpus-report script) for #95 public-track scoring; keep absolute threshold edits out of band until operators approve. For live shadow dogfood steps and sign-off, see the [shadow dogfood protocol](docs/qa/shadow-dogfood-protocol.md).
+
+#### TwinRouterBench full static track (SP-200 / #107)
+
+First-class **local / optional nightly** path for the pinned ~970-row bank. **Do not check the full JSON into git.**
+
+| Item | Location / command |
+|------|--------------------|
+| **One-shot** | `npm run routing:twinrouterbench:full-track` |
+| **Cache (gitignored)** | `.pi-smart-router/eval-cache/twinrouterbench/` (override with `TRB_CACHE_DIR`) |
+| **Steps** | pin fetch → `routing:ingest-twinrouterbench` **without** `--limit` → harness `--summary-only` + gates `--report-only` |
+| **Nightly** | `.github/workflows/twinrouterbench-full-nightly.yml` (`schedule` + `workflow_dispatch`) — advisory only |
+| **Provenance** | `tests/eval/corpus/twinrouterbench/PROVENANCE.md` |
+
+PR corpus smoke remains the vendored ≤150 subset. Absolute `config/release-gates.json` thresholds and `release:functional-smoke` stay fixture-backed.
+
+**[#95 dual-gate protocol](https://github.com/beettlle/pi-smart-router/issues/95):** (1) live shadow dogfood (`docs/qa/shadow-dogfood-protocol.md` · `npm run qa:shadow-dogfood`) and (2) public static-track soft-feed (CI subset report, or full-track report above). Neither path edits absolute release thresholds.
 
 **Deferred:** RouterBench classic (outcome-matrix) smoke is out of scope for SP-188; prefer TwinRouterBench static track + dogfood for #95.
 
+#### LLMRouterBench offline regret (SP-192 / SP-193)
+
+Optional local / nightly report on the **pinned code/tool subset** — not part of PR CI (no full HF corpus download).
+
+| Item | Location / command |
+|------|--------------------|
+| **Pinned upstream** | HF `NPULH/LLMRouterBench` `@0e5af1b84bf73437a01a1849c0f1d2468baa93fc` + git schema `@c77cb0506949d8f959e97967d2fefca0e8ff1b05` (MIT) |
+| **CI subset** | `tests/eval/corpus/llmrouterbench/ci-subset.json` (≤20 synthetic offline records) |
+| **Provenance / refresh** | `tests/eval/corpus/llmrouterbench/PROVENANCE.md` (quarterly pin refresh; re-run report after catalog/subset changes) |
+| **Regenerate subset** | `npm run routing:ingest-llmrouterbench -- --input <jsonl> --output tests/eval/corpus/llmrouterbench/ci-subset.json --limit 20 --prefer-code-tool` |
+| **Regret / CS report** | `npm run routing:llmrouterbench-regret` |
+| **Community Track C** | `npm run routing:community-bench -- --llmrouterbench` (same vendored subset; optional) |
+
+PR CI continues to smoke TwinRouterBench only (`routing:eval-harness:corpus-smoke`). Absolute `config/release-gates.json` thresholds are unchanged. See [Contribute a community bench report](#contribute-a-community-bench-report) for Track A + optional Track C sharing.
+
+#### Contribute a community bench report
+
+Share a privacy-safe setup fingerprint + Track A (TwinRouterBench) gate result with maintainers. No SMTP auto-send and no upload server — you copy artifacts yourself.
+
+**Maintainer contact** (must match the CLI footer constant `COMMUNITY_BENCH_MAINTAINER_CONTACT`):
+
+`https://github.com/beettlle/pi-smart-router/issues/new?labels=community-bench`
+
+```bash
+# Track A offline smoke (vendored TwinRouterBench corpus — no network)
+npm run routing:community-bench -- \
+  --output /tmp/community-bench-report.json \
+  --email-file /tmp/community-bench-report.txt
+
+# Optional Track C: offline LLMRouterBench regret/CS on the vendored subset (no full HF download)
+npm run routing:community-bench -- \
+  --llmrouterbench \
+  --output /tmp/community-bench-report.json \
+  --email-file /tmp/community-bench-report.txt
+```
+
+**How to send:**
+
+1. **Email `.txt`** — open `/tmp/community-bench-report.txt` (or your `--email-file` path). It includes a `Subject:` line, privacy blurb, fingerprint, Track A PASS/FAIL, and optional Track C metrics. Paste into your mail client; attach `community-bench-report.json` if useful. Do **not** expect the CLI to send mail.
+2. **GitHub issue** — open the [maintainer contact](https://github.com/beettlle/pi-smart-router/issues/new?labels=community-bench) URL, or run with `--print-issue-body` and paste stdout into a new issue. Issues list: https://github.com/beettlle/pi-smart-router/issues
+
+**Tracks:**
+
+| Track | Corpus | When |
+|-------|--------|------|
+| **A (required)** | [TwinRouterBench CI corpus](#twinrouterbench-ci-corpus-sp-186--sp-187--sp-188) (`tests/eval/corpus/twinrouterbench`) | Always |
+| **B (optional)** | Dogfood export (`--dogfood-export PATH`) | Skips with an explicit reason until [#95](https://github.com/beettlle/pi-smart-router/issues/95) adapter lands — never invents labels |
+| **C (optional)** | [LLMRouterBench offline subset](#llmrouterbench-offline-regret-sp-192--sp-193) (`tests/eval/corpus/llmrouterbench`) | `--llmrouterbench` or `--full`; offline only |
+
+PR CI does **not** download full TwinRouterBench / LLMRouterBench corpora. Absolute gate thresholds in `config/release-gates.json` are unchanged by this CLI.
+
 Sample fixtures under `tests/eval/fixtures/twinrouterbench/` remain the small adapter unit-test inputs and are unchanged by corpus ingest.
 
 Capability scores in `config/benchmark-profiles.json` are grounded from public leaderboard snapshots under `tests/fixtures/benchmark-leaderboards/` (and optional **recorded** live snapshots under `tests/fixtures/benchmark-leaderboards/recorded/`). Each artifact records provenance (`source_urls`, `scrape_date`, `catalog_freeze_date`) in its header.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-smart-router",
-  "version": "0.9.2",
+  "version": "0.10.0",
   "description": "Auto-model router middleware for the pi.dev coding agent",
   "type": "module",
   "license": "MIT",
@@ -73,13 +73,20 @@
     "routing:eval-harness:smoke": "tsx scripts/eval/run-harness.ts --summary-only",
     "routing:eval-harness:corpus-smoke": "tsx scripts/eval/run-harness.ts --fixtures tests/eval/corpus/twinrouterbench --summary-only",
     "routing:assert-release-gates:corpus-report": "tsx scripts/eval/assert-release-gates.ts --fixtures tests/eval/corpus/twinrouterbench --report-only",
+    "routing:twinrouterbench:full-ingest": "bash -c 'set -euo pipefail; CACHE=\"${TRB_CACHE_DIR:-.pi-smart-router/eval-cache/twinrouterbench}\"; mkdir -p \"$CACHE\"; npm run routing:ingest-twinrouterbench -- --input \"$CACHE/question_bank.jsonl\" --output \"$CACHE/full-static-track.json\"'",
+    "routing:twinrouterbench:full-report": "bash -c 'set -euo pipefail; CACHE=\"${TRB_CACHE_DIR:-.pi-smart-router/eval-cache/twinrouterbench}\"; DIR=\"$CACHE/full-track\"; mkdir -p \"$DIR\"; cp \"$CACHE/full-static-track.json\" \"$DIR/full-static-track.json\"; npm run routing:eval-harness -- --fixtures \"$DIR\" --summary-only; npx tsx scripts/eval/assert-release-gates.ts --fixtures \"$DIR\" --report-only'",
+    "routing:twinrouterbench:full-track": "bash scripts/eval/twinrouterbench-full-track.sh",
+    "routing:community-bench": "tsx scripts/eval/community-bench.ts",
     "routing:ingest-twinrouterbench": "tsx scripts/eval/ingest-twinrouterbench-corpus.ts",
+    "routing:ingest-llmrouterbench": "tsx scripts/eval/ingest-llmrouterbench-subset.ts",
+    "routing:llmrouterbench-regret": "tsx scripts/eval/llmrouterbench-regret-report.ts",
     "routing:ingest-swe-gym": "tsx scripts/ingest-swe-gym-labels.ts",
     "routing:ingest-fc-rewardbench": "tsx scripts/ingest-fc-rewardbench-labels.ts",
     "routing:ingest-twinrouterbench-weak": "tsx scripts/ingest-twinrouterbench-weak-labels.ts",
     "routing:capture-baseline": "tsx scripts/eval/capture-baseline.ts",
     "routing:test-projection": "npm run build && node --experimental-strip-types scripts/test-hydra-projection.ts",
     "benchmark:encoder": "tsx scripts/benchmark-encoder-latency.ts",
+    "qa:shadow-dogfood": "bash scripts/qa/shadow-dogfood-session.sh",
     "verify:ci": "npm run build && npm run typecheck && npm run lint && npm test && npm run coverage:check"
   },
   "peerDependencies": {