pi-smart-router 0.9.3 → 0.10.0

package/README.md

@@ -590,6 +590,11 @@ npm run routing:ingest-fc-rewardbench -- \
   --output /tmp/fc-rewardbench-pack.jsonl
 
 # Optional weak TwinRouterBench tier proxy (exclude from holdout ECE)
+# Preferred input: checked-in CI subset (SP-199/SP-201); full-track after SP-200 is local-only.
+npm run routing:ingest-twinrouterbench-weak -- \
+  --input tests/eval/corpus/twinrouterbench/ci-subset.json \
+  --output /tmp/trb-weak-from-ci-subset.jsonl
+
 npm run routing:ingest-twinrouterbench-weak -- \
   --input tests/eval/corpus/label-packs/twinrouterbench-weak/ci-fixture.jsonl \
   --output /tmp/trb-weak-pack.jsonl
@@ -604,6 +609,11 @@ npm run routing:calibration-dry-run
 # Operator packs (schema-valid JSONL)
 npm run routing:calibration-dry-run -- --packs /tmp/swe-gym-pack.jsonl /tmp/fc-rewardbench-pack.jsonl
 
+# Warm-start: weak / exclude_from_holdout_ece rows join the **fit** pool only
+npm run routing:calibration-dry-run -- \
+  --packs /tmp/swe-gym-pack.jsonl /tmp/trb-weak-from-ci-subset.jsonl \
+  --include-excluded-in-fit
+
 # Soft ECE advisory fail (threshold 0.25 calibrated ECE; not a release-gate absolute)
 npm run routing:calibration-dry-run -- --packs /tmp/swe-gym-pack.jsonl --enforce-soft-ece
 ```
@@ -615,9 +625,10 @@ Dry-run behavior:
 | < 30 ECE-eligible rows | `SAMPLE_STARVED` report-only (exit 0); no soft pass/fail |
 | ≥ 30 ECE-eligible rows | Fit logistic + isotonic; report `holdout_ece_raw` / `holdout_ece_calibrated` |
 | Rows with `exclude_from_holdout_ece` | Counted separately; **never** enter holdout ECE metrics (weak TwinRouterBench) |
+| `--include-excluded-in-fit` | Weak rows may warm-start the fit pool; `ece_eligible` / holdout ECE stay verifier-grade |
 | Soft threshold | Advisory `0.25` calibrated ECE — **does not** change `config/release-gates.json` |
 
-**#96 / `modernbert_k4` advisory:** when deciding whether to enable ModernBERT K=4 heads, use **pack holdout ECE / Top-1 error on verifier-grade packs** (SWE-Gym + FC-RewardBench), not fixture-only QR. Weak TwinRouterBench rows are warm-start only. This task does **not** flip `modernbert_k4` defaults.
+**#96 / `modernbert_k4` advisory:** when deciding whether to enable ModernBERT K=4 heads, use **pack holdout ECE / Top-1 error on verifier-grade packs** (SWE-Gym + FC-RewardBench), not fixture-only QR and **not** weak-fit ECE. Weak TwinRouterBench rows are warm-start only. This task does **not** flip `modernbert_k4` defaults.
 
 ### Operator tuning (frugality slider)
 
@@ -842,10 +853,10 @@ Contributors must run `npm run build` before publishing or consuming the library
 | `npm run routing:train-calibration` | Train routing calibration artifact bundle |
 | `npm run routing:train-p-success` | Train standalone `config/p-success-weights.json` (synthetic fixture by default) |
 | `npm run routing:verify-calibration` | Verify calibration bundle against benchmark prompts |
-| `npm run routing:calibration-dry-run` | Pack-fed isotonic dry-run: holdout ECE on label packs (CI fixtures by default) |
+| `npm run routing:calibration-dry-run` | Pack-fed isotonic dry-run: holdout ECE on label packs (CI fixtures by default); optional `--include-excluded-in-fit` |
 | `npm run routing:ingest-swe-gym` | Convert SWE-Gym verifier-style JSONL → privacy-safe label pack |
 | `npm run routing:ingest-fc-rewardbench` | Convert FC-RewardBench JSONL → privacy-safe label pack |
-| `npm run routing:ingest-twinrouterbench-weak` | Convert TwinRouterBench weak tier labels → pack (exclude from ECE) |
+| `npm run routing:ingest-twinrouterbench-weak` | Convert TwinRouterBench weak tier labels → pack (prefer `ci-subset.json`; exclude from ECE) |
 | `npm run routing:ingest-benchmarks` | Regenerate `config/benchmark-profiles.json` from leaderboard fixtures |
 | `npm run routing:verify-benchmark-profiles` | CI smoke: assert checked-in profiles match fixture ingest |
 | `npm run routing:eval-replay` | Counterfactual replay on eval trace fixtures |
@@ -853,6 +864,9 @@ Contributors must run `npm run build` before publishing or consuming the library
 | `npm run routing:eval-harness:smoke` | Harness summary JSON only (CI smoke; no network) |
 | `npm run routing:eval-harness:corpus-smoke` | Harness summary on TwinRouterBench CI corpus subset (`tests/eval/corpus/twinrouterbench`) |
 | `npm run routing:assert-release-gates:corpus-report` | Soft-feed: assert corpus vs absolute gates with `--report-only` (exit 0; does not gate releases) |
+| `npm run routing:twinrouterbench:full-track` | Local/nightly: pin fetch → full convert (no `--limit`) → harness + gates `--report-only` (gitignored cache) |
+| `npm run routing:twinrouterbench:full-ingest` | Convert cached `question_bank.jsonl` → full static-track JSON (no `--limit`) |
+| `npm run routing:twinrouterbench:full-report` | Harness summary + gates `--report-only` on cached full track |
 | `npm run routing:ingest-twinrouterbench` | Convert TwinRouterBench `question_bank.jsonl` → CI subset / full corpus JSON |
 | `npm run routing:ingest-llmrouterbench` | Convert LLMRouterBench BaselineRecord JSONL → static-track subset JSON |
 | `npm run routing:llmrouterbench-regret` | Offline regret / CS report on vendored LLMRouterBench subset (optional; not PR CI) |
@@ -874,7 +888,7 @@ npm run routing:eval-harness
 # CI-style summary only (default fixtures under tests/eval/fixtures)
 npm run routing:eval-harness:smoke
 
-# TwinRouterBench CI corpus subset (≤50 code/tool records; offline)
+# TwinRouterBench CI corpus subset (≤150 code/tool records; offline)
 npm run routing:eval-harness:corpus-smoke
 
 # Custom fixture directory (includes TwinRouterBench static track subdirs)
@@ -882,24 +896,44 @@ npm run routing:eval-harness -- --fixtures tests/eval/fixtures
 
 # Counterfactual replay only (SP-151)
 npm run routing:eval-replay
+
+# Full ~970-row static track (local / nightly only — do not check JSON into git)
+npm run routing:twinrouterbench:full-track
 ```
 
-**CI smoke:** `.github/workflows/eval-harness-smoke.yml` runs on PRs that touch eval scripts, fixtures, or the workflow. It executes `routing:eval-harness:smoke`, `routing:eval-harness:corpus-smoke`, and eval unit tests — fast, offline, no provider network calls. Job timeout stays at 10 minutes.
+**CI smoke:** `.github/workflows/eval-harness-smoke.yml` runs on PRs that touch eval scripts, fixtures, or the workflow. It executes `routing:eval-harness:smoke`, `routing:eval-harness:corpus-smoke`, and eval unit tests — fast, offline, no provider network calls. Job timeout stays at 10 minutes. The optional full-track nightly (`.github/workflows/twinrouterbench-full-nightly.yml`, `schedule` + `workflow_dispatch` only) is **not** on `pull_request` and must not be configured as a required status check — failures there do not gate PR CI or `release:functional-smoke`.
 
 **TwinRouterBench static track:** import step-level router-visible prefixes with execution-verified target tiers (`track: "static"`). The adapter in `scripts/eval/twinrouterbench-adapter.ts` converts static track records into native eval fixtures for the three-track harness. See `docs/gemini-research.md` §9 for methodology context.
 
-#### TwinRouterBench CI corpus (SP-186 / SP-187 / SP-188)
+#### TwinRouterBench CI corpus (SP-186 / SP-187 / SP-188 / SP-199)
 
 | Item | Location / command |
 |------|--------------------|
 | **Pinned upstream** | CommonstackAI/TwinRouterBench `@430acecac71141de77afd8e5e13690d236d58e93` (Apache-2.0) |
-| **CI subset** | `tests/eval/corpus/twinrouterbench/ci-subset.json` (≤50 code/tool records) |
+| **CI subset** | `tests/eval/corpus/twinrouterbench/ci-subset.json` (≤150 code/tool records) |
 | **Provenance** | `tests/eval/corpus/twinrouterbench/PROVENANCE.md` |
-| **Regenerate** | `npm run routing:ingest-twinrouterbench -- --input <question_bank.jsonl> --output tests/eval/corpus/twinrouterbench/ci-subset.json --limit 50 --prefer-code-tool` |
+| **Regenerate** | `npm run routing:ingest-twinrouterbench -- --input <question_bank.jsonl> --output tests/eval/corpus/twinrouterbench/ci-subset.json --limit 150 --prefer-code-tool` |
 | **Harness smoke** | `npm run routing:eval-harness:corpus-smoke` |
 | **Gate soft-feed** | `npm run routing:assert-release-gates:corpus-report` |
+| **Human QA protocol** | [`docs/qa/shadow-dogfood-protocol.md`](docs/qa/shadow-dogfood-protocol.md) · `npm run qa:shadow-dogfood` |
+
+**Absolute release gates stay on default fixtures.** `npm run release:functional-smoke` continues to assert `tests/eval/fixtures` against `config/release-gates.json` — do not point it at the corpus without operator review. Today the corpus subset fails `mean_over_routing_rate_max` (≈0.85 vs absolute max 0.15); that gap is intentional soft signal for the [#95](https://github.com/beettlle/pi-smart-router/issues/95) public static-track acceptance criteria alongside live dogfood traces. Use `--fixtures tests/eval/corpus/twinrouterbench` (or the corpus-report script) for #95 public-track scoring; keep absolute threshold edits out of band until operators approve. For live shadow dogfood steps and sign-off, see the [shadow dogfood protocol](docs/qa/shadow-dogfood-protocol.md).
+
+#### TwinRouterBench full static track (SP-200 / #107)
+
+First-class **local / optional nightly** path for the pinned ~970-row bank. **Do not check the full JSON into git.**
+
+| Item | Location / command |
+|------|--------------------|
+| **One-shot** | `npm run routing:twinrouterbench:full-track` |
+| **Cache (gitignored)** | `.pi-smart-router/eval-cache/twinrouterbench/` (override with `TRB_CACHE_DIR`) |
+| **Steps** | pin fetch → `routing:ingest-twinrouterbench` **without** `--limit` → harness `--summary-only` + gates `--report-only` |
+| **Nightly** | `.github/workflows/twinrouterbench-full-nightly.yml` (`schedule` + `workflow_dispatch`) — advisory only |
+| **Provenance** | `tests/eval/corpus/twinrouterbench/PROVENANCE.md` |
+
+PR corpus smoke remains the vendored ≤150 subset. Absolute `config/release-gates.json` thresholds and `release:functional-smoke` stay fixture-backed.
 
-**Absolute release gates stay on default fixtures.** `npm run release:functional-smoke` continues to assert `tests/eval/fixtures` against `config/release-gates.json` — do not point it at the corpus without operator review. Today the corpus subset fails `mean_over_routing_rate_max` (≈0.85 vs absolute max 0.15); that gap is intentional soft signal for the [#95](https://github.com/beettlle/pi-smart-router/issues/95) public static-track acceptance criteria alongside live dogfood traces. Use `--fixtures tests/eval/corpus/twinrouterbench` (or the corpus-report script) for #95 public-track scoring; keep absolute threshold edits out of band until operators approve.
+**[#95 dual-gate protocol](https://github.com/beettlle/pi-smart-router/issues/95):** (1) live shadow dogfood (`docs/qa/shadow-dogfood-protocol.md` · `npm run qa:shadow-dogfood`) and (2) public static-track soft-feed (CI subset report, or full-track report above). Neither path edits absolute release thresholds.
 
 **Deferred:** RouterBench classic (outcome-matrix) smoke is out of scope for SP-188; prefer TwinRouterBench static track + dogfood for #95.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-smart-router",
-  "version": "0.9.3",
+  "version": "0.10.0",
   "description": "Auto-model router middleware for the pi.dev coding agent",
   "type": "module",
   "license": "MIT",
@@ -73,6 +73,9 @@
     "routing:eval-harness:smoke": "tsx scripts/eval/run-harness.ts --summary-only",
     "routing:eval-harness:corpus-smoke": "tsx scripts/eval/run-harness.ts --fixtures tests/eval/corpus/twinrouterbench --summary-only",
     "routing:assert-release-gates:corpus-report": "tsx scripts/eval/assert-release-gates.ts --fixtures tests/eval/corpus/twinrouterbench --report-only",
+    "routing:twinrouterbench:full-ingest": "bash -c 'set -euo pipefail; CACHE=\"${TRB_CACHE_DIR:-.pi-smart-router/eval-cache/twinrouterbench}\"; mkdir -p \"$CACHE\"; npm run routing:ingest-twinrouterbench -- --input \"$CACHE/question_bank.jsonl\" --output \"$CACHE/full-static-track.json\"'",
+    "routing:twinrouterbench:full-report": "bash -c 'set -euo pipefail; CACHE=\"${TRB_CACHE_DIR:-.pi-smart-router/eval-cache/twinrouterbench}\"; DIR=\"$CACHE/full-track\"; mkdir -p \"$DIR\"; cp \"$CACHE/full-static-track.json\" \"$DIR/full-static-track.json\"; npm run routing:eval-harness -- --fixtures \"$DIR\" --summary-only; npx tsx scripts/eval/assert-release-gates.ts --fixtures \"$DIR\" --report-only'",
+    "routing:twinrouterbench:full-track": "bash scripts/eval/twinrouterbench-full-track.sh",
     "routing:community-bench": "tsx scripts/eval/community-bench.ts",
     "routing:ingest-twinrouterbench": "tsx scripts/eval/ingest-twinrouterbench-corpus.ts",
     "routing:ingest-llmrouterbench": "tsx scripts/eval/ingest-llmrouterbench-subset.ts",
@@ -83,6 +86,7 @@
     "routing:capture-baseline": "tsx scripts/eval/capture-baseline.ts",
     "routing:test-projection": "npm run build && node --experimental-strip-types scripts/test-hydra-projection.ts",
     "benchmark:encoder": "tsx scripts/benchmark-encoder-latency.ts",
+    "qa:shadow-dogfood": "bash scripts/qa/shadow-dogfood-session.sh",
     "verify:ci": "npm run build && npm run typecheck && npm run lint && npm test && npm run coverage:check"
   },
   "peerDependencies": {