misroute 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. misroute-0.2.0/LICENSE +21 -0
  2. misroute-0.2.0/PKG-INFO +265 -0
  3. misroute-0.2.0/README.md +232 -0
  4. misroute-0.2.0/pyproject.toml +75 -0
  5. misroute-0.2.0/src/misroute/__init__.py +3 -0
  6. misroute-0.2.0/src/misroute/agent_server.py +98 -0
  7. misroute-0.2.0/src/misroute/check.py +151 -0
  8. misroute-0.2.0/src/misroute/cli.py +460 -0
  9. misroute-0.2.0/src/misroute/config.py +95 -0
  10. misroute-0.2.0/src/misroute/detectors/__init__.py +0 -0
  11. misroute-0.2.0/src/misroute/detectors/codemode.py +60 -0
  12. misroute-0.2.0/src/misroute/detectors/confusion.py +40 -0
  13. misroute-0.2.0/src/misroute/detectors/hedge.py +17 -0
  14. misroute-0.2.0/src/misroute/detectors/nearmiss.py +66 -0
  15. misroute-0.2.0/src/misroute/detectors/rank.py +72 -0
  16. misroute-0.2.0/src/misroute/detectors/removal.py +11 -0
  17. misroute-0.2.0/src/misroute/fix/__init__.py +0 -0
  18. misroute-0.2.0/src/misroute/fix/clause.py +94 -0
  19. misroute-0.2.0/src/misroute/fix/stats.py +52 -0
  20. misroute-0.2.0/src/misroute/guard/__init__.py +0 -0
  21. misroute-0.2.0/src/misroute/guard/proxy.py +44 -0
  22. misroute-0.2.0/src/misroute/guard/rules.py +49 -0
  23. misroute-0.2.0/src/misroute/guard/wrapper.py +119 -0
  24. misroute-0.2.0/src/misroute/lint/__init__.py +0 -0
  25. misroute-0.2.0/src/misroute/lint/descriptions.py +50 -0
  26. misroute-0.2.0/src/misroute/models/__init__.py +16 -0
  27. misroute-0.2.0/src/misroute/models/anthropic.py +70 -0
  28. misroute-0.2.0/src/misroute/models/base.py +79 -0
  29. misroute-0.2.0/src/misroute/models/mock.py +158 -0
  30. misroute-0.2.0/src/misroute/models/openai_compat.py +118 -0
  31. misroute-0.2.0/src/misroute/modes/__init__.py +0 -0
  32. misroute-0.2.0/src/misroute/modes/code_mode.py +87 -0
  33. misroute-0.2.0/src/misroute/modes/json_mode.py +71 -0
  34. misroute-0.2.0/src/misroute/modes/stubs.py +40 -0
  35. misroute-0.2.0/src/misroute/project.py +433 -0
  36. misroute-0.2.0/src/misroute/py.typed +0 -0
  37. misroute-0.2.0/src/misroute/report/__init__.py +0 -0
  38. misroute-0.2.0/src/misroute/report/html.py +46 -0
  39. misroute-0.2.0/src/misroute/report/templates/report.html.j2 +102 -0
  40. misroute-0.2.0/src/misroute/requests/__init__.py +0 -0
  41. misroute-0.2.0/src/misroute/requests/gate.py +107 -0
  42. misroute-0.2.0/src/misroute/requests/generate.py +74 -0
  43. misroute-0.2.0/src/misroute/requests/validate.py +30 -0
  44. misroute-0.2.0/src/misroute/sandbox/__init__.py +0 -0
  45. misroute-0.2.0/src/misroute/sandbox/fake_tools.py +90 -0
  46. misroute-0.2.0/src/misroute/sandbox/monty_runner.py +45 -0
  47. misroute-0.2.0/src/misroute/server/__init__.py +0 -0
  48. misroute-0.2.0/src/misroute/server/catalog.py +110 -0
  49. misroute-0.2.0/src/misroute/server/connect.py +58 -0
  50. misroute-0.2.0/src/misroute/site/__init__.py +0 -0
  51. misroute-0.2.0/src/misroute/site/build.py +41 -0
  52. misroute-0.2.0/src/misroute/site/templates/index.html.j2 +49 -0
  53. misroute-0.2.0/src/misroute/store/__init__.py +0 -0
  54. misroute-0.2.0/src/misroute/store/cache.py +76 -0
  55. misroute-0.2.0/src/misroute/store/runs.py +90 -0
  56. misroute-0.2.0/src/misroute/testing/__init__.py +0 -0
  57. misroute-0.2.0/src/misroute/testing/fake_server.py +56 -0
  58. misroute-0.2.0/src/misroute/trial.py +67 -0
  59. misroute-0.2.0/src/misroute/ui/__init__.py +0 -0
  60. misroute-0.2.0/src/misroute/ui/app.py +164 -0
  61. misroute-0.2.0/src/misroute/ui/index.html +149 -0
misroute-0.2.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Philip André Haugen
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,265 @@
1
+ Metadata-Version: 2.4
2
+ Name: misroute
3
+ Version: 0.2.0
4
+ Summary: Find the tools your agent confuses and test whether a fix helps, against a neutral edit
5
+ Keywords: mcp,agents,llm,tool-use,evaluation
6
+ Author: philippyt
7
+ Author-email: philippyt <philipaha@proton.me>
8
+ License-Expression: MIT
9
+ License-File: LICENSE
10
+ Classifier: Development Status :: 3 - Alpha
11
+ Classifier: Intended Audience :: Developers
12
+ Classifier: Programming Language :: Python :: 3
13
+ Classifier: Programming Language :: Python :: 3.11
14
+ Classifier: Programming Language :: Python :: 3.12
15
+ Classifier: Programming Language :: Python :: 3.13
16
+ Classifier: Topic :: Software Development :: Testing
17
+ Requires-Dist: anthropic>=2.3.0
18
+ Requires-Dist: duckdb>=2.0.0
19
+ Requires-Dist: jinja2>=3.1.6
20
+ Requires-Dist: mcp>=2.3.0
21
+ Requires-Dist: numpy>=2.4.6
22
+ Requires-Dist: openai>=3.41.0
23
+ Requires-Dist: pydantic>=2.14.2
24
+ Requires-Dist: pydantic-monty>=1.4.0
25
+ Requires-Dist: pyyaml>=6.0.3
26
+ Requires-Dist: rich>=15.0.0
27
+ Requires-Dist: scipy>=1.17.1
28
+ Requires-Dist: typer>=0.27.3
29
+ Requires-Dist: opentelemetry-sdk>=1.45.2 ; extra == 'otel'
30
+ Requires-Python: >=3.11
31
+ Provides-Extra: otel
32
+ Description-Content-Type: text/markdown
33
+
34
+ # misroute
35
+ Find the tools your agent confuses, and test that a fix helps before you deploy it.
36
+
37
+ misroute connects to any MCP server, writes test requests for every tool, and runs them on your models in both JSON mode and code mode. It shows where each request ends up, which tools get called together, and where code-mode programs fail. For the worst cases it writes a fix and compares it with a neutral edit of the same length that adds no new information, so you only deploy a fix when its content makes the difference.
38
+
39
+ ## Highlights
40
+
41
+ - **No pair list needed.** misroute watches how your models behave and finds the confusions from that, including between tools with unrelated names.
42
+ - **JSON mode and code mode.** Code-mode programs run in a [Monty](https://github.com/pydantic/monty) sandbox against recorded fake tools, so nothing touches your server.
43
+ - **Hedging counts as a failure.** Calling both sibling tools gets flagged, and in red when the tools write data.
44
+ - **Fixes tested against a neutral edit.** Every boundary clause is compared with an edit of the same length, with errors tracked in both directions and leakage to every other tool.
45
+ - **Any model.** Anthropic, Azure Foundry, Bedrock, OpenRouter, vLLM, Ollama, LM Studio and any OpenAI-compatible endpoint.
46
+ - **Private.** Runs on your machine or in CI, and data only goes to the endpoints you configure.
47
+ - **Cost-aware.** Shows an estimate before every paid run, stops at a hard budget, and caches responses so reruns are free.
48
+ - **Production guard.** An MCP proxy that logs every real call and holds a second write to a sibling tool until someone approves it.
49
+
50
+ ## Installation
51
+
52
+ ```console
53
+ $ uv tool install misroute
54
+ ```
55
+
56
+ Or with pip:
57
+
58
+ ```console
59
+ $ pip install misroute
60
+ ```
61
+
62
+ ## Quick start
63
+
64
+ Example output from the bundled test server, run with Claude Opus 5.5 through OpenRouter pinned to the Anthropic provider:
65
+
66
+ ```console
67
+ $ misroute init
68
+ wrote misroute.yaml
69
+
70
+ $ misroute lint --draft
71
+ 9 tools, 2 with weak descriptions
72
+ get_attachment missing
73
+ draft: Returns the attachment identified by attachment_id
74
+ get_document path only
75
+ draft: Returns the full details of a single document identified by its ID
76
+
77
+ $ misroute discover
78
+ 495 model calls, est $3.43
79
+ continue [y/N]: y
80
+
81
+ tool sibling score signals seen in
82
+ get_attachment get_document 1.00 wrong 0.33 hedge 0.42 removal 1.00 opus code, opus json
83
+ get_case_comments get_case_events 0.50 hedge 0.25 removal 1.00 opus code, opus json
84
+ get_case_events update_case_meta 0.25 wrong 0.25 opus code
85
+
86
+ $ misroute fix --pair get_case_comments,get_case_events --modes json
87
+ tool sibling model mode original baseline with clause p verdict
88
+ get_case_comments get_case_events opus json 10 13 neutral 6 0.065 no difference, consider a guard rule or merging the tools
89
+
90
+ $ misroute report
91
+ wrote report.html
92
+ ```
93
+
94
+ ## Commands
95
+
96
+ | Command | What it does |
97
+ |---|---|
98
+ | `misroute lint` | Flags tools with missing, name-only or path-only descriptions, and drafts replacements for you to review |
99
+ | `misroute discover` | Runs requests for every tool against the full tool list and ranks the risky pairs |
100
+ | `misroute probe` | Tests risky pairs with harder requests that borrow words from the sibling tool |
101
+ | `misroute fix` | Compares a boundary clause with a neutral edit, or with a drafted description for undescribed tools |
102
+ | `misroute check` | Checks whether one proposed description beats the current one on held out requests |
103
+ | `misroute report` | Writes a self-contained HTML report |
104
+ | `misroute ui` | Opens a local web interface for results, runs and held writes |
105
+ | `misroute ci` | Runs discover and exits with an error when limits are exceeded |
106
+ | `misroute guard serve` | Runs an MCP proxy that logs and guards every real tool call |
107
+ | `misroute site` | Builds a static results site from one or more runs |
108
+ | `misroute serve-mcp` | Runs misroute itself as an MCP server for agents |
109
+ | `misroute catalog` | Prints tool hashes and descriptions for change-aware CI |
110
+
111
+ Every command takes `--json`, and paid commands take `--yes`, `--max-cost` and `--estimate-only`. Exit codes are `0` passed, `1` limits failed or worse, `2` not enough data to decide, `3` budget or config problem.
112
+
113
+ ## Status
114
+
115
+ | Feature | Status |
116
+ |---|---|
117
+ | lint, discover, probe and fix in JSON and code mode | tested on Opus 5.5 through OpenRouter and on a local model through Ollama |
118
+ | OpenAI compatible adapter | tested on OpenRouter and Ollama |
119
+ | Native Anthropic, Foundry and Bedrock adapters | not yet tested against real endpoints |
120
+ | Near misses | experimental, not yet validated, kept out of the ranking |
121
+ | Local UI | API and pages checked in a browser, not yet used on a real project |
122
+ | Guard proxy | tested against a local test server only |
123
+ | check, ci with changed tools, serve-mcp | tested with a mock model only |
124
+ | Multi turn mode | tested with a mock model only |
125
+
126
+ ## Features
127
+
128
+ ### Discover
129
+ Each tool gets requests that describe what the user wants without using the tool's own words. A second model keeps a request only if it picks the right tool, and a permutation gate checks that the request sets can't be told apart by length or word overlap alone. Requests run against the full tool list in rotated orders, because models pick the wrong tool more often when the distractor comes first.
130
+
131
+ Discover combines four signals:
132
+
133
+ - **Confusion matrix:** where each tool's requests land, including tools with unrelated names
134
+ - **Co-calls:** tools called together in one turn or one program
135
+ - **Removal test:** remove a tool, rerun its requests and see where they go
136
+ - **Near misses:** the probability spread across tools for local models through vLLM, and disagreement across repeated runs for hosted models
137
+
138
+ You can add real questions from your logs with `real_requests: logs.jsonl`. misroute reports them separately from the generated ones.
139
+
140
+ ### Code mode
141
+
142
+ In code mode the model gets a single `run_code` tool, with your tools available as async Python functions. misroute runs each program in Monty against recorded fakes and judges it by the calls it actually made. It catches:
143
+
144
+ - faked tools that return invented data
145
+ - pasted stubs that shadow the real function
146
+ - programs that never call anything
147
+ - invented or reused pagination cursors, and pages left unread
148
+ - unknown functions, syntax errors and runtime errors
149
+ - answers with no program at all
150
+
151
+ ### Fix
152
+
153
+ For each risky pair, misroute writes a boundary clause: one sentence per tool saying what it covers and when the other tool is the better choice. For best results, use a model from a different family to write it. Each pair then runs three ways on the same requests: the original description, a baseline, and the baseline with the clause. The baseline is a neutral edit of the same length. When lint flags a tool as undescribed, the baseline is lint's drafted description instead, since writing any description already adds information. That tells you whether the clause adds anything beyond a basic description.
154
+
155
+ - Per-pair verdicts use an exact McNemar test with Holm correction.
156
+ - Across pairs, misroute uses an exact sign-flip test.
157
+ - Errors are reported in both directions, along with any leakage to other tools.
158
+ - When the clause does no better than the baseline, misroute tells you and suggests a guard rule or merging the tools.
159
+
160
+ `misroute fix --cheap` runs fewer trials and gives indicative numbers.
161
+
162
+ ### Guard
163
+
164
+ Put misroute in front of your MCP server to watch real traffic:
165
+
166
+ ```console
167
+ $ misroute guard serve -c misroute.yaml
168
+ $ misroute guard confirm 8ba64a3fc925
169
+ approved 8ba64a3fc925
170
+ ```
171
+
172
+ Or wrap a client session in Python:
173
+
174
+ ```python
175
+ from misroute.guard.wrapper import wrap
176
+
177
+ guard = wrap(session, rules, ".misroute/guard.jsonl")
178
+ await guard.call_tool("update_case", {"case_id": "7", "status": "closed"})
179
+ ```
180
+
181
+ Guard logs every call, including calls made inside code-mode programs. It flags hedging, quick switches between sibling tools and faked tools, and it holds a second write to a sibling tool until a person approves it. To send logs to OpenTelemetry, set `guard: {otel: true}`. Spotting users who rephrase their question needs the user's messages, so that only works through the Python wrapper.
182
+
183
+ ### CI
184
+
185
+ ```yaml
186
+ limits:
187
+ max_wrong_rate: 0.05
188
+ max_write_hedge_rate: 0.0
189
+ max_code_failure_rate: 0.10
190
+ ```
191
+
192
+ ```console
193
+ $ misroute ci
194
+ opus json wrong rate 0.105 above 0.05
195
+ ```
196
+
197
+ To test only the tools a pull request changed, save the catalog from the base branch and compare:
198
+
199
+ ```yaml
200
+ - run: git checkout origin/main && misroute catalog > tools.json && git checkout -
201
+ - run: misroute ci --changed --baseline tools.json --markdown comment.md --max-cost 2
202
+ - run: gh pr comment ${{ github.event.number }} --body-file comment.md
203
+ ```
204
+
205
+ A cheap local model catches regressions in its own behaviour, which may differ from your production model's. Schedule regular runs against the model you actually deploy as well.
206
+
207
+ ## Configuration
208
+
209
+ ```yaml
210
+ server:
211
+ url: https://mcp.example.com/mcp # or command: [npx, your-server]
212
+ auth: bearer_env:MCP_TOKEN
213
+ include_tools: ["*issue*", "*pull_request*"] # optional, test a slice of a large server
214
+
215
+ models:
216
+ - name: opus-eu
217
+ provider: anthropic-foundry # anthropic, anthropic-foundry, bedrock, openai-compatible
218
+ model: claude-opus-5-5
219
+ price_in: 4
220
+ price_out: 20
221
+ - name: local
222
+ provider: openai-compatible
223
+ base_url: http://localhost:8000/v1
224
+ model: Qwen/Qwen3-14B
225
+ scoring: true # near misses from token probabilities, vLLM only
226
+
227
+ generator: local # writes the test requests
228
+ validator: opus-eu # checks them
229
+ writer: local # writes clauses and neutral edits
230
+
231
+ modes: [json, code]
232
+ max_turns: 3 # feed fake results back so a read before a write is followed to the write
233
+ budget_usd: 5.00
234
+ ```
235
+
236
+ With `max_turns: 1`, the default, a write request answered only with reads is labelled read first and kept apart from wrong picks. With more turns misroute returns recorded fake results and judges the first write the model makes.
237
+
238
+ Results are saved in `.misroute/`: a DuckDB database you can query with SQL, JSONL transcripts of every run, and a response cache.
239
+
240
+ ```console
241
+ $ duckdb .misroute/misroute.duckdb "select target, chosen, count(*) from trials where wrong group by all order by 3 desc"
242
+ ```
243
+
244
+ ## For agents
245
+
246
+ A coding agent that edits tool descriptions can check its own edits:
247
+
248
+ ```console
249
+ $ misroute check --tool get_case_comments --description "Written comments on a case. Not for status changes." --json --max-cost 1
250
+ {"schema_version": 1, "command": "check", "verdict": "better | worse | no_difference | insufficient_data", "trials": <int>, "held_out": true, "cost_usd": <float>, ...}
251
+ ```
252
+
253
+ Requests are split into a tuning set and a held out set. The verdict uses only the held out set, and after three verdicts on the same tool it is spent until you pass `--refresh`, so an agent cannot tune a description to the test requests. Verdicts are `better`, `worse`, `no_difference` or `insufficient_data`.
254
+
255
+ `misroute serve-mcp` exposes `lint_descriptions`, `draft_description`, `estimate_check`, `check_description` and `get_report` as MCP tools. [AGENTS.md](AGENTS.md), [llms.txt](llms.txt) and a [Claude Code skill](skills/misroute/SKILL.md) describe when to use each. misroute never edits your tools.
256
+
257
+ ## Python API
258
+
259
+ ```python
260
+ from misroute import Project
261
+
262
+ proj = Project.from_yaml("misroute.yaml")
263
+ found = await proj.discover(models=["local"])
264
+ result = await proj.fix(top=3, trials=96)
265
+ ```
@@ -0,0 +1,232 @@
1
+ # misroute
2
+ Find the tools your agent confuses, and test that a fix helps before you deploy it.
3
+
4
+ misroute connects to any MCP server, writes test requests for every tool, and runs them on your models in both JSON mode and code mode. It shows where each request ends up, which tools get called together, and where code-mode programs fail. For the worst cases it writes a fix and compares it with a neutral edit of the same length that adds no new information, so you only deploy a fix when its content makes the difference.
5
+
6
+ ## Highlights
7
+
8
+ - **No pair list needed.** misroute watches how your models behave and finds the confusions from that, including between tools with unrelated names.
9
+ - **JSON mode and code mode.** Code-mode programs run in a [Monty](https://github.com/pydantic/monty) sandbox against recorded fake tools, so nothing touches your server.
10
+ - **Hedging counts as a failure.** Calling both sibling tools gets flagged, and in red when the tools write data.
11
+ - **Fixes tested against a neutral edit.** Every boundary clause is compared with an edit of the same length, with errors tracked in both directions and leakage to every other tool.
12
+ - **Any model.** Anthropic, Azure Foundry, Bedrock, OpenRouter, vLLM, Ollama, LM Studio and any OpenAI-compatible endpoint.
13
+ - **Private.** Runs on your machine or in CI, and data only goes to the endpoints you configure.
14
+ - **Cost-aware.** Shows an estimate before every paid run, stops at a hard budget, and caches responses so reruns are free.
15
+ - **Production guard.** An MCP proxy that logs every real call and holds a second write to a sibling tool until someone approves it.
16
+
17
+ ## Installation
18
+
19
+ ```console
20
+ $ uv tool install misroute
21
+ ```
22
+
23
+ Or with pip:
24
+
25
+ ```console
26
+ $ pip install misroute
27
+ ```
28
+
29
+ ## Quick start
30
+
31
+ Example output from the bundled test server, run with Claude Opus 5.5 through OpenRouter pinned to the Anthropic provider:
32
+
33
+ ```console
34
+ $ misroute init
35
+ wrote misroute.yaml
36
+
37
+ $ misroute lint --draft
38
+ 9 tools, 2 with weak descriptions
39
+ get_attachment missing
40
+ draft: Returns the attachment identified by attachment_id
41
+ get_document path only
42
+ draft: Returns the full details of a single document identified by its ID
43
+
44
+ $ misroute discover
45
+ 495 model calls, est $3.43
46
+ continue [y/N]: y
47
+
48
+ tool sibling score signals seen in
49
+ get_attachment get_document 1.00 wrong 0.33 hedge 0.42 removal 1.00 opus code, opus json
50
+ get_case_comments get_case_events 0.50 hedge 0.25 removal 1.00 opus code, opus json
51
+ get_case_events update_case_meta 0.25 wrong 0.25 opus code
52
+
53
+ $ misroute fix --pair get_case_comments,get_case_events --modes json
54
+ tool sibling model mode original baseline with clause p verdict
55
+ get_case_comments get_case_events opus json 10 13 neutral 6 0.065 no difference, consider a guard rule or merging the tools
56
+
57
+ $ misroute report
58
+ wrote report.html
59
+ ```
60
+
61
+ ## Commands
62
+
63
+ | Command | What it does |
64
+ |---|---|
65
+ | `misroute lint` | Flags tools with missing, name-only or path-only descriptions, and drafts replacements for you to review |
66
+ | `misroute discover` | Runs requests for every tool against the full tool list and ranks the risky pairs |
67
+ | `misroute probe` | Tests risky pairs with harder requests that borrow words from the sibling tool |
68
+ | `misroute fix` | Compares a boundary clause with a neutral edit, or with a drafted description for undescribed tools |
69
+ | `misroute check` | Checks whether one proposed description beats the current one on held out requests |
70
+ | `misroute report` | Writes a self-contained HTML report |
71
+ | `misroute ui` | Opens a local web interface for results, runs and held writes |
72
+ | `misroute ci` | Runs discover and exits with an error when limits are exceeded |
73
+ | `misroute guard serve` | Runs an MCP proxy that logs and guards every real tool call |
74
+ | `misroute site` | Builds a static results site from one or more runs |
75
+ | `misroute serve-mcp` | Runs misroute itself as an MCP server for agents |
76
+ | `misroute catalog` | Prints tool hashes and descriptions for change-aware CI |
77
+
78
+ Every command takes `--json`, and paid commands take `--yes`, `--max-cost` and `--estimate-only`. Exit codes are `0` passed, `1` limits failed or worse, `2` not enough data to decide, `3` budget or config problem.
79
+
80
+ ## Status
81
+
82
+ | Feature | Status |
83
+ |---|---|
84
+ | lint, discover, probe and fix in JSON and code mode | tested on Opus 5.5 through OpenRouter and on a local model through Ollama |
85
+ | OpenAI compatible adapter | tested on OpenRouter and Ollama |
86
+ | Native Anthropic, Foundry and Bedrock adapters | not yet tested against real endpoints |
87
+ | Near misses | experimental, not yet validated, kept out of the ranking |
88
+ | Local UI | API and pages checked in a browser, not yet used on a real project |
89
+ | Guard proxy | tested against a local test server only |
90
+ | check, ci with changed tools, serve-mcp | tested with a mock model only |
91
+ | Multi turn mode | tested with a mock model only |
92
+
93
+ ## Features
94
+
95
+ ### Discover
96
+ Each tool gets requests that describe what the user wants without using the tool's own words. A second model keeps a request only if it picks the right tool, and a permutation gate checks that the request sets can't be told apart by length or word overlap alone. Requests run against the full tool list in rotated orders, because models pick the wrong tool more often when the distractor comes first.
97
+
98
+ Discover combines four signals:
99
+
100
+ - **Confusion matrix:** where each tool's requests land, including tools with unrelated names
101
+ - **Co-calls:** tools called together in one turn or one program
102
+ - **Removal test:** remove a tool, rerun its requests and see where they go
103
+ - **Near misses:** the probability spread across tools for local models through vLLM, and disagreement across repeated runs for hosted models
104
+
105
+ You can add real questions from your logs with `real_requests: logs.jsonl`. misroute reports them separately from the generated ones.
106
+
107
+ ### Code mode
108
+
109
+ In code mode the model gets a single `run_code` tool, with your tools available as async Python functions. misroute runs each program in Monty against recorded fakes and judges it by the calls it actually made. It catches:
110
+
111
+ - faked tools that return invented data
112
+ - pasted stubs that shadow the real function
113
+ - programs that never call anything
114
+ - invented or reused pagination cursors, and pages left unread
115
+ - unknown functions, syntax errors and runtime errors
116
+ - answers with no program at all
117
+
118
+ ### Fix
119
+
120
+ For each risky pair, misroute writes a boundary clause: one sentence per tool saying what it covers and when the other tool is the better choice. For best results, use a model from a different family to write it. Each pair then runs three ways on the same requests: the original description, a baseline, and the baseline with the clause. The baseline is a neutral edit of the same length. When lint flags a tool as undescribed, the baseline is lint's drafted description instead, since writing any description already adds information. That tells you whether the clause adds anything beyond a basic description.
121
+
122
+ - Per-pair verdicts use an exact McNemar test with Holm correction.
123
+ - Across pairs, misroute uses an exact sign-flip test.
124
+ - Errors are reported in both directions, along with any leakage to other tools.
125
+ - When the clause does no better than the baseline, misroute tells you and suggests a guard rule or merging the tools.
126
+
127
+ `misroute fix --cheap` runs fewer trials and gives indicative numbers.
128
+
129
+ ### Guard
130
+
131
+ Put misroute in front of your MCP server to watch real traffic:
132
+
133
+ ```console
134
+ $ misroute guard serve -c misroute.yaml
135
+ $ misroute guard confirm 8ba64a3fc925
136
+ approved 8ba64a3fc925
137
+ ```
138
+
139
+ Or wrap a client session in Python:
140
+
141
+ ```python
142
+ from misroute.guard.wrapper import wrap
143
+
144
+ guard = wrap(session, rules, ".misroute/guard.jsonl")
145
+ await guard.call_tool("update_case", {"case_id": "7", "status": "closed"})
146
+ ```
147
+
148
+ Guard logs every call, including calls made inside code-mode programs. It flags hedging, quick switches between sibling tools and faked tools, and it holds a second write to a sibling tool until a person approves it. To send logs to OpenTelemetry, set `guard: {otel: true}`. Spotting users who rephrase their question needs the user's messages, so that only works through the Python wrapper.
149
+
150
+ ### CI
151
+
152
+ ```yaml
153
+ limits:
154
+ max_wrong_rate: 0.05
155
+ max_write_hedge_rate: 0.0
156
+ max_code_failure_rate: 0.10
157
+ ```
158
+
159
+ ```console
160
+ $ misroute ci
161
+ opus json wrong rate 0.105 above 0.05
162
+ ```
163
+
164
+ To test only the tools a pull request changed, save the catalog from the base branch and compare:
165
+
166
+ ```yaml
167
+ - run: git checkout origin/main && misroute catalog > tools.json && git checkout -
168
+ - run: misroute ci --changed --baseline tools.json --markdown comment.md --max-cost 2
169
+ - run: gh pr comment ${{ github.event.number }} --body-file comment.md
170
+ ```
171
+
172
+ A cheap local model catches regressions in its own behaviour, which may differ from your production model's. Schedule regular runs against the model you actually deploy as well.
173
+
174
+ ## Configuration
175
+
176
+ ```yaml
177
+ server:
178
+ url: https://mcp.example.com/mcp # or command: [npx, your-server]
179
+ auth: bearer_env:MCP_TOKEN
180
+ include_tools: ["*issue*", "*pull_request*"] # optional, test a slice of a large server
181
+
182
+ models:
183
+ - name: opus-eu
184
+ provider: anthropic-foundry # anthropic, anthropic-foundry, bedrock, openai-compatible
185
+ model: claude-opus-5-5
186
+ price_in: 4
187
+ price_out: 20
188
+ - name: local
189
+ provider: openai-compatible
190
+ base_url: http://localhost:8000/v1
191
+ model: Qwen/Qwen3-14B
192
+ scoring: true # near misses from token probabilities, vLLM only
193
+
194
+ generator: local # writes the test requests
195
+ validator: opus-eu # checks them
196
+ writer: local # writes clauses and neutral edits
197
+
198
+ modes: [json, code]
199
+ max_turns: 3 # feed fake results back so a read before a write is followed to the write
200
+ budget_usd: 5.00
201
+ ```
202
+
203
+ With `max_turns: 1`, the default, a write request answered only with reads is labelled read first and kept apart from wrong picks. With more turns misroute returns recorded fake results and judges the first write the model makes.
204
+
205
+ Results are saved in `.misroute/`: a DuckDB database you can query with SQL, JSONL transcripts of every run, and a response cache.
206
+
207
+ ```console
208
+ $ duckdb .misroute/misroute.duckdb "select target, chosen, count(*) from trials where wrong group by all order by 3 desc"
209
+ ```
210
+
211
+ ## For agents
212
+
213
+ A coding agent that edits tool descriptions can check its own edits:
214
+
215
+ ```console
216
+ $ misroute check --tool get_case_comments --description "Written comments on a case. Not for status changes." --json --max-cost 1
217
+ {"schema_version": 1, "command": "check", "verdict": "better | worse | no_difference | insufficient_data", "trials": <int>, "held_out": true, "cost_usd": <float>, ...}
218
+ ```
219
+
220
+ Requests are split into a tuning set and a held out set. The verdict uses only the held out set, and after three verdicts on the same tool it is spent until you pass `--refresh`, so an agent cannot tune a description to the test requests. Verdicts are `better`, `worse`, `no_difference` or `insufficient_data`.
221
+
222
+ `misroute serve-mcp` exposes `lint_descriptions`, `draft_description`, `estimate_check`, `check_description` and `get_report` as MCP tools. [AGENTS.md](AGENTS.md), [llms.txt](llms.txt) and a [Claude Code skill](skills/misroute/SKILL.md) describe when to use each. misroute never edits your tools.
223
+
224
+ ## Python API
225
+
226
+ ```python
227
+ from misroute import Project
228
+
229
+ proj = Project.from_yaml("misroute.yaml")
230
+ found = await proj.discover(models=["local"])
231
+ result = await proj.fix(top=3, trials=96)
232
+ ```
@@ -0,0 +1,75 @@
1
+ [project]
2
+ name = "misroute"
3
+ version = "0.2.0"
4
+ description = "Find the tools your agent confuses and test whether a fix helps, against a neutral edit"
5
+ authors = [
6
+ { name = "philippyt", email = "philipaha@proton.me" }
7
+ ]
8
+ requires-python = ">=3.11"
9
+ readme = "README.md"
10
+ license = "MIT"
11
+ license-files = ["LICENSE"]
12
+ keywords = ["mcp", "agents", "llm", "tool-use", "evaluation"]
13
+ classifiers = [
14
+ "Development Status :: 3 - Alpha",
15
+ "Intended Audience :: Developers",
16
+ "Programming Language :: Python :: 3",
17
+ "Programming Language :: Python :: 3.11",
18
+ "Programming Language :: Python :: 3.12",
19
+ "Programming Language :: Python :: 3.13",
20
+ "Topic :: Software Development :: Testing",
21
+ ]
22
+ dependencies = [
23
+ "anthropic>=2.3.0",
24
+ "duckdb>=2.0.0",
25
+ "jinja2>=3.1.6",
26
+ "mcp>=2.3.0",
27
+ "numpy>=2.4.6",
28
+ "openai>=3.41.0",
29
+ "pydantic>=2.14.2",
30
+ "pydantic-monty>=1.4.0",
31
+ "pyyaml>=6.0.3",
32
+ "rich>=15.0.0",
33
+ "scipy>=1.17.1",
34
+ "typer>=0.27.3",
35
+ ]
36
+
37
+ [project.optional-dependencies]
38
+ otel = [
39
+ "opentelemetry-sdk>=1.45.2",
40
+ ]
41
+
42
+ [project.scripts]
43
+ misroute = "misroute.cli:app"
44
+
45
+ [build-system]
46
+ requires = ["uv_build>=0.10.3,<0.11.0"]
47
+ build-backend = "uv_build"
48
+
49
+ [dependency-groups]
50
+ dev = [
51
+ "pytest>=9.2.0",
52
+ "pytest-asyncio>=1.4.1",
53
+ "ruff>=0.17.3",
54
+ "ty>=0.0.95",
55
+ ]
56
+
57
+ [tool.ruff]
58
+ line-length = 120
59
+ target-version = "py311"
60
+
61
+ [tool.ruff.lint]
62
+ select = ["E", "F", "I", "B", "UP", "SIM", "RUF"]
63
+ ignore = ["RUF001", "RUF002", "RUF003", "SIM905"]
64
+
65
+ [tool.ruff.lint.isort]
66
+ no-lines-before = ["future", "standard-library", "third-party", "first-party", "local-folder"]
67
+
68
+ [tool.pytest.ini_options]
69
+ asyncio_mode = "auto"
70
+ asyncio_default_fixture_loop_scope = "function"
71
+ markers = ["live: needs a running model endpoint"]
72
+ addopts = "-m 'not live'"
73
+
74
+ [tool.ty.environment]
75
+ python-version = "3.11"
@@ -0,0 +1,3 @@
1
+ from misroute.project import Project
2
+
3
+ __all__ = ["Project"]