misroute 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- misroute-0.2.0/LICENSE +21 -0
- misroute-0.2.0/PKG-INFO +265 -0
- misroute-0.2.0/README.md +232 -0
- misroute-0.2.0/pyproject.toml +75 -0
- misroute-0.2.0/src/misroute/__init__.py +3 -0
- misroute-0.2.0/src/misroute/agent_server.py +98 -0
- misroute-0.2.0/src/misroute/check.py +151 -0
- misroute-0.2.0/src/misroute/cli.py +460 -0
- misroute-0.2.0/src/misroute/config.py +95 -0
- misroute-0.2.0/src/misroute/detectors/__init__.py +0 -0
- misroute-0.2.0/src/misroute/detectors/codemode.py +60 -0
- misroute-0.2.0/src/misroute/detectors/confusion.py +40 -0
- misroute-0.2.0/src/misroute/detectors/hedge.py +17 -0
- misroute-0.2.0/src/misroute/detectors/nearmiss.py +66 -0
- misroute-0.2.0/src/misroute/detectors/rank.py +72 -0
- misroute-0.2.0/src/misroute/detectors/removal.py +11 -0
- misroute-0.2.0/src/misroute/fix/__init__.py +0 -0
- misroute-0.2.0/src/misroute/fix/clause.py +94 -0
- misroute-0.2.0/src/misroute/fix/stats.py +52 -0
- misroute-0.2.0/src/misroute/guard/__init__.py +0 -0
- misroute-0.2.0/src/misroute/guard/proxy.py +44 -0
- misroute-0.2.0/src/misroute/guard/rules.py +49 -0
- misroute-0.2.0/src/misroute/guard/wrapper.py +119 -0
- misroute-0.2.0/src/misroute/lint/__init__.py +0 -0
- misroute-0.2.0/src/misroute/lint/descriptions.py +50 -0
- misroute-0.2.0/src/misroute/models/__init__.py +16 -0
- misroute-0.2.0/src/misroute/models/anthropic.py +70 -0
- misroute-0.2.0/src/misroute/models/base.py +79 -0
- misroute-0.2.0/src/misroute/models/mock.py +158 -0
- misroute-0.2.0/src/misroute/models/openai_compat.py +118 -0
- misroute-0.2.0/src/misroute/modes/__init__.py +0 -0
- misroute-0.2.0/src/misroute/modes/code_mode.py +87 -0
- misroute-0.2.0/src/misroute/modes/json_mode.py +71 -0
- misroute-0.2.0/src/misroute/modes/stubs.py +40 -0
- misroute-0.2.0/src/misroute/project.py +433 -0
- misroute-0.2.0/src/misroute/py.typed +0 -0
- misroute-0.2.0/src/misroute/report/__init__.py +0 -0
- misroute-0.2.0/src/misroute/report/html.py +46 -0
- misroute-0.2.0/src/misroute/report/templates/report.html.j2 +102 -0
- misroute-0.2.0/src/misroute/requests/__init__.py +0 -0
- misroute-0.2.0/src/misroute/requests/gate.py +107 -0
- misroute-0.2.0/src/misroute/requests/generate.py +74 -0
- misroute-0.2.0/src/misroute/requests/validate.py +30 -0
- misroute-0.2.0/src/misroute/sandbox/__init__.py +0 -0
- misroute-0.2.0/src/misroute/sandbox/fake_tools.py +90 -0
- misroute-0.2.0/src/misroute/sandbox/monty_runner.py +45 -0
- misroute-0.2.0/src/misroute/server/__init__.py +0 -0
- misroute-0.2.0/src/misroute/server/catalog.py +110 -0
- misroute-0.2.0/src/misroute/server/connect.py +58 -0
- misroute-0.2.0/src/misroute/site/__init__.py +0 -0
- misroute-0.2.0/src/misroute/site/build.py +41 -0
- misroute-0.2.0/src/misroute/site/templates/index.html.j2 +49 -0
- misroute-0.2.0/src/misroute/store/__init__.py +0 -0
- misroute-0.2.0/src/misroute/store/cache.py +76 -0
- misroute-0.2.0/src/misroute/store/runs.py +90 -0
- misroute-0.2.0/src/misroute/testing/__init__.py +0 -0
- misroute-0.2.0/src/misroute/testing/fake_server.py +56 -0
- misroute-0.2.0/src/misroute/trial.py +67 -0
- misroute-0.2.0/src/misroute/ui/__init__.py +0 -0
- misroute-0.2.0/src/misroute/ui/app.py +164 -0
- misroute-0.2.0/src/misroute/ui/index.html +149 -0
misroute-0.2.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Philip André Haugen
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
misroute-0.2.0/PKG-INFO
ADDED
|
@@ -0,0 +1,265 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: misroute
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Find the tools your agent confuses and test whether a fix helps, against a neutral edit
|
|
5
|
+
Keywords: mcp,agents,llm,tool-use,evaluation
|
|
6
|
+
Author: philippyt
|
|
7
|
+
Author-email: philippyt <philipaha@proton.me>
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Classifier: Topic :: Software Development :: Testing
|
|
17
|
+
Requires-Dist: anthropic>=2.3.0
|
|
18
|
+
Requires-Dist: duckdb>=2.0.0
|
|
19
|
+
Requires-Dist: jinja2>=3.1.6
|
|
20
|
+
Requires-Dist: mcp>=2.3.0
|
|
21
|
+
Requires-Dist: numpy>=2.4.6
|
|
22
|
+
Requires-Dist: openai>=3.41.0
|
|
23
|
+
Requires-Dist: pydantic>=2.14.2
|
|
24
|
+
Requires-Dist: pydantic-monty>=1.4.0
|
|
25
|
+
Requires-Dist: pyyaml>=6.0.3
|
|
26
|
+
Requires-Dist: rich>=15.0.0
|
|
27
|
+
Requires-Dist: scipy>=1.17.1
|
|
28
|
+
Requires-Dist: typer>=0.27.3
|
|
29
|
+
Requires-Dist: opentelemetry-sdk>=1.45.2 ; extra == 'otel'
|
|
30
|
+
Requires-Python: >=3.11
|
|
31
|
+
Provides-Extra: otel
|
|
32
|
+
Description-Content-Type: text/markdown
|
|
33
|
+
|
|
34
|
+
# misroute
|
|
35
|
+
Find the tools your agent confuses, and test that a fix helps before you deploy it.
|
|
36
|
+
|
|
37
|
+
misroute connects to any MCP server, writes test requests for every tool, and runs them on your models in both JSON mode and code mode. It shows where each request ends up, which tools get called together, and where code-mode programs fail. For the worst cases it writes a fix and compares it with a neutral edit of the same length that adds no new information, so you only deploy a fix when its content makes the difference.
|
|
38
|
+
|
|
39
|
+
## Highlights
|
|
40
|
+
|
|
41
|
+
- **No pair list needed.** misroute watches how your models behave and finds the confusions from that, including between tools with unrelated names.
|
|
42
|
+
- **JSON mode and code mode.** Code-mode programs run in a [Monty](https://github.com/pydantic/monty) sandbox against recorded fake tools, so nothing touches your server.
|
|
43
|
+
- **Hedging counts as a failure.** Calling both sibling tools gets flagged, and in red when the tools write data.
|
|
44
|
+
- **Fixes tested against a neutral edit.** Every boundary clause is compared with an edit of the same length, with errors tracked in both directions and leakage to every other tool.
|
|
45
|
+
- **Any model.** Anthropic, Azure Foundry, Bedrock, OpenRouter, vLLM, Ollama, LM Studio and any OpenAI-compatible endpoint.
|
|
46
|
+
- **Private.** Runs on your machine or in CI, and data only goes to the endpoints you configure.
|
|
47
|
+
- **Cost-aware.** Shows an estimate before every paid run, stops at a hard budget, and caches responses so reruns are free.
|
|
48
|
+
- **Production guard.** An MCP proxy that logs every real call and holds a second write to a sibling tool until someone approves it.
|
|
49
|
+
|
|
50
|
+
## Installation
|
|
51
|
+
|
|
52
|
+
```console
|
|
53
|
+
$ uv tool install misroute
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
Or with pip:
|
|
57
|
+
|
|
58
|
+
```console
|
|
59
|
+
$ pip install misroute
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
## Quick start
|
|
63
|
+
|
|
64
|
+
Example output from the bundled test server, run with Claude Opus 5.5 through OpenRouter pinned to the Anthropic provider:
|
|
65
|
+
|
|
66
|
+
```console
|
|
67
|
+
$ misroute init
|
|
68
|
+
wrote misroute.yaml
|
|
69
|
+
|
|
70
|
+
$ misroute lint --draft
|
|
71
|
+
9 tools, 2 with weak descriptions
|
|
72
|
+
get_attachment missing
|
|
73
|
+
draft: Returns the attachment identified by attachment_id
|
|
74
|
+
get_document path only
|
|
75
|
+
draft: Returns the full details of a single document identified by its ID
|
|
76
|
+
|
|
77
|
+
$ misroute discover
|
|
78
|
+
495 model calls, est $3.43
|
|
79
|
+
continue [y/N]: y
|
|
80
|
+
|
|
81
|
+
tool sibling score signals seen in
|
|
82
|
+
get_attachment get_document 1.00 wrong 0.33 hedge 0.42 removal 1.00 opus code, opus json
|
|
83
|
+
get_case_comments get_case_events 0.50 hedge 0.25 removal 1.00 opus code, opus json
|
|
84
|
+
get_case_events update_case_meta 0.25 wrong 0.25 opus code
|
|
85
|
+
|
|
86
|
+
$ misroute fix --pair get_case_comments,get_case_events --modes json
|
|
87
|
+
tool sibling model mode original baseline with clause p verdict
|
|
88
|
+
get_case_comments get_case_events opus json 10 13 neutral 6 0.065 no difference, consider a guard rule or merging the tools
|
|
89
|
+
|
|
90
|
+
$ misroute report
|
|
91
|
+
wrote report.html
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
## Commands
|
|
95
|
+
|
|
96
|
+
| Command | What it does |
|
|
97
|
+
|---|---|
|
|
98
|
+
| `misroute lint` | Flags tools with missing, name-only or path-only descriptions, and drafts replacements for you to review |
|
|
99
|
+
| `misroute discover` | Runs requests for every tool against the full tool list and ranks the risky pairs |
|
|
100
|
+
| `misroute probe` | Tests risky pairs with harder requests that borrow words from the sibling tool |
|
|
101
|
+
| `misroute fix` | Compares a boundary clause with a neutral edit, or with a drafted description for undescribed tools |
|
|
102
|
+
| `misroute check` | Checks whether one proposed description beats the current one on held out requests |
|
|
103
|
+
| `misroute report` | Writes a self-contained HTML report |
|
|
104
|
+
| `misroute ui` | Opens a local web interface for results, runs and held writes |
|
|
105
|
+
| `misroute ci` | Runs discover and exits with an error when limits are exceeded |
|
|
106
|
+
| `misroute guard serve` | Runs an MCP proxy that logs and guards every real tool call |
|
|
107
|
+
| `misroute site` | Builds a static results site from one or more runs |
|
|
108
|
+
| `misroute serve-mcp` | Runs misroute itself as an MCP server for agents |
|
|
109
|
+
| `misroute catalog` | Prints tool hashes and descriptions for change-aware CI |
|
|
110
|
+
|
|
111
|
+
Every command takes `--json`, and paid commands take `--yes`, `--max-cost` and `--estimate-only`. Exit codes are `0` passed, `1` limits failed or worse, `2` not enough data to decide, `3` budget or config problem.
|
|
112
|
+
|
|
113
|
+
## Status
|
|
114
|
+
|
|
115
|
+
| Feature | Status |
|
|
116
|
+
|---|---|
|
|
117
|
+
| lint, discover, probe and fix in JSON and code mode | tested on Opus 5.5 through OpenRouter and on a local model through Ollama |
|
|
118
|
+
| OpenAI compatible adapter | tested on OpenRouter and Ollama |
|
|
119
|
+
| Native Anthropic, Foundry and Bedrock adapters | not yet tested against real endpoints |
|
|
120
|
+
| Near misses | experimental, not yet validated, kept out of the ranking |
|
|
121
|
+
| Local UI | API and pages checked in a browser, not yet used on a real project |
|
|
122
|
+
| Guard proxy | tested against a local test server only |
|
|
123
|
+
| check, ci with changed tools, serve-mcp | tested with a mock model only |
|
|
124
|
+
| Multi turn mode | tested with a mock model only |
|
|
125
|
+
|
|
126
|
+
## Features
|
|
127
|
+
|
|
128
|
+
### Discover
|
|
129
|
+
Each tool gets requests that describe what the user wants without using the tool's own words. A second model keeps a request only if it picks the right tool, and a permutation gate checks that the request sets can't be told apart by length or word overlap alone. Requests run against the full tool list in rotated orders, because models pick the wrong tool more often when the distractor comes first.
|
|
130
|
+
|
|
131
|
+
Discover combines four signals:
|
|
132
|
+
|
|
133
|
+
- **Confusion matrix:** where each tool's requests land, including tools with unrelated names
|
|
134
|
+
- **Co-calls:** tools called together in one turn or one program
|
|
135
|
+
- **Removal test:** remove a tool, rerun its requests and see where they go
|
|
136
|
+
- **Near misses:** the probability spread across tools for local models through vLLM, and disagreement across repeated runs for hosted models
|
|
137
|
+
|
|
138
|
+
You can add real questions from your logs with `real_requests: logs.jsonl`. misroute reports them separately from the generated ones.
|
|
139
|
+
|
|
140
|
+
### Code mode
|
|
141
|
+
|
|
142
|
+
In code mode the model gets a single `run_code` tool, with your tools available as async Python functions. misroute runs each program in Monty against recorded fakes and judges it by the calls it actually made. It catches:
|
|
143
|
+
|
|
144
|
+
- faked tools that return invented data
|
|
145
|
+
- pasted stubs that shadow the real function
|
|
146
|
+
- programs that never call anything
|
|
147
|
+
- invented or reused pagination cursors, and pages left unread
|
|
148
|
+
- unknown functions, syntax errors and runtime errors
|
|
149
|
+
- answers with no program at all
|
|
150
|
+
|
|
151
|
+
### Fix
|
|
152
|
+
|
|
153
|
+
For each risky pair, misroute writes a boundary clause: one sentence per tool saying what it covers and when the other tool is the better choice. For best results, use a model from a different family to write it. Each pair then runs three ways on the same requests: the original description, a baseline, and the baseline with the clause. The baseline is a neutral edit of the same length. When lint flags a tool as undescribed, the baseline is lint's drafted description instead, since writing any description already adds information. That tells you whether the clause adds anything beyond a basic description.
|
|
154
|
+
|
|
155
|
+
- Per-pair verdicts use an exact McNemar test with Holm correction.
|
|
156
|
+
- Across pairs, misroute uses an exact sign-flip test.
|
|
157
|
+
- Errors are reported in both directions, along with any leakage to other tools.
|
|
158
|
+
- When the clause does no better than the baseline, misroute tells you and suggests a guard rule or merging the tools.
|
|
159
|
+
|
|
160
|
+
`misroute fix --cheap` runs fewer trials and gives indicative numbers.
|
|
161
|
+
|
|
162
|
+
### Guard
|
|
163
|
+
|
|
164
|
+
Put misroute in front of your MCP server to watch real traffic:
|
|
165
|
+
|
|
166
|
+
```console
|
|
167
|
+
$ misroute guard serve -c misroute.yaml
|
|
168
|
+
$ misroute guard confirm 8ba64a3fc925
|
|
169
|
+
approved 8ba64a3fc925
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
Or wrap a client session in Python:
|
|
173
|
+
|
|
174
|
+
```python
|
|
175
|
+
from misroute.guard.wrapper import wrap
|
|
176
|
+
|
|
177
|
+
guard = wrap(session, rules, ".misroute/guard.jsonl")
|
|
178
|
+
await guard.call_tool("update_case", {"case_id": "7", "status": "closed"})
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
Guard logs every call, including calls made inside code-mode programs. It flags hedging, quick switches between sibling tools and faked tools, and it holds a second write to a sibling tool until a person approves it. To send logs to OpenTelemetry, set `guard: {otel: true}`. Spotting users who rephrase their question needs the user's messages, so that only works through the Python wrapper.
|
|
182
|
+
|
|
183
|
+
### CI
|
|
184
|
+
|
|
185
|
+
```yaml
|
|
186
|
+
limits:
|
|
187
|
+
max_wrong_rate: 0.05
|
|
188
|
+
max_write_hedge_rate: 0.0
|
|
189
|
+
max_code_failure_rate: 0.10
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
```console
|
|
193
|
+
$ misroute ci
|
|
194
|
+
opus json wrong rate 0.105 above 0.05
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
To test only the tools a pull request changed, save the catalog from the base branch and compare:
|
|
198
|
+
|
|
199
|
+
```yaml
|
|
200
|
+
- run: git checkout origin/main && misroute catalog > tools.json && git checkout -
|
|
201
|
+
- run: misroute ci --changed --baseline tools.json --markdown comment.md --max-cost 2
|
|
202
|
+
- run: gh pr comment ${{ github.event.number }} --body-file comment.md
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
A cheap local model catches regressions in its own behaviour, which may differ from your production model's. Schedule regular runs against the model you actually deploy as well.
|
|
206
|
+
|
|
207
|
+
## Configuration
|
|
208
|
+
|
|
209
|
+
```yaml
|
|
210
|
+
server:
|
|
211
|
+
url: https://mcp.example.com/mcp # or command: [npx, your-server]
|
|
212
|
+
auth: bearer_env:MCP_TOKEN
|
|
213
|
+
include_tools: ["*issue*", "*pull_request*"] # optional, test a slice of a large server
|
|
214
|
+
|
|
215
|
+
models:
|
|
216
|
+
- name: opus-eu
|
|
217
|
+
provider: anthropic-foundry # anthropic, anthropic-foundry, bedrock, openai-compatible
|
|
218
|
+
model: claude-opus-5-5
|
|
219
|
+
price_in: 4
|
|
220
|
+
price_out: 20
|
|
221
|
+
- name: local
|
|
222
|
+
provider: openai-compatible
|
|
223
|
+
base_url: http://localhost:8000/v1
|
|
224
|
+
model: Qwen/Qwen3-14B
|
|
225
|
+
scoring: true # near misses from token probabilities, vLLM only
|
|
226
|
+
|
|
227
|
+
generator: local # writes the test requests
|
|
228
|
+
validator: opus-eu # checks them
|
|
229
|
+
writer: local # writes clauses and neutral edits
|
|
230
|
+
|
|
231
|
+
modes: [json, code]
|
|
232
|
+
max_turns: 3 # feed fake results back so a read before a write is followed to the write
|
|
233
|
+
budget_usd: 5.00
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
With `max_turns: 1`, the default, a write request answered only with reads is labelled read first and kept apart from wrong picks. With more turns misroute returns recorded fake results and judges the first write the model makes.
|
|
237
|
+
|
|
238
|
+
Results are saved in `.misroute/`: a DuckDB database you can query with SQL, JSONL transcripts of every run, and a response cache.
|
|
239
|
+
|
|
240
|
+
```console
|
|
241
|
+
$ duckdb .misroute/misroute.duckdb "select target, chosen, count(*) from trials where wrong group by all order by 3 desc"
|
|
242
|
+
```
|
|
243
|
+
|
|
244
|
+
## For agents
|
|
245
|
+
|
|
246
|
+
A coding agent that edits tool descriptions can check its own edits:
|
|
247
|
+
|
|
248
|
+
```console
|
|
249
|
+
$ misroute check --tool get_case_comments --description "Written comments on a case. Not for status changes." --json --max-cost 1
|
|
250
|
+
{"schema_version": 1, "command": "check", "verdict": "better | worse | no_difference | insufficient_data", "trials": <int>, "held_out": true, "cost_usd": <float>, ...}
|
|
251
|
+
```
|
|
252
|
+
|
|
253
|
+
Requests are split into a tuning set and a held out set. The verdict uses only the held out set, and after three verdicts on the same tool it is spent until you pass `--refresh`, so an agent cannot tune a description to the test requests. Verdicts are `better`, `worse`, `no_difference` or `insufficient_data`.
|
|
254
|
+
|
|
255
|
+
`misroute serve-mcp` exposes `lint_descriptions`, `draft_description`, `estimate_check`, `check_description` and `get_report` as MCP tools. [AGENTS.md](AGENTS.md), [llms.txt](llms.txt) and a [Claude Code skill](skills/misroute/SKILL.md) describe when to use each. misroute never edits your tools.
|
|
256
|
+
|
|
257
|
+
## Python API
|
|
258
|
+
|
|
259
|
+
```python
|
|
260
|
+
from misroute import Project
|
|
261
|
+
|
|
262
|
+
proj = Project.from_yaml("misroute.yaml")
|
|
263
|
+
found = await proj.discover(models=["local"])
|
|
264
|
+
result = await proj.fix(top=3, trials=96)
|
|
265
|
+
```
|
misroute-0.2.0/README.md
ADDED
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
# misroute
|
|
2
|
+
Find the tools your agent confuses, and test that a fix helps before you deploy it.
|
|
3
|
+
|
|
4
|
+
misroute connects to any MCP server, writes test requests for every tool, and runs them on your models in both JSON mode and code mode. It shows where each request ends up, which tools get called together, and where code-mode programs fail. For the worst cases it writes a fix and compares it with a neutral edit of the same length that adds no new information, so you only deploy a fix when its content makes the difference.
|
|
5
|
+
|
|
6
|
+
## Highlights
|
|
7
|
+
|
|
8
|
+
- **No pair list needed.** misroute watches how your models behave and finds the confusions from that, including between tools with unrelated names.
|
|
9
|
+
- **JSON mode and code mode.** Code-mode programs run in a [Monty](https://github.com/pydantic/monty) sandbox against recorded fake tools, so nothing touches your server.
|
|
10
|
+
- **Hedging counts as a failure.** Calling both sibling tools gets flagged, and in red when the tools write data.
|
|
11
|
+
- **Fixes tested against a neutral edit.** Every boundary clause is compared with an edit of the same length, with errors tracked in both directions and leakage to every other tool.
|
|
12
|
+
- **Any model.** Anthropic, Azure Foundry, Bedrock, OpenRouter, vLLM, Ollama, LM Studio and any OpenAI-compatible endpoint.
|
|
13
|
+
- **Private.** Runs on your machine or in CI, and data only goes to the endpoints you configure.
|
|
14
|
+
- **Cost-aware.** Shows an estimate before every paid run, stops at a hard budget, and caches responses so reruns are free.
|
|
15
|
+
- **Production guard.** An MCP proxy that logs every real call and holds a second write to a sibling tool until someone approves it.
|
|
16
|
+
|
|
17
|
+
## Installation
|
|
18
|
+
|
|
19
|
+
```console
|
|
20
|
+
$ uv tool install misroute
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
Or with pip:
|
|
24
|
+
|
|
25
|
+
```console
|
|
26
|
+
$ pip install misroute
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
## Quick start
|
|
30
|
+
|
|
31
|
+
Example output from the bundled test server, run with Claude Opus 5.5 through OpenRouter pinned to the Anthropic provider:
|
|
32
|
+
|
|
33
|
+
```console
|
|
34
|
+
$ misroute init
|
|
35
|
+
wrote misroute.yaml
|
|
36
|
+
|
|
37
|
+
$ misroute lint --draft
|
|
38
|
+
9 tools, 2 with weak descriptions
|
|
39
|
+
get_attachment missing
|
|
40
|
+
draft: Returns the attachment identified by attachment_id
|
|
41
|
+
get_document path only
|
|
42
|
+
draft: Returns the full details of a single document identified by its ID
|
|
43
|
+
|
|
44
|
+
$ misroute discover
|
|
45
|
+
495 model calls, est $3.43
|
|
46
|
+
continue [y/N]: y
|
|
47
|
+
|
|
48
|
+
tool sibling score signals seen in
|
|
49
|
+
get_attachment get_document 1.00 wrong 0.33 hedge 0.42 removal 1.00 opus code, opus json
|
|
50
|
+
get_case_comments get_case_events 0.50 hedge 0.25 removal 1.00 opus code, opus json
|
|
51
|
+
get_case_events update_case_meta 0.25 wrong 0.25 opus code
|
|
52
|
+
|
|
53
|
+
$ misroute fix --pair get_case_comments,get_case_events --modes json
|
|
54
|
+
tool sibling model mode original baseline with clause p verdict
|
|
55
|
+
get_case_comments get_case_events opus json 10 13 neutral 6 0.065 no difference, consider a guard rule or merging the tools
|
|
56
|
+
|
|
57
|
+
$ misroute report
|
|
58
|
+
wrote report.html
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
## Commands
|
|
62
|
+
|
|
63
|
+
| Command | What it does |
|
|
64
|
+
|---|---|
|
|
65
|
+
| `misroute lint` | Flags tools with missing, name-only or path-only descriptions, and drafts replacements for you to review |
|
|
66
|
+
| `misroute discover` | Runs requests for every tool against the full tool list and ranks the risky pairs |
|
|
67
|
+
| `misroute probe` | Tests risky pairs with harder requests that borrow words from the sibling tool |
|
|
68
|
+
| `misroute fix` | Compares a boundary clause with a neutral edit, or with a drafted description for undescribed tools |
|
|
69
|
+
| `misroute check` | Checks whether one proposed description beats the current one on held out requests |
|
|
70
|
+
| `misroute report` | Writes a self-contained HTML report |
|
|
71
|
+
| `misroute ui` | Opens a local web interface for results, runs and held writes |
|
|
72
|
+
| `misroute ci` | Runs discover and exits with an error when limits are exceeded |
|
|
73
|
+
| `misroute guard serve` | Runs an MCP proxy that logs and guards every real tool call |
|
|
74
|
+
| `misroute site` | Builds a static results site from one or more runs |
|
|
75
|
+
| `misroute serve-mcp` | Runs misroute itself as an MCP server for agents |
|
|
76
|
+
| `misroute catalog` | Prints tool hashes and descriptions for change-aware CI |
|
|
77
|
+
|
|
78
|
+
Every command takes `--json`, and paid commands take `--yes`, `--max-cost` and `--estimate-only`. Exit codes are `0` passed, `1` limits failed or worse, `2` not enough data to decide, `3` budget or config problem.
|
|
79
|
+
|
|
80
|
+
## Status
|
|
81
|
+
|
|
82
|
+
| Feature | Status |
|
|
83
|
+
|---|---|
|
|
84
|
+
| lint, discover, probe and fix in JSON and code mode | tested on Opus 5.5 through OpenRouter and on a local model through Ollama |
|
|
85
|
+
| OpenAI compatible adapter | tested on OpenRouter and Ollama |
|
|
86
|
+
| Native Anthropic, Foundry and Bedrock adapters | not yet tested against real endpoints |
|
|
87
|
+
| Near misses | experimental, not yet validated, kept out of the ranking |
|
|
88
|
+
| Local UI | API and pages checked in a browser, not yet used on a real project |
|
|
89
|
+
| Guard proxy | tested against a local test server only |
|
|
90
|
+
| check, ci with changed tools, serve-mcp | tested with a mock model only |
|
|
91
|
+
| Multi turn mode | tested with a mock model only |
|
|
92
|
+
|
|
93
|
+
## Features
|
|
94
|
+
|
|
95
|
+
### Discover
|
|
96
|
+
Each tool gets requests that describe what the user wants without using the tool's own words. A second model keeps a request only if it picks the right tool, and a permutation gate checks that the request sets can't be told apart by length or word overlap alone. Requests run against the full tool list in rotated orders, because models pick the wrong tool more often when the distractor comes first.
|
|
97
|
+
|
|
98
|
+
Discover combines four signals:
|
|
99
|
+
|
|
100
|
+
- **Confusion matrix:** where each tool's requests land, including tools with unrelated names
|
|
101
|
+
- **Co-calls:** tools called together in one turn or one program
|
|
102
|
+
- **Removal test:** remove a tool, rerun its requests and see where they go
|
|
103
|
+
- **Near misses:** the probability spread across tools for local models through vLLM, and disagreement across repeated runs for hosted models
|
|
104
|
+
|
|
105
|
+
You can add real questions from your logs with `real_requests: logs.jsonl`. misroute reports them separately from the generated ones.
|
|
106
|
+
|
|
107
|
+
### Code mode
|
|
108
|
+
|
|
109
|
+
In code mode the model gets a single `run_code` tool, with your tools available as async Python functions. misroute runs each program in Monty against recorded fakes and judges it by the calls it actually made. It catches:
|
|
110
|
+
|
|
111
|
+
- faked tools that return invented data
|
|
112
|
+
- pasted stubs that shadow the real function
|
|
113
|
+
- programs that never call anything
|
|
114
|
+
- invented or reused pagination cursors, and pages left unread
|
|
115
|
+
- unknown functions, syntax errors and runtime errors
|
|
116
|
+
- answers with no program at all
|
|
117
|
+
|
|
118
|
+
### Fix
|
|
119
|
+
|
|
120
|
+
For each risky pair, misroute writes a boundary clause: one sentence per tool saying what it covers and when the other tool is the better choice. For best results, use a model from a different family to write it. Each pair then runs three ways on the same requests: the original description, a baseline, and the baseline with the clause. The baseline is a neutral edit of the same length. When lint flags a tool as undescribed, the baseline is lint's drafted description instead, since writing any description already adds information. That tells you whether the clause adds anything beyond a basic description.
|
|
121
|
+
|
|
122
|
+
- Per-pair verdicts use an exact McNemar test with Holm correction.
|
|
123
|
+
- Across pairs, misroute uses an exact sign-flip test.
|
|
124
|
+
- Errors are reported in both directions, along with any leakage to other tools.
|
|
125
|
+
- When the clause does no better than the baseline, misroute tells you and suggests a guard rule or merging the tools.
|
|
126
|
+
|
|
127
|
+
`misroute fix --cheap` runs fewer trials and gives indicative numbers.
|
|
128
|
+
|
|
129
|
+
### Guard
|
|
130
|
+
|
|
131
|
+
Put misroute in front of your MCP server to watch real traffic:
|
|
132
|
+
|
|
133
|
+
```console
|
|
134
|
+
$ misroute guard serve -c misroute.yaml
|
|
135
|
+
$ misroute guard confirm 8ba64a3fc925
|
|
136
|
+
approved 8ba64a3fc925
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
Or wrap a client session in Python:
|
|
140
|
+
|
|
141
|
+
```python
|
|
142
|
+
from misroute.guard.wrapper import wrap
|
|
143
|
+
|
|
144
|
+
guard = wrap(session, rules, ".misroute/guard.jsonl")
|
|
145
|
+
await guard.call_tool("update_case", {"case_id": "7", "status": "closed"})
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
Guard logs every call, including calls made inside code-mode programs. It flags hedging, quick switches between sibling tools and faked tools, and it holds a second write to a sibling tool until a person approves it. To send logs to OpenTelemetry, set `guard: {otel: true}`. Spotting users who rephrase their question needs the user's messages, so that only works through the Python wrapper.
|
|
149
|
+
|
|
150
|
+
### CI
|
|
151
|
+
|
|
152
|
+
```yaml
|
|
153
|
+
limits:
|
|
154
|
+
max_wrong_rate: 0.05
|
|
155
|
+
max_write_hedge_rate: 0.0
|
|
156
|
+
max_code_failure_rate: 0.10
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
```console
|
|
160
|
+
$ misroute ci
|
|
161
|
+
opus json wrong rate 0.105 above 0.05
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
To test only the tools a pull request changed, save the catalog from the base branch and compare:
|
|
165
|
+
|
|
166
|
+
```yaml
|
|
167
|
+
- run: git checkout origin/main && misroute catalog > tools.json && git checkout -
|
|
168
|
+
- run: misroute ci --changed --baseline tools.json --markdown comment.md --max-cost 2
|
|
169
|
+
- run: gh pr comment ${{ github.event.number }} --body-file comment.md
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
A cheap local model catches regressions in its own behaviour, which may differ from your production model's. Schedule regular runs against the model you actually deploy as well.
|
|
173
|
+
|
|
174
|
+
## Configuration
|
|
175
|
+
|
|
176
|
+
```yaml
|
|
177
|
+
server:
|
|
178
|
+
url: https://mcp.example.com/mcp # or command: [npx, your-server]
|
|
179
|
+
auth: bearer_env:MCP_TOKEN
|
|
180
|
+
include_tools: ["*issue*", "*pull_request*"] # optional, test a slice of a large server
|
|
181
|
+
|
|
182
|
+
models:
|
|
183
|
+
- name: opus-eu
|
|
184
|
+
provider: anthropic-foundry # anthropic, anthropic-foundry, bedrock, openai-compatible
|
|
185
|
+
model: claude-opus-5-5
|
|
186
|
+
price_in: 4
|
|
187
|
+
price_out: 20
|
|
188
|
+
- name: local
|
|
189
|
+
provider: openai-compatible
|
|
190
|
+
base_url: http://localhost:8000/v1
|
|
191
|
+
model: Qwen/Qwen3-14B
|
|
192
|
+
scoring: true # near misses from token probabilities, vLLM only
|
|
193
|
+
|
|
194
|
+
generator: local # writes the test requests
|
|
195
|
+
validator: opus-eu # checks them
|
|
196
|
+
writer: local # writes clauses and neutral edits
|
|
197
|
+
|
|
198
|
+
modes: [json, code]
|
|
199
|
+
max_turns: 3 # feed fake results back so a read before a write is followed to the write
|
|
200
|
+
budget_usd: 5.00
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
With `max_turns: 1`, the default, a write request answered only with reads is labelled read first and kept apart from wrong picks. With more turns misroute returns recorded fake results and judges the first write the model makes.
|
|
204
|
+
|
|
205
|
+
Results are saved in `.misroute/`: a DuckDB database you can query with SQL, JSONL transcripts of every run, and a response cache.
|
|
206
|
+
|
|
207
|
+
```console
|
|
208
|
+
$ duckdb .misroute/misroute.duckdb "select target, chosen, count(*) from trials where wrong group by all order by 3 desc"
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
## For agents
|
|
212
|
+
|
|
213
|
+
A coding agent that edits tool descriptions can check its own edits:
|
|
214
|
+
|
|
215
|
+
```console
|
|
216
|
+
$ misroute check --tool get_case_comments --description "Written comments on a case. Not for status changes." --json --max-cost 1
|
|
217
|
+
{"schema_version": 1, "command": "check", "verdict": "better | worse | no_difference | insufficient_data", "trials": <int>, "held_out": true, "cost_usd": <float>, ...}
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
Requests are split into a tuning set and a held out set. The verdict uses only the held out set, and after three verdicts on the same tool it is spent until you pass `--refresh`, so an agent cannot tune a description to the test requests. Verdicts are `better`, `worse`, `no_difference` or `insufficient_data`.
|
|
221
|
+
|
|
222
|
+
`misroute serve-mcp` exposes `lint_descriptions`, `draft_description`, `estimate_check`, `check_description` and `get_report` as MCP tools. [AGENTS.md](AGENTS.md), [llms.txt](llms.txt) and a [Claude Code skill](skills/misroute/SKILL.md) describe when to use each. misroute never edits your tools.
|
|
223
|
+
|
|
224
|
+
## Python API
|
|
225
|
+
|
|
226
|
+
```python
|
|
227
|
+
from misroute import Project
|
|
228
|
+
|
|
229
|
+
proj = Project.from_yaml("misroute.yaml")
|
|
230
|
+
found = await proj.discover(models=["local"])
|
|
231
|
+
result = await proj.fix(top=3, trials=96)
|
|
232
|
+
```
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "misroute"
|
|
3
|
+
version = "0.2.0"
|
|
4
|
+
description = "Find the tools your agent confuses and test whether a fix helps, against a neutral edit"
|
|
5
|
+
authors = [
|
|
6
|
+
{ name = "philippyt", email = "philipaha@proton.me" }
|
|
7
|
+
]
|
|
8
|
+
requires-python = ">=3.11"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
keywords = ["mcp", "agents", "llm", "tool-use", "evaluation"]
|
|
13
|
+
classifiers = [
|
|
14
|
+
"Development Status :: 3 - Alpha",
|
|
15
|
+
"Intended Audience :: Developers",
|
|
16
|
+
"Programming Language :: Python :: 3",
|
|
17
|
+
"Programming Language :: Python :: 3.11",
|
|
18
|
+
"Programming Language :: Python :: 3.12",
|
|
19
|
+
"Programming Language :: Python :: 3.13",
|
|
20
|
+
"Topic :: Software Development :: Testing",
|
|
21
|
+
]
|
|
22
|
+
dependencies = [
|
|
23
|
+
"anthropic>=2.3.0",
|
|
24
|
+
"duckdb>=2.0.0",
|
|
25
|
+
"jinja2>=3.1.6",
|
|
26
|
+
"mcp>=2.3.0",
|
|
27
|
+
"numpy>=2.4.6",
|
|
28
|
+
"openai>=3.41.0",
|
|
29
|
+
"pydantic>=2.14.2",
|
|
30
|
+
"pydantic-monty>=1.4.0",
|
|
31
|
+
"pyyaml>=6.0.3",
|
|
32
|
+
"rich>=15.0.0",
|
|
33
|
+
"scipy>=1.17.1",
|
|
34
|
+
"typer>=0.27.3",
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
[project.optional-dependencies]
|
|
38
|
+
otel = [
|
|
39
|
+
"opentelemetry-sdk>=1.45.2",
|
|
40
|
+
]
|
|
41
|
+
|
|
42
|
+
[project.scripts]
|
|
43
|
+
misroute = "misroute.cli:app"
|
|
44
|
+
|
|
45
|
+
[build-system]
|
|
46
|
+
requires = ["uv_build>=0.10.3,<0.11.0"]
|
|
47
|
+
build-backend = "uv_build"
|
|
48
|
+
|
|
49
|
+
[dependency-groups]
|
|
50
|
+
dev = [
|
|
51
|
+
"pytest>=9.2.0",
|
|
52
|
+
"pytest-asyncio>=1.4.1",
|
|
53
|
+
"ruff>=0.17.3",
|
|
54
|
+
"ty>=0.0.95",
|
|
55
|
+
]
|
|
56
|
+
|
|
57
|
+
[tool.ruff]
|
|
58
|
+
line-length = 120
|
|
59
|
+
target-version = "py311"
|
|
60
|
+
|
|
61
|
+
[tool.ruff.lint]
|
|
62
|
+
select = ["E", "F", "I", "B", "UP", "SIM", "RUF"]
|
|
63
|
+
ignore = ["RUF001", "RUF002", "RUF003", "SIM905"]
|
|
64
|
+
|
|
65
|
+
[tool.ruff.lint.isort]
|
|
66
|
+
no-lines-before = ["future", "standard-library", "third-party", "first-party", "local-folder"]
|
|
67
|
+
|
|
68
|
+
[tool.pytest.ini_options]
|
|
69
|
+
asyncio_mode = "auto"
|
|
70
|
+
asyncio_default_fixture_loop_scope = "function"
|
|
71
|
+
markers = ["live: needs a running model endpoint"]
|
|
72
|
+
addopts = "-m 'not live'"
|
|
73
|
+
|
|
74
|
+
[tool.ty.environment]
|
|
75
|
+
python-version = "3.11"
|