toolbloat 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
toolbloat/discover.py ADDED
@@ -0,0 +1,100 @@
1
+ """Start each MCP server over stdio and list its tools. Tools are never called."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import os
6
+ import shutil
7
+ import tempfile
8
+ from dataclasses import dataclass, field
9
+
10
+ import anyio
11
+
12
+ from .config import ServerSpec
13
+ from .errors import ServerStartError
14
+
15
+
16
+ @dataclass
17
+ class ToolDef:
18
+ server: str
19
+ name: str
20
+ description: str
21
+ input_schema: dict
22
+ tokens: int = 0
23
+
24
+ @property
25
+ def key(self) -> str:
26
+ return f"{self.server}/{self.name}"
27
+
28
+
29
+ @dataclass
30
+ class ServerResult:
31
+ name: str
32
+ command: str # display form, secrets redacted
33
+ tools: list[ToolDef] = field(default_factory=list)
34
+ error: str | None = None
35
+ server_version: str | None = None
36
+ source: str = "--server" # config file it came from; shown locally, never written to reports
37
+
38
+ @property
39
+ def tokens(self) -> int:
40
+ return sum(t.tokens for t in self.tools)
41
+
42
+
43
+ async def _list_one(spec: ServerSpec, timeout: float) -> ServerResult:
44
+ from mcp import ClientSession
45
+ from mcp.client.stdio import StdioServerParameters, stdio_client
46
+
47
+ result = ServerResult(name=spec.name, command=spec.display(), source=spec.source)
48
+ if shutil.which(spec.command) is None and not os.path.isfile(spec.command):
49
+ result.error = f"command not found: {spec.command}"
50
+ return result
51
+ params = StdioServerParameters(command=spec.command, args=spec.args, env=spec.env or None, cwd=spec.cwd)
52
+ with tempfile.TemporaryFile("w+", encoding="utf-8") as errlog:
53
+ try:
54
+ with anyio.fail_after(timeout):
55
+ async with stdio_client(params, errlog=errlog) as (read, write):
56
+ async with ClientSession(read, write) as session:
57
+ init = await session.initialize()
58
+ info = getattr(init, "server_info", None)
59
+ if info is not None:
60
+ result.server_version = f"{info.name} {info.version}"
61
+ cursor = None
62
+ while True:
63
+ from mcp.types import PaginatedRequestParams
64
+
65
+ page = await session.list_tools(
66
+ params=PaginatedRequestParams(cursor=cursor) if cursor else None
67
+ )
68
+ for t in page.tools:
69
+ result.tools.append(
70
+ ToolDef(
71
+ server=spec.name,
72
+ name=t.name,
73
+ description=t.description or "",
74
+ input_schema=t.input_schema or {"type": "object", "properties": {}},
75
+ )
76
+ )
77
+ cursor = page.next_cursor
78
+ if not cursor:
79
+ break
80
+ except TimeoutError:
81
+ result.error = f"no answer to tools/list within {timeout:.0f}s"
82
+ except Exception as e: # server crashed, bad protocol, ...
83
+ errlog.seek(0)
84
+ tail = errlog.read()[-400:].strip()
85
+ result.error = f"{type(e).__name__}: {e}"[:300] + (f" | stderr: {tail}" if tail else "")
86
+ return result
87
+
88
+
89
+ def list_tools(specs: list[ServerSpec], timeout: float = 90.0) -> list[ServerResult]:
90
+ """List tools of every server, one after another (keeps memory use low)."""
91
+
92
+ async def run() -> list[ServerResult]:
93
+ return [await _list_one(s, timeout) for s in specs]
94
+
95
+ results = anyio.run(run)
96
+ if results and all(r.error for r in results):
97
+ raise ServerStartError(
98
+ "No MCP server could be started:\n" + "\n".join(f" - {r.name} ({r.command}): {r.error}" for r in results)
99
+ )
100
+ return results
toolbloat/errors.py ADDED
@@ -0,0 +1,25 @@
1
+ """User-facing errors. The CLI prints these without a traceback."""
2
+
3
+
4
+ class ToolbloatError(Exception):
5
+ """An error with a message that tells the user what to do next."""
6
+
7
+
8
+ class MissingApiKey(ToolbloatError):
9
+ pass
10
+
11
+
12
+ class ProviderUnreachable(ToolbloatError):
13
+ pass
14
+
15
+
16
+ class ModelNotFound(ToolbloatError):
17
+ pass
18
+
19
+
20
+ class BudgetExceeded(ToolbloatError):
21
+ pass
22
+
23
+
24
+ class ServerStartError(ToolbloatError):
25
+ pass
toolbloat/evaluate.py ADDED
@@ -0,0 +1,245 @@
1
+ """Ask each model which tool it would call, with one server and with the whole stack.
2
+
3
+ Tools are never executed: we only read the tool call the model proposes.
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ import hashlib
9
+ import json
10
+ import threading
11
+ import time
12
+ from collections.abc import Callable
13
+ from dataclasses import asdict, dataclass
14
+ from pathlib import Path
15
+
16
+ from .discover import ToolDef
17
+ from .errors import BudgetExceeded, MissingApiKey, ModelNotFound, ProviderUnreachable, ToolbloatError
18
+ from .intents import Intent
19
+ from .llm import LLM
20
+ from .tokens import function_name, openai_tool
21
+
22
+ SYSTEM = (
23
+ "You are an assistant connected to tools. Handle the user's request by calling exactly"
24
+ " one tool, the one that fits best, with complete arguments. Do not answer in text."
25
+ )
26
+
27
+ CONDITIONS = ("solo", "stack")
28
+
29
+
30
+ @dataclass
31
+ class Call:
32
+ model: str
33
+ intent_id: str
34
+ condition: str # solo | stack
35
+ chosen: str | None # server/tool, "?:<name>" for an unknown name, None for no call
36
+ correct: bool
37
+ args_valid: bool | None
38
+ prompt_tokens: int
39
+ completion_tokens: int
40
+ cost: float
41
+ seconds: float
42
+ error: str | None = None
43
+
44
+
45
+ def validate_args(arguments: str | None, schema: dict) -> bool:
46
+ import jsonschema
47
+
48
+ try:
49
+ args = json.loads(arguments or "{}")
50
+ except json.JSONDecodeError:
51
+ return False
52
+ if not isinstance(args, dict):
53
+ return False
54
+ try:
55
+ cls = jsonschema.validators.validator_for(schema)
56
+ cls(schema).validate(args)
57
+ except jsonschema.ValidationError:
58
+ return False
59
+ except (jsonschema.SchemaError, jsonschema.exceptions.UnknownType, RecursionError):
60
+ # The server ships an invalid schema; only check that we got an object.
61
+ return True
62
+ return True
63
+
64
+
65
+ def _key(model: str, intent: Intent, condition: str, stack_hash: str) -> str:
66
+ raw = json.dumps([model, intent.tool, intent.text, sorted(intent.also_ok), condition, stack_hash])
67
+ return hashlib.sha256(raw.encode()).hexdigest()[:20]
68
+
69
+
70
+ def stack_hash(tools: list[ToolDef]) -> str:
71
+ raw = json.dumps([openai_tool(t) for t in tools], sort_keys=True)
72
+ return hashlib.sha256(raw.encode()).hexdigest()[:16]
73
+
74
+
75
+ class Cache:
76
+ """Append-only JSONL of finished calls, so reruns and interrupted runs reuse work."""
77
+
78
+ def __init__(self, path: Path | None):
79
+ self.path = path
80
+ self.data: dict[str, dict] = {}
81
+ self.lock = threading.Lock()
82
+ if path and path.exists():
83
+ for line in path.read_text(encoding="utf-8").splitlines():
84
+ try:
85
+ row = json.loads(line)
86
+ self.data[row["key"]] = row["call"]
87
+ except (json.JSONDecodeError, KeyError):
88
+ continue
89
+
90
+ def get(self, key: str) -> Call | None:
91
+ row = self.data.get(key)
92
+ return Call(**row) if row else None
93
+
94
+ def put(self, key: str, call: Call) -> None:
95
+ if call.error:
96
+ return
97
+ with self.lock:
98
+ self.data[key] = asdict(call)
99
+ if self.path:
100
+ self.path.parent.mkdir(parents=True, exist_ok=True)
101
+ with self.path.open("a", encoding="utf-8") as f:
102
+ f.write(json.dumps({"key": key, "call": asdict(call)}) + "\n")
103
+
104
+
105
+ def run(
106
+ llms: list[LLM],
107
+ tools: list[ToolDef],
108
+ intents: list[Intent],
109
+ *,
110
+ cache: Cache,
111
+ on_call: Callable[[Call], None] | None = None,
112
+ cached_only: bool = False,
113
+ ) -> tuple[list[Call], str | None]:
114
+ """Returns the calls and, if the run stopped early, the reason."""
115
+ by_name = {function_name(t.server, t.name): t for t in tools}
116
+ by_key = {t.key: t for t in tools}
117
+ all_defs = [openai_tool(t) for t in tools]
118
+ all_tokens = sum(t.tokens for t in tools)
119
+ server_defs: dict[str, list[dict]] = {}
120
+ server_tokens: dict[str, int] = {}
121
+ for t in tools:
122
+ server_defs.setdefault(t.server, []).append(openai_tool(t))
123
+ server_tokens[t.server] = server_tokens.get(t.server, 0) + t.tokens
124
+ h = stack_hash(tools)
125
+ todo = _round_robin([i for i in intents if not i.skip and i.tool in by_key])
126
+
127
+ calls: list[Call] = []
128
+ lock = threading.Lock()
129
+ stop: dict[str, str | None] = {"reason": None}
130
+ problems: list[str] = []
131
+
132
+ def one(llm: LLM, intent: Intent, condition: str) -> Call | None:
133
+ key = _key(llm.spec.id, intent, condition, h)
134
+ cached = cache.get(key)
135
+ if cached or cached_only:
136
+ return cached
137
+ defs = server_defs[intent.server] if condition == "solo" else all_defs
138
+ est = (server_tokens[intent.server] if condition == "solo" else all_tokens) + 200
139
+ system = SYSTEM
140
+ t0 = time.monotonic()
141
+ try:
142
+ out = llm.chat(
143
+ [{"role": "system", "content": system}, {"role": "user", "content": intent.text}],
144
+ tools=defs,
145
+ max_tokens=2048,
146
+ est_prompt_tokens=est,
147
+ )
148
+ except (BudgetExceeded, MissingApiKey, ModelNotFound, ProviderUnreachable):
149
+ raise
150
+ except ToolbloatError as e:
151
+ return Call(
152
+ llm.spec.id,
153
+ intent.id,
154
+ condition,
155
+ None,
156
+ False,
157
+ None,
158
+ 0,
159
+ 0,
160
+ 0.0,
161
+ time.monotonic() - t0,
162
+ error=str(e)[:300],
163
+ )
164
+ chosen = None
165
+ args_valid = None
166
+ if out.tool_name:
167
+ t = by_name.get(out.tool_name)
168
+ if t is None:
169
+ # Some models drop the server prefix; map a unique bare name back.
170
+ bare = [x for x in tools if x.name == out.tool_name]
171
+ t = bare[0] if len(bare) == 1 else None
172
+ chosen = t.key if t else f"?:{out.tool_name}"
173
+ if t is not None:
174
+ args_valid = validate_args(out.arguments, t.input_schema)
175
+ call = Call(
176
+ model=llm.spec.id,
177
+ intent_id=intent.id,
178
+ condition=condition,
179
+ chosen=chosen,
180
+ correct=intent.accepts(chosen),
181
+ args_valid=args_valid,
182
+ prompt_tokens=out.prompt_tokens,
183
+ completion_tokens=out.completion_tokens,
184
+ cost=round(out.cost, 8),
185
+ seconds=round(time.monotonic() - t0, 2),
186
+ )
187
+ cache.put(key, call)
188
+ return call
189
+
190
+ def worker(group: list[LLM]) -> None:
191
+ for llm in group:
192
+ try:
193
+ for intent in todo:
194
+ for condition in CONDITIONS:
195
+ if stop["reason"]:
196
+ return
197
+ c = one(llm, intent, condition)
198
+ if c is None:
199
+ continue
200
+ with lock:
201
+ calls.append(c)
202
+ if on_call:
203
+ on_call(c)
204
+ except BudgetExceeded as e:
205
+ stop["reason"] = str(e)
206
+ return
207
+ except ToolbloatError as e:
208
+ # This model is unusable; the others keep going.
209
+ with lock:
210
+ problems.append(f"{llm.spec.id}: {e}")
211
+
212
+ # One thread per provider, models of a provider run one after another
213
+ # (low concurrency keeps rate limits and local RAM in check).
214
+ groups: dict[str, list[LLM]] = {}
215
+ for llm in llms:
216
+ groups.setdefault(llm.spec.provider, []).append(llm)
217
+ threads = [threading.Thread(target=worker, args=(g,), daemon=True) for g in groups.values()]
218
+ for t in threads:
219
+ t.start()
220
+ try:
221
+ for t in threads:
222
+ while t.is_alive():
223
+ t.join(0.5)
224
+ except KeyboardInterrupt:
225
+ stop["reason"] = "interrupted by the user (Ctrl-C); partial results kept"
226
+ reasons = [r for r in [stop["reason"], *problems] if r]
227
+ return calls, "; ".join(reasons) or None
228
+
229
+
230
+ def _round_robin(items: list[Intent]) -> list[Intent]:
231
+ """Interleave servers so a run stopped early (budget, Ctrl-C) still covers every server."""
232
+ queues: dict[str, list[Intent]] = {}
233
+ for i in items:
234
+ queues.setdefault(i.server, []).append(i)
235
+ out: list[Intent] = []
236
+ while any(queues.values()):
237
+ for q in queues.values():
238
+ if q:
239
+ out.append(q.pop(0))
240
+ return out
241
+
242
+
243
+ def total_calls(llms: list[LLM], tools: list[ToolDef], intents: list[Intent]) -> int:
244
+ keys = {t.key for t in tools}
245
+ return len(llms) * len(CONDITIONS) * sum(1 for i in intents if not i.skip and i.tool in keys)
toolbloat/intents.py ADDED
@@ -0,0 +1,167 @@
1
+ """User requests ("intents") that should map to one tool. LLM-generated, cached as YAML, editable."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import re
7
+ from dataclasses import asdict, dataclass, field
8
+ from datetime import UTC, datetime
9
+ from pathlib import Path
10
+
11
+ import yaml
12
+
13
+ from .discover import ToolDef
14
+ from .errors import ToolbloatError
15
+ from .llm import LLM
16
+
17
+ HEADER = """\
18
+ # toolbloat intents: realistic user requests and the tool that should answer each.
19
+ # Edit freely. `tool` is server/tool. `also_ok` lists other acceptable tools.
20
+ # Delete an entry to have it regenerated on the next run; set `skip: true` to ignore it.
21
+ """
22
+
23
+ PROMPT = """You write test prompts for an AI assistant that is connected to tools.
24
+
25
+ Tool under test: {name}
26
+ Description: {description}
27
+ Input schema: {schema}
28
+
29
+ Other tools on the same server: {siblings}
30
+
31
+ Write {n} different, realistic requests a user might type where THIS tool is clearly
32
+ the right one to call (not one of the other tools). Rules:
33
+ - Natural language, one or two sentences, as a real user would write.
34
+ - Do not mention the tool name or the word "tool".
35
+ - Include the concrete details the tool's required arguments need (paths, names, values).
36
+ - Vary the wording.
37
+
38
+ Answer with JSON only, a list of plain strings: {{"requests": ["...", "..."]}}"""
39
+
40
+
41
+ @dataclass
42
+ class Intent:
43
+ id: str
44
+ tool: str # server/tool
45
+ text: str
46
+ also_ok: list[str] = field(default_factory=list)
47
+ skip: bool = False
48
+
49
+ @property
50
+ def server(self) -> str:
51
+ return self.tool.split("/", 1)[0]
52
+
53
+ def accepts(self, key: str | None) -> bool:
54
+ return key is not None and (key == self.tool or key in self.also_ok)
55
+
56
+
57
+ def load(path: Path) -> tuple[list[Intent], dict]:
58
+ if not path.exists():
59
+ return [], {}
60
+ try:
61
+ data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
62
+ except yaml.YAMLError as e:
63
+ raise ToolbloatError(f"{path} is not valid YAML: {e}") from e
64
+ items = []
65
+ for raw in data.get("intents") or []:
66
+ if not isinstance(raw, dict) or "tool" not in raw or "text" not in raw:
67
+ raise ToolbloatError(f"{path}: every intent needs `tool` and `text` (got {raw!r})")
68
+ items.append(
69
+ Intent(
70
+ id=str(raw.get("id") or f"{raw['tool'].replace('/', '-')}-{len(items) + 1}"),
71
+ tool=str(raw["tool"]),
72
+ text=str(raw["text"]),
73
+ also_ok=[str(x) for x in raw.get("also_ok") or []],
74
+ skip=bool(raw.get("skip", False)),
75
+ )
76
+ )
77
+ meta = {k: v for k, v in data.items() if k != "intents"}
78
+ return items, meta
79
+
80
+
81
+ def save(path: Path, intents: list[Intent], meta: dict) -> None:
82
+ path.parent.mkdir(parents=True, exist_ok=True)
83
+ body = dict(meta)
84
+ body["intents"] = [
85
+ {k: v for k, v in asdict(i).items() if not (k == "also_ok" and not v) and not (k == "skip" and not v)}
86
+ for i in intents
87
+ ]
88
+ text = yaml.safe_dump(body, sort_keys=False, allow_unicode=True, width=100)
89
+ path.write_text(HEADER + text, encoding="utf-8")
90
+
91
+
92
+ def parse_requests(text: str) -> list[str]:
93
+ text = re.sub(r"<think>.*?</think>", "", text, flags=re.S)
94
+ m = re.search(r"\{.*\}", text, flags=re.S)
95
+ if not m:
96
+ return []
97
+ try:
98
+ data = json.loads(m.group(0))
99
+ except json.JSONDecodeError:
100
+ return []
101
+ reqs = data.get("requests") if isinstance(data, dict) else None
102
+ if not isinstance(reqs, list):
103
+ return []
104
+ out = []
105
+ for r in reqs:
106
+ if isinstance(r, dict):
107
+ # Some models wrap each request in an object; keep its first string field.
108
+ r = next((v for v in r.values() if isinstance(v, str)), None)
109
+ if isinstance(r, str) and r.strip():
110
+ out.append(r.strip())
111
+ return out
112
+
113
+
114
+ def generate_for_tool(llm: LLM, tool: ToolDef, siblings: list[str], n: int) -> list[str]:
115
+ prompt = PROMPT.format(
116
+ name=tool.name,
117
+ description=tool.description.strip()[:2000] or "(no description)",
118
+ schema=json.dumps(tool.input_schema, ensure_ascii=False)[:2000],
119
+ siblings=", ".join(siblings) or "(none)",
120
+ n=n,
121
+ )
122
+ for _ in range(2):
123
+ out = llm.chat(
124
+ [{"role": "user", "content": prompt}],
125
+ max_tokens=1500,
126
+ temperature=0.7,
127
+ est_prompt_tokens=len(prompt) // 3,
128
+ json_mode=llm.spec.provider != "ollama",
129
+ )
130
+ reqs = parse_requests(out.text)
131
+ if reqs:
132
+ return reqs[:n]
133
+ return []
134
+
135
+
136
+ def ensure(
137
+ path: Path,
138
+ tools: list[ToolDef],
139
+ llm: LLM | None,
140
+ per_tool: int,
141
+ on_progress=None,
142
+ ) -> list[Intent]:
143
+ """Load cached intents and generate the missing ones."""
144
+ intents, meta = load(path)
145
+ have = {i.tool for i in intents}
146
+ missing = [t for t in tools if t.key not in have]
147
+ if missing and llm is None:
148
+ raise ToolbloatError(
149
+ f"{len(missing)} tool(s) have no intents in {path} and no model is available to write them."
150
+ )
151
+ by_server: dict[str, list[str]] = {}
152
+ for t in tools:
153
+ by_server.setdefault(t.server, []).append(t.name)
154
+ for n, t in enumerate(missing, 1):
155
+ if on_progress:
156
+ on_progress(n, len(missing), t.key)
157
+ reqs = generate_for_tool(llm, t, [s for s in by_server[t.server] if s != t.name], per_tool)
158
+ for k, r in enumerate(reqs, 1):
159
+ intents.append(Intent(id=f"{t.server}-{t.name}-{k}", tool=t.key, text=r))
160
+ # Save after every tool so an interrupted run keeps its work.
161
+ if llm is not None:
162
+ meta.setdefault("generated_by", llm.spec.id)
163
+ meta["generated_at"] = datetime.now(UTC).strftime("%Y-%m-%d")
164
+ save(path, intents, meta)
165
+ if not missing and not path.exists():
166
+ save(path, intents, meta)
167
+ return intents