toolbloat 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- toolbloat/__init__.py +3 -0
- toolbloat/__main__.py +3 -0
- toolbloat/cli.py +439 -0
- toolbloat/config.py +175 -0
- toolbloat/data/demo/results.json +3677 -0
- toolbloat/discover.py +100 -0
- toolbloat/errors.py +25 -0
- toolbloat/evaluate.py +245 -0
- toolbloat/intents.py +167 -0
- toolbloat/llm.py +261 -0
- toolbloat/metrics.py +197 -0
- toolbloat/py.typed +0 -0
- toolbloat/report.py +67 -0
- toolbloat/templates/report.html.j2 +376 -0
- toolbloat/terminal.py +132 -0
- toolbloat/tokens.py +55 -0
- toolbloat-0.2.0.dist-info/METADATA +296 -0
- toolbloat-0.2.0.dist-info/RECORD +22 -0
- toolbloat-0.2.0.dist-info/WHEEL +4 -0
- toolbloat-0.2.0.dist-info/entry_points.txt +2 -0
- toolbloat-0.2.0.dist-info/licenses/LICENSE +21 -0
- toolbloat-0.2.0.dist-info/licenses/NOTICE +232 -0
toolbloat/discover.py
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"""Start each MCP server over stdio and list its tools. Tools are never called."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import shutil
|
|
7
|
+
import tempfile
|
|
8
|
+
from dataclasses import dataclass, field
|
|
9
|
+
|
|
10
|
+
import anyio
|
|
11
|
+
|
|
12
|
+
from .config import ServerSpec
|
|
13
|
+
from .errors import ServerStartError
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass
|
|
17
|
+
class ToolDef:
|
|
18
|
+
server: str
|
|
19
|
+
name: str
|
|
20
|
+
description: str
|
|
21
|
+
input_schema: dict
|
|
22
|
+
tokens: int = 0
|
|
23
|
+
|
|
24
|
+
@property
|
|
25
|
+
def key(self) -> str:
|
|
26
|
+
return f"{self.server}/{self.name}"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass
|
|
30
|
+
class ServerResult:
|
|
31
|
+
name: str
|
|
32
|
+
command: str # display form, secrets redacted
|
|
33
|
+
tools: list[ToolDef] = field(default_factory=list)
|
|
34
|
+
error: str | None = None
|
|
35
|
+
server_version: str | None = None
|
|
36
|
+
source: str = "--server" # config file it came from; shown locally, never written to reports
|
|
37
|
+
|
|
38
|
+
@property
|
|
39
|
+
def tokens(self) -> int:
|
|
40
|
+
return sum(t.tokens for t in self.tools)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
async def _list_one(spec: ServerSpec, timeout: float) -> ServerResult:
|
|
44
|
+
from mcp import ClientSession
|
|
45
|
+
from mcp.client.stdio import StdioServerParameters, stdio_client
|
|
46
|
+
|
|
47
|
+
result = ServerResult(name=spec.name, command=spec.display(), source=spec.source)
|
|
48
|
+
if shutil.which(spec.command) is None and not os.path.isfile(spec.command):
|
|
49
|
+
result.error = f"command not found: {spec.command}"
|
|
50
|
+
return result
|
|
51
|
+
params = StdioServerParameters(command=spec.command, args=spec.args, env=spec.env or None, cwd=spec.cwd)
|
|
52
|
+
with tempfile.TemporaryFile("w+", encoding="utf-8") as errlog:
|
|
53
|
+
try:
|
|
54
|
+
with anyio.fail_after(timeout):
|
|
55
|
+
async with stdio_client(params, errlog=errlog) as (read, write):
|
|
56
|
+
async with ClientSession(read, write) as session:
|
|
57
|
+
init = await session.initialize()
|
|
58
|
+
info = getattr(init, "server_info", None)
|
|
59
|
+
if info is not None:
|
|
60
|
+
result.server_version = f"{info.name} {info.version}"
|
|
61
|
+
cursor = None
|
|
62
|
+
while True:
|
|
63
|
+
from mcp.types import PaginatedRequestParams
|
|
64
|
+
|
|
65
|
+
page = await session.list_tools(
|
|
66
|
+
params=PaginatedRequestParams(cursor=cursor) if cursor else None
|
|
67
|
+
)
|
|
68
|
+
for t in page.tools:
|
|
69
|
+
result.tools.append(
|
|
70
|
+
ToolDef(
|
|
71
|
+
server=spec.name,
|
|
72
|
+
name=t.name,
|
|
73
|
+
description=t.description or "",
|
|
74
|
+
input_schema=t.input_schema or {"type": "object", "properties": {}},
|
|
75
|
+
)
|
|
76
|
+
)
|
|
77
|
+
cursor = page.next_cursor
|
|
78
|
+
if not cursor:
|
|
79
|
+
break
|
|
80
|
+
except TimeoutError:
|
|
81
|
+
result.error = f"no answer to tools/list within {timeout:.0f}s"
|
|
82
|
+
except Exception as e: # server crashed, bad protocol, ...
|
|
83
|
+
errlog.seek(0)
|
|
84
|
+
tail = errlog.read()[-400:].strip()
|
|
85
|
+
result.error = f"{type(e).__name__}: {e}"[:300] + (f" | stderr: {tail}" if tail else "")
|
|
86
|
+
return result
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def list_tools(specs: list[ServerSpec], timeout: float = 90.0) -> list[ServerResult]:
|
|
90
|
+
"""List tools of every server, one after another (keeps memory use low)."""
|
|
91
|
+
|
|
92
|
+
async def run() -> list[ServerResult]:
|
|
93
|
+
return [await _list_one(s, timeout) for s in specs]
|
|
94
|
+
|
|
95
|
+
results = anyio.run(run)
|
|
96
|
+
if results and all(r.error for r in results):
|
|
97
|
+
raise ServerStartError(
|
|
98
|
+
"No MCP server could be started:\n" + "\n".join(f" - {r.name} ({r.command}): {r.error}" for r in results)
|
|
99
|
+
)
|
|
100
|
+
return results
|
toolbloat/errors.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""User-facing errors. The CLI prints these without a traceback."""
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class ToolbloatError(Exception):
|
|
5
|
+
"""An error with a message that tells the user what to do next."""
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class MissingApiKey(ToolbloatError):
|
|
9
|
+
pass
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class ProviderUnreachable(ToolbloatError):
|
|
13
|
+
pass
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class ModelNotFound(ToolbloatError):
|
|
17
|
+
pass
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class BudgetExceeded(ToolbloatError):
|
|
21
|
+
pass
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class ServerStartError(ToolbloatError):
|
|
25
|
+
pass
|
toolbloat/evaluate.py
ADDED
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
"""Ask each model which tool it would call, with one server and with the whole stack.
|
|
2
|
+
|
|
3
|
+
Tools are never executed: we only read the tool call the model proposes.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import hashlib
|
|
9
|
+
import json
|
|
10
|
+
import threading
|
|
11
|
+
import time
|
|
12
|
+
from collections.abc import Callable
|
|
13
|
+
from dataclasses import asdict, dataclass
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
from .discover import ToolDef
|
|
17
|
+
from .errors import BudgetExceeded, MissingApiKey, ModelNotFound, ProviderUnreachable, ToolbloatError
|
|
18
|
+
from .intents import Intent
|
|
19
|
+
from .llm import LLM
|
|
20
|
+
from .tokens import function_name, openai_tool
|
|
21
|
+
|
|
22
|
+
SYSTEM = (
|
|
23
|
+
"You are an assistant connected to tools. Handle the user's request by calling exactly"
|
|
24
|
+
" one tool, the one that fits best, with complete arguments. Do not answer in text."
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
CONDITIONS = ("solo", "stack")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass
|
|
31
|
+
class Call:
|
|
32
|
+
model: str
|
|
33
|
+
intent_id: str
|
|
34
|
+
condition: str # solo | stack
|
|
35
|
+
chosen: str | None # server/tool, "?:<name>" for an unknown name, None for no call
|
|
36
|
+
correct: bool
|
|
37
|
+
args_valid: bool | None
|
|
38
|
+
prompt_tokens: int
|
|
39
|
+
completion_tokens: int
|
|
40
|
+
cost: float
|
|
41
|
+
seconds: float
|
|
42
|
+
error: str | None = None
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def validate_args(arguments: str | None, schema: dict) -> bool:
|
|
46
|
+
import jsonschema
|
|
47
|
+
|
|
48
|
+
try:
|
|
49
|
+
args = json.loads(arguments or "{}")
|
|
50
|
+
except json.JSONDecodeError:
|
|
51
|
+
return False
|
|
52
|
+
if not isinstance(args, dict):
|
|
53
|
+
return False
|
|
54
|
+
try:
|
|
55
|
+
cls = jsonschema.validators.validator_for(schema)
|
|
56
|
+
cls(schema).validate(args)
|
|
57
|
+
except jsonschema.ValidationError:
|
|
58
|
+
return False
|
|
59
|
+
except (jsonschema.SchemaError, jsonschema.exceptions.UnknownType, RecursionError):
|
|
60
|
+
# The server ships an invalid schema; only check that we got an object.
|
|
61
|
+
return True
|
|
62
|
+
return True
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _key(model: str, intent: Intent, condition: str, stack_hash: str) -> str:
|
|
66
|
+
raw = json.dumps([model, intent.tool, intent.text, sorted(intent.also_ok), condition, stack_hash])
|
|
67
|
+
return hashlib.sha256(raw.encode()).hexdigest()[:20]
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def stack_hash(tools: list[ToolDef]) -> str:
|
|
71
|
+
raw = json.dumps([openai_tool(t) for t in tools], sort_keys=True)
|
|
72
|
+
return hashlib.sha256(raw.encode()).hexdigest()[:16]
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class Cache:
|
|
76
|
+
"""Append-only JSONL of finished calls, so reruns and interrupted runs reuse work."""
|
|
77
|
+
|
|
78
|
+
def __init__(self, path: Path | None):
|
|
79
|
+
self.path = path
|
|
80
|
+
self.data: dict[str, dict] = {}
|
|
81
|
+
self.lock = threading.Lock()
|
|
82
|
+
if path and path.exists():
|
|
83
|
+
for line in path.read_text(encoding="utf-8").splitlines():
|
|
84
|
+
try:
|
|
85
|
+
row = json.loads(line)
|
|
86
|
+
self.data[row["key"]] = row["call"]
|
|
87
|
+
except (json.JSONDecodeError, KeyError):
|
|
88
|
+
continue
|
|
89
|
+
|
|
90
|
+
def get(self, key: str) -> Call | None:
|
|
91
|
+
row = self.data.get(key)
|
|
92
|
+
return Call(**row) if row else None
|
|
93
|
+
|
|
94
|
+
def put(self, key: str, call: Call) -> None:
|
|
95
|
+
if call.error:
|
|
96
|
+
return
|
|
97
|
+
with self.lock:
|
|
98
|
+
self.data[key] = asdict(call)
|
|
99
|
+
if self.path:
|
|
100
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
101
|
+
with self.path.open("a", encoding="utf-8") as f:
|
|
102
|
+
f.write(json.dumps({"key": key, "call": asdict(call)}) + "\n")
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def run(
|
|
106
|
+
llms: list[LLM],
|
|
107
|
+
tools: list[ToolDef],
|
|
108
|
+
intents: list[Intent],
|
|
109
|
+
*,
|
|
110
|
+
cache: Cache,
|
|
111
|
+
on_call: Callable[[Call], None] | None = None,
|
|
112
|
+
cached_only: bool = False,
|
|
113
|
+
) -> tuple[list[Call], str | None]:
|
|
114
|
+
"""Returns the calls and, if the run stopped early, the reason."""
|
|
115
|
+
by_name = {function_name(t.server, t.name): t for t in tools}
|
|
116
|
+
by_key = {t.key: t for t in tools}
|
|
117
|
+
all_defs = [openai_tool(t) for t in tools]
|
|
118
|
+
all_tokens = sum(t.tokens for t in tools)
|
|
119
|
+
server_defs: dict[str, list[dict]] = {}
|
|
120
|
+
server_tokens: dict[str, int] = {}
|
|
121
|
+
for t in tools:
|
|
122
|
+
server_defs.setdefault(t.server, []).append(openai_tool(t))
|
|
123
|
+
server_tokens[t.server] = server_tokens.get(t.server, 0) + t.tokens
|
|
124
|
+
h = stack_hash(tools)
|
|
125
|
+
todo = _round_robin([i for i in intents if not i.skip and i.tool in by_key])
|
|
126
|
+
|
|
127
|
+
calls: list[Call] = []
|
|
128
|
+
lock = threading.Lock()
|
|
129
|
+
stop: dict[str, str | None] = {"reason": None}
|
|
130
|
+
problems: list[str] = []
|
|
131
|
+
|
|
132
|
+
def one(llm: LLM, intent: Intent, condition: str) -> Call | None:
|
|
133
|
+
key = _key(llm.spec.id, intent, condition, h)
|
|
134
|
+
cached = cache.get(key)
|
|
135
|
+
if cached or cached_only:
|
|
136
|
+
return cached
|
|
137
|
+
defs = server_defs[intent.server] if condition == "solo" else all_defs
|
|
138
|
+
est = (server_tokens[intent.server] if condition == "solo" else all_tokens) + 200
|
|
139
|
+
system = SYSTEM
|
|
140
|
+
t0 = time.monotonic()
|
|
141
|
+
try:
|
|
142
|
+
out = llm.chat(
|
|
143
|
+
[{"role": "system", "content": system}, {"role": "user", "content": intent.text}],
|
|
144
|
+
tools=defs,
|
|
145
|
+
max_tokens=2048,
|
|
146
|
+
est_prompt_tokens=est,
|
|
147
|
+
)
|
|
148
|
+
except (BudgetExceeded, MissingApiKey, ModelNotFound, ProviderUnreachable):
|
|
149
|
+
raise
|
|
150
|
+
except ToolbloatError as e:
|
|
151
|
+
return Call(
|
|
152
|
+
llm.spec.id,
|
|
153
|
+
intent.id,
|
|
154
|
+
condition,
|
|
155
|
+
None,
|
|
156
|
+
False,
|
|
157
|
+
None,
|
|
158
|
+
0,
|
|
159
|
+
0,
|
|
160
|
+
0.0,
|
|
161
|
+
time.monotonic() - t0,
|
|
162
|
+
error=str(e)[:300],
|
|
163
|
+
)
|
|
164
|
+
chosen = None
|
|
165
|
+
args_valid = None
|
|
166
|
+
if out.tool_name:
|
|
167
|
+
t = by_name.get(out.tool_name)
|
|
168
|
+
if t is None:
|
|
169
|
+
# Some models drop the server prefix; map a unique bare name back.
|
|
170
|
+
bare = [x for x in tools if x.name == out.tool_name]
|
|
171
|
+
t = bare[0] if len(bare) == 1 else None
|
|
172
|
+
chosen = t.key if t else f"?:{out.tool_name}"
|
|
173
|
+
if t is not None:
|
|
174
|
+
args_valid = validate_args(out.arguments, t.input_schema)
|
|
175
|
+
call = Call(
|
|
176
|
+
model=llm.spec.id,
|
|
177
|
+
intent_id=intent.id,
|
|
178
|
+
condition=condition,
|
|
179
|
+
chosen=chosen,
|
|
180
|
+
correct=intent.accepts(chosen),
|
|
181
|
+
args_valid=args_valid,
|
|
182
|
+
prompt_tokens=out.prompt_tokens,
|
|
183
|
+
completion_tokens=out.completion_tokens,
|
|
184
|
+
cost=round(out.cost, 8),
|
|
185
|
+
seconds=round(time.monotonic() - t0, 2),
|
|
186
|
+
)
|
|
187
|
+
cache.put(key, call)
|
|
188
|
+
return call
|
|
189
|
+
|
|
190
|
+
def worker(group: list[LLM]) -> None:
|
|
191
|
+
for llm in group:
|
|
192
|
+
try:
|
|
193
|
+
for intent in todo:
|
|
194
|
+
for condition in CONDITIONS:
|
|
195
|
+
if stop["reason"]:
|
|
196
|
+
return
|
|
197
|
+
c = one(llm, intent, condition)
|
|
198
|
+
if c is None:
|
|
199
|
+
continue
|
|
200
|
+
with lock:
|
|
201
|
+
calls.append(c)
|
|
202
|
+
if on_call:
|
|
203
|
+
on_call(c)
|
|
204
|
+
except BudgetExceeded as e:
|
|
205
|
+
stop["reason"] = str(e)
|
|
206
|
+
return
|
|
207
|
+
except ToolbloatError as e:
|
|
208
|
+
# This model is unusable; the others keep going.
|
|
209
|
+
with lock:
|
|
210
|
+
problems.append(f"{llm.spec.id}: {e}")
|
|
211
|
+
|
|
212
|
+
# One thread per provider, models of a provider run one after another
|
|
213
|
+
# (low concurrency keeps rate limits and local RAM in check).
|
|
214
|
+
groups: dict[str, list[LLM]] = {}
|
|
215
|
+
for llm in llms:
|
|
216
|
+
groups.setdefault(llm.spec.provider, []).append(llm)
|
|
217
|
+
threads = [threading.Thread(target=worker, args=(g,), daemon=True) for g in groups.values()]
|
|
218
|
+
for t in threads:
|
|
219
|
+
t.start()
|
|
220
|
+
try:
|
|
221
|
+
for t in threads:
|
|
222
|
+
while t.is_alive():
|
|
223
|
+
t.join(0.5)
|
|
224
|
+
except KeyboardInterrupt:
|
|
225
|
+
stop["reason"] = "interrupted by the user (Ctrl-C); partial results kept"
|
|
226
|
+
reasons = [r for r in [stop["reason"], *problems] if r]
|
|
227
|
+
return calls, "; ".join(reasons) or None
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def _round_robin(items: list[Intent]) -> list[Intent]:
|
|
231
|
+
"""Interleave servers so a run stopped early (budget, Ctrl-C) still covers every server."""
|
|
232
|
+
queues: dict[str, list[Intent]] = {}
|
|
233
|
+
for i in items:
|
|
234
|
+
queues.setdefault(i.server, []).append(i)
|
|
235
|
+
out: list[Intent] = []
|
|
236
|
+
while any(queues.values()):
|
|
237
|
+
for q in queues.values():
|
|
238
|
+
if q:
|
|
239
|
+
out.append(q.pop(0))
|
|
240
|
+
return out
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def total_calls(llms: list[LLM], tools: list[ToolDef], intents: list[Intent]) -> int:
|
|
244
|
+
keys = {t.key for t in tools}
|
|
245
|
+
return len(llms) * len(CONDITIONS) * sum(1 for i in intents if not i.skip and i.tool in keys)
|
toolbloat/intents.py
ADDED
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
"""User requests ("intents") that should map to one tool. LLM-generated, cached as YAML, editable."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import re
|
|
7
|
+
from dataclasses import asdict, dataclass, field
|
|
8
|
+
from datetime import UTC, datetime
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
import yaml
|
|
12
|
+
|
|
13
|
+
from .discover import ToolDef
|
|
14
|
+
from .errors import ToolbloatError
|
|
15
|
+
from .llm import LLM
|
|
16
|
+
|
|
17
|
+
HEADER = """\
|
|
18
|
+
# toolbloat intents: realistic user requests and the tool that should answer each.
|
|
19
|
+
# Edit freely. `tool` is server/tool. `also_ok` lists other acceptable tools.
|
|
20
|
+
# Delete an entry to have it regenerated on the next run; set `skip: true` to ignore it.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
PROMPT = """You write test prompts for an AI assistant that is connected to tools.
|
|
24
|
+
|
|
25
|
+
Tool under test: {name}
|
|
26
|
+
Description: {description}
|
|
27
|
+
Input schema: {schema}
|
|
28
|
+
|
|
29
|
+
Other tools on the same server: {siblings}
|
|
30
|
+
|
|
31
|
+
Write {n} different, realistic requests a user might type where THIS tool is clearly
|
|
32
|
+
the right one to call (not one of the other tools). Rules:
|
|
33
|
+
- Natural language, one or two sentences, as a real user would write.
|
|
34
|
+
- Do not mention the tool name or the word "tool".
|
|
35
|
+
- Include the concrete details the tool's required arguments need (paths, names, values).
|
|
36
|
+
- Vary the wording.
|
|
37
|
+
|
|
38
|
+
Answer with JSON only, a list of plain strings: {{"requests": ["...", "..."]}}"""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass
|
|
42
|
+
class Intent:
|
|
43
|
+
id: str
|
|
44
|
+
tool: str # server/tool
|
|
45
|
+
text: str
|
|
46
|
+
also_ok: list[str] = field(default_factory=list)
|
|
47
|
+
skip: bool = False
|
|
48
|
+
|
|
49
|
+
@property
|
|
50
|
+
def server(self) -> str:
|
|
51
|
+
return self.tool.split("/", 1)[0]
|
|
52
|
+
|
|
53
|
+
def accepts(self, key: str | None) -> bool:
|
|
54
|
+
return key is not None and (key == self.tool or key in self.also_ok)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def load(path: Path) -> tuple[list[Intent], dict]:
|
|
58
|
+
if not path.exists():
|
|
59
|
+
return [], {}
|
|
60
|
+
try:
|
|
61
|
+
data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
|
|
62
|
+
except yaml.YAMLError as e:
|
|
63
|
+
raise ToolbloatError(f"{path} is not valid YAML: {e}") from e
|
|
64
|
+
items = []
|
|
65
|
+
for raw in data.get("intents") or []:
|
|
66
|
+
if not isinstance(raw, dict) or "tool" not in raw or "text" not in raw:
|
|
67
|
+
raise ToolbloatError(f"{path}: every intent needs `tool` and `text` (got {raw!r})")
|
|
68
|
+
items.append(
|
|
69
|
+
Intent(
|
|
70
|
+
id=str(raw.get("id") or f"{raw['tool'].replace('/', '-')}-{len(items) + 1}"),
|
|
71
|
+
tool=str(raw["tool"]),
|
|
72
|
+
text=str(raw["text"]),
|
|
73
|
+
also_ok=[str(x) for x in raw.get("also_ok") or []],
|
|
74
|
+
skip=bool(raw.get("skip", False)),
|
|
75
|
+
)
|
|
76
|
+
)
|
|
77
|
+
meta = {k: v for k, v in data.items() if k != "intents"}
|
|
78
|
+
return items, meta
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def save(path: Path, intents: list[Intent], meta: dict) -> None:
|
|
82
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
83
|
+
body = dict(meta)
|
|
84
|
+
body["intents"] = [
|
|
85
|
+
{k: v for k, v in asdict(i).items() if not (k == "also_ok" and not v) and not (k == "skip" and not v)}
|
|
86
|
+
for i in intents
|
|
87
|
+
]
|
|
88
|
+
text = yaml.safe_dump(body, sort_keys=False, allow_unicode=True, width=100)
|
|
89
|
+
path.write_text(HEADER + text, encoding="utf-8")
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def parse_requests(text: str) -> list[str]:
|
|
93
|
+
text = re.sub(r"<think>.*?</think>", "", text, flags=re.S)
|
|
94
|
+
m = re.search(r"\{.*\}", text, flags=re.S)
|
|
95
|
+
if not m:
|
|
96
|
+
return []
|
|
97
|
+
try:
|
|
98
|
+
data = json.loads(m.group(0))
|
|
99
|
+
except json.JSONDecodeError:
|
|
100
|
+
return []
|
|
101
|
+
reqs = data.get("requests") if isinstance(data, dict) else None
|
|
102
|
+
if not isinstance(reqs, list):
|
|
103
|
+
return []
|
|
104
|
+
out = []
|
|
105
|
+
for r in reqs:
|
|
106
|
+
if isinstance(r, dict):
|
|
107
|
+
# Some models wrap each request in an object; keep its first string field.
|
|
108
|
+
r = next((v for v in r.values() if isinstance(v, str)), None)
|
|
109
|
+
if isinstance(r, str) and r.strip():
|
|
110
|
+
out.append(r.strip())
|
|
111
|
+
return out
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def generate_for_tool(llm: LLM, tool: ToolDef, siblings: list[str], n: int) -> list[str]:
|
|
115
|
+
prompt = PROMPT.format(
|
|
116
|
+
name=tool.name,
|
|
117
|
+
description=tool.description.strip()[:2000] or "(no description)",
|
|
118
|
+
schema=json.dumps(tool.input_schema, ensure_ascii=False)[:2000],
|
|
119
|
+
siblings=", ".join(siblings) or "(none)",
|
|
120
|
+
n=n,
|
|
121
|
+
)
|
|
122
|
+
for _ in range(2):
|
|
123
|
+
out = llm.chat(
|
|
124
|
+
[{"role": "user", "content": prompt}],
|
|
125
|
+
max_tokens=1500,
|
|
126
|
+
temperature=0.7,
|
|
127
|
+
est_prompt_tokens=len(prompt) // 3,
|
|
128
|
+
json_mode=llm.spec.provider != "ollama",
|
|
129
|
+
)
|
|
130
|
+
reqs = parse_requests(out.text)
|
|
131
|
+
if reqs:
|
|
132
|
+
return reqs[:n]
|
|
133
|
+
return []
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def ensure(
|
|
137
|
+
path: Path,
|
|
138
|
+
tools: list[ToolDef],
|
|
139
|
+
llm: LLM | None,
|
|
140
|
+
per_tool: int,
|
|
141
|
+
on_progress=None,
|
|
142
|
+
) -> list[Intent]:
|
|
143
|
+
"""Load cached intents and generate the missing ones."""
|
|
144
|
+
intents, meta = load(path)
|
|
145
|
+
have = {i.tool for i in intents}
|
|
146
|
+
missing = [t for t in tools if t.key not in have]
|
|
147
|
+
if missing and llm is None:
|
|
148
|
+
raise ToolbloatError(
|
|
149
|
+
f"{len(missing)} tool(s) have no intents in {path} and no model is available to write them."
|
|
150
|
+
)
|
|
151
|
+
by_server: dict[str, list[str]] = {}
|
|
152
|
+
for t in tools:
|
|
153
|
+
by_server.setdefault(t.server, []).append(t.name)
|
|
154
|
+
for n, t in enumerate(missing, 1):
|
|
155
|
+
if on_progress:
|
|
156
|
+
on_progress(n, len(missing), t.key)
|
|
157
|
+
reqs = generate_for_tool(llm, t, [s for s in by_server[t.server] if s != t.name], per_tool)
|
|
158
|
+
for k, r in enumerate(reqs, 1):
|
|
159
|
+
intents.append(Intent(id=f"{t.server}-{t.name}-{k}", tool=t.key, text=r))
|
|
160
|
+
# Save after every tool so an interrupted run keeps its work.
|
|
161
|
+
if llm is not None:
|
|
162
|
+
meta.setdefault("generated_by", llm.spec.id)
|
|
163
|
+
meta["generated_at"] = datetime.now(UTC).strftime("%Y-%m-%d")
|
|
164
|
+
save(path, intents, meta)
|
|
165
|
+
if not missing and not path.exists():
|
|
166
|
+
save(path, intents, meta)
|
|
167
|
+
return intents
|