hud-python 0.4.20__py3-none-any.whl → 0.4.22__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of hud-python might be problematic. Click here for more details.
- hud/__init__.py +7 -0
- hud/agents/base.py +42 -10
- hud/agents/claude.py +24 -14
- hud/agents/grounded_openai.py +280 -0
- hud/agents/tests/test_client.py +11 -27
- hud/agents/tests/test_grounded_openai_agent.py +155 -0
- hud/cli/__init__.py +50 -20
- hud/cli/build.py +3 -44
- hud/cli/eval.py +25 -6
- hud/cli/init.py +4 -4
- hud/cli/push.py +3 -1
- hud/cli/tests/test_push.py +6 -6
- hud/cli/utils/interactive.py +1 -1
- hud/clients/__init__.py +3 -2
- hud/clients/base.py +20 -9
- hud/clients/mcp_use.py +44 -22
- hud/datasets/task.py +6 -2
- hud/native/__init__.py +6 -0
- hud/native/comparator.py +546 -0
- hud/native/tests/__init__.py +1 -0
- hud/native/tests/test_comparator.py +539 -0
- hud/native/tests/test_native_init.py +79 -0
- hud/otel/instrumentation.py +0 -2
- hud/server/server.py +9 -2
- hud/settings.py +6 -0
- hud/shared/exceptions.py +204 -31
- hud/shared/hints.py +177 -0
- hud/shared/requests.py +15 -3
- hud/shared/tests/test_exceptions.py +385 -144
- hud/tools/__init__.py +2 -0
- hud/tools/executors/tests/test_base_executor.py +1 -1
- hud/tools/executors/xdo.py +1 -1
- hud/tools/grounding/__init__.py +13 -0
- hud/tools/grounding/config.py +54 -0
- hud/tools/grounding/grounded_tool.py +314 -0
- hud/tools/grounding/grounder.py +301 -0
- hud/tools/grounding/tests/__init__.py +1 -0
- hud/tools/grounding/tests/test_grounded_tool.py +196 -0
- hud/tools/submit.py +66 -0
- hud/tools/tests/test_playwright_tool.py +1 -1
- hud/tools/tests/test_tools_init.py +1 -1
- hud/tools/tests/test_utils.py +2 -2
- hud/types.py +33 -5
- hud/utils/agent_factories.py +86 -0
- hud/utils/design.py +57 -0
- hud/utils/mcp.py +6 -0
- hud/utils/pretty_errors.py +68 -0
- hud/utils/tests/test_version.py +1 -1
- hud/version.py +1 -1
- {hud_python-0.4.20.dist-info → hud_python-0.4.22.dist-info}/METADATA +2 -4
- {hud_python-0.4.20.dist-info → hud_python-0.4.22.dist-info}/RECORD +54 -37
- {hud_python-0.4.20.dist-info → hud_python-0.4.22.dist-info}/WHEEL +0 -0
- {hud_python-0.4.20.dist-info → hud_python-0.4.22.dist-info}/entry_points.txt +0 -0
- {hud_python-0.4.20.dist-info → hud_python-0.4.22.dist-info}/licenses/LICENSE +0 -0
hud/cli/__init__.py
CHANGED
|
@@ -43,6 +43,12 @@ app = typer.Typer(
|
|
|
43
43
|
|
|
44
44
|
console = Console()
|
|
45
45
|
|
|
46
|
+
# Standard support hint appended to error outputs
|
|
47
|
+
SUPPORT_HINT = (
|
|
48
|
+
"If this looks like an issue with the sdk, please make a github issue at "
|
|
49
|
+
"https://github.com/hud-evals/hud-python/issues"
|
|
50
|
+
)
|
|
51
|
+
|
|
46
52
|
|
|
47
53
|
# Capture IMAGE and any following Docker args as a single variadic argument list.
|
|
48
54
|
@app.command(context_settings={"allow_extra_args": True, "ignore_unknown_options": True})
|
|
@@ -818,6 +824,11 @@ def eval(
|
|
|
818
824
|
"--max-concurrent-per-worker",
|
|
819
825
|
help="Maximum concurrent tasks per worker in parallel mode",
|
|
820
826
|
),
|
|
827
|
+
verbose: bool = typer.Option(
|
|
828
|
+
False,
|
|
829
|
+
"--verbose",
|
|
830
|
+
help="Enable verbose output from the agent",
|
|
831
|
+
),
|
|
821
832
|
) -> None:
|
|
822
833
|
"""🚀 Run evaluation on datasets or individual tasks with agents."""
|
|
823
834
|
from hud.utils.design import HUDDesign
|
|
@@ -912,6 +923,7 @@ def eval(
|
|
|
912
923
|
parallel=parallel,
|
|
913
924
|
max_workers=max_workers,
|
|
914
925
|
max_concurrent_per_worker=max_concurrent_per_worker,
|
|
926
|
+
verbose=verbose,
|
|
915
927
|
)
|
|
916
928
|
|
|
917
929
|
|
|
@@ -950,27 +962,45 @@ def hf(
|
|
|
950
962
|
|
|
951
963
|
def main() -> None:
|
|
952
964
|
"""Main entry point for the CLI."""
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
965
|
+
try:
|
|
966
|
+
# Show header for main help
|
|
967
|
+
if len(sys.argv) == 1 or (len(sys.argv) == 2 and sys.argv[1] in ["--help", "-h"]):
|
|
968
|
+
console.print(
|
|
969
|
+
Panel.fit(
|
|
970
|
+
"[bold cyan]🚀 HUD CLI[/bold cyan]\nMCP Environment Analysis & Debugging",
|
|
971
|
+
border_style="cyan",
|
|
972
|
+
)
|
|
959
973
|
)
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
+
console.print("\n[yellow]Quick Start:[/yellow]")
|
|
975
|
+
console.print(
|
|
976
|
+
" 1. Create a new environment: [cyan]hud init my-env && cd my-env[/cyan]"
|
|
977
|
+
)
|
|
978
|
+
console.print(" 2. Develop with hot-reload: [cyan]hud dev --interactive[/cyan]")
|
|
979
|
+
console.print(" 3. Build for production: [cyan]hud build[/cyan]")
|
|
980
|
+
console.print(" 4. Share your environment: [cyan]hud push[/cyan]")
|
|
981
|
+
console.print(" 5. Get shared environments: [cyan]hud pull <org/name:tag>[/cyan]")
|
|
982
|
+
console.print(" 6. Run and test: [cyan]hud run <image>[/cyan]")
|
|
983
|
+
console.print("\n[yellow]RL Training:[/yellow]")
|
|
984
|
+
console.print(" 1. Generate config: [cyan]hud rl init my-env:latest[/cyan]")
|
|
985
|
+
console.print(
|
|
986
|
+
" 2. Create dataset: [cyan]hud hf tasks.json --name my-org/my-tasks[/cyan]"
|
|
987
|
+
)
|
|
988
|
+
console.print(" 3. Start training: [cyan]hud rl --model Qwen/Qwen2.5-3B[/cyan]\n")
|
|
989
|
+
|
|
990
|
+
app()
|
|
991
|
+
except typer.Exit as e:
|
|
992
|
+
# Append SDK support hint for non-zero exits
|
|
993
|
+
try:
|
|
994
|
+
exit_code = getattr(e, "exit_code", 0)
|
|
995
|
+
except Exception:
|
|
996
|
+
exit_code = 1
|
|
997
|
+
if exit_code != 0:
|
|
998
|
+
from hud.utils.design import design
|
|
999
|
+
|
|
1000
|
+
design.info(SUPPORT_HINT)
|
|
1001
|
+
raise
|
|
1002
|
+
except Exception:
|
|
1003
|
+
raise
|
|
974
1004
|
|
|
975
1005
|
|
|
976
1006
|
if __name__ == "__main__":
|
hud/cli/build.py
CHANGED
|
@@ -204,30 +204,10 @@ async def analyze_mcp_environment(
|
|
|
204
204
|
"success": True,
|
|
205
205
|
}
|
|
206
206
|
except Exception as e:
|
|
207
|
-
import
|
|
207
|
+
from hud.shared.exceptions import HudException
|
|
208
208
|
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
design.error(f"Failed to analyze environment: {error_msg}")
|
|
212
|
-
design.error(f"Traceback:\n{traceback.format_exc()}")
|
|
213
|
-
|
|
214
|
-
# Common issues
|
|
215
|
-
if "Connection reset" in error_msg or "EOF" in error_msg:
|
|
216
|
-
design.warning(
|
|
217
|
-
"The MCP server may have crashed on startup. Check your server.py for errors."
|
|
218
|
-
)
|
|
219
|
-
elif "timeout" in error_msg:
|
|
220
|
-
design.warning(
|
|
221
|
-
"The MCP server took too long to initialize. It might need more startup time."
|
|
222
|
-
)
|
|
223
|
-
|
|
224
|
-
return {
|
|
225
|
-
"initializeMs": 0,
|
|
226
|
-
"toolCount": 0,
|
|
227
|
-
"tools": [],
|
|
228
|
-
"success": False,
|
|
229
|
-
"error": error_msg,
|
|
230
|
-
}
|
|
209
|
+
# Convert to HudException for better error messages and hints
|
|
210
|
+
raise HudException from e
|
|
231
211
|
finally:
|
|
232
212
|
# Only shutdown if we successfully initialized
|
|
233
213
|
if initialized:
|
|
@@ -340,27 +320,6 @@ def build_environment(
|
|
|
340
320
|
finally:
|
|
341
321
|
loop.close()
|
|
342
322
|
|
|
343
|
-
if not analysis["success"]:
|
|
344
|
-
design.error("Failed to analyze MCP environment")
|
|
345
|
-
if "error" in analysis:
|
|
346
|
-
design.error(f"Error: {analysis['error']}")
|
|
347
|
-
|
|
348
|
-
# Provide helpful debugging tips
|
|
349
|
-
design.section_title("Debugging Tips")
|
|
350
|
-
design.info("1. Debug your environment build:")
|
|
351
|
-
design.command_example("hud debug . --build")
|
|
352
|
-
design.dim_info(" This will", "test MCP server connection and show detailed logs")
|
|
353
|
-
design.info("")
|
|
354
|
-
design.info("2. Check for common issues:")
|
|
355
|
-
design.info(" - Server crashes on startup")
|
|
356
|
-
design.info(" - Missing dependencies")
|
|
357
|
-
design.info(" - Syntax errors in server.py")
|
|
358
|
-
design.info("")
|
|
359
|
-
design.info("3. Run with verbose mode:")
|
|
360
|
-
design.command_example("hud build . --verbose")
|
|
361
|
-
|
|
362
|
-
raise typer.Exit(1)
|
|
363
|
-
|
|
364
323
|
design.success(f"Analyzed environment: {analysis['toolCount']} tools found")
|
|
365
324
|
|
|
366
325
|
# Extract environment variables from Dockerfile
|
hud/cli/eval.py
CHANGED
|
@@ -22,6 +22,7 @@ def build_agent(
|
|
|
22
22
|
*,
|
|
23
23
|
model: str | None = None,
|
|
24
24
|
allowed_tools: list[str] | None = None,
|
|
25
|
+
verbose: bool = False,
|
|
25
26
|
) -> Any:
|
|
26
27
|
"""Create and return the requested agent type."""
|
|
27
28
|
|
|
@@ -39,9 +40,10 @@ def build_agent(
|
|
|
39
40
|
if allowed_tools:
|
|
40
41
|
return OperatorAgent(
|
|
41
42
|
allowed_tools=allowed_tools,
|
|
43
|
+
verbose=verbose,
|
|
42
44
|
)
|
|
43
45
|
else:
|
|
44
|
-
return OperatorAgent()
|
|
46
|
+
return OperatorAgent(verbose=verbose)
|
|
45
47
|
|
|
46
48
|
# Fallback Claude agent (Anthropic)
|
|
47
49
|
try:
|
|
@@ -59,10 +61,12 @@ def build_agent(
|
|
|
59
61
|
return ClaudeAgent(
|
|
60
62
|
model=model,
|
|
61
63
|
allowed_tools=allowed_tools,
|
|
64
|
+
verbose=verbose,
|
|
62
65
|
)
|
|
63
66
|
else:
|
|
64
67
|
return ClaudeAgent(
|
|
65
68
|
model=model,
|
|
69
|
+
verbose=verbose,
|
|
66
70
|
)
|
|
67
71
|
|
|
68
72
|
|
|
@@ -73,6 +77,7 @@ async def run_single_task(
|
|
|
73
77
|
model: str | None = None,
|
|
74
78
|
allowed_tools: list[str] | None = None,
|
|
75
79
|
max_steps: int = 10,
|
|
80
|
+
verbose: bool = False,
|
|
76
81
|
) -> None:
|
|
77
82
|
"""Load one task and execute it, or detect if JSON contains a list and run as dataset."""
|
|
78
83
|
|
|
@@ -82,7 +87,7 @@ async def run_single_task(
|
|
|
82
87
|
except ImportError as e:
|
|
83
88
|
design.error(
|
|
84
89
|
"Dataset dependencies are not installed. "
|
|
85
|
-
"Please install with: pip install 'hud-python
|
|
90
|
+
"Please install with: pip install 'hud-python\u27e6agent\u27e7'"
|
|
86
91
|
)
|
|
87
92
|
raise typer.Exit(1) from e
|
|
88
93
|
|
|
@@ -106,11 +111,11 @@ async def run_single_task(
|
|
|
106
111
|
except ImportError as e:
|
|
107
112
|
design.error(
|
|
108
113
|
"OpenAI agent dependencies are not installed. "
|
|
109
|
-
"Please install with: pip install 'hud-python
|
|
114
|
+
"Please install with: pip install 'hud-python\u27e6agent\u27e7'"
|
|
110
115
|
)
|
|
111
116
|
raise typer.Exit(1) from e
|
|
112
117
|
|
|
113
|
-
agent_config: dict[str, Any] = {}
|
|
118
|
+
agent_config: dict[str, Any] = {"verbose": verbose}
|
|
114
119
|
if allowed_tools:
|
|
115
120
|
agent_config["allowed_tools"] = allowed_tools
|
|
116
121
|
|
|
@@ -128,6 +133,7 @@ async def run_single_task(
|
|
|
128
133
|
|
|
129
134
|
agent_config = {
|
|
130
135
|
"model": model or "claude-sonnet-4-20250514",
|
|
136
|
+
"verbose": verbose,
|
|
131
137
|
}
|
|
132
138
|
if allowed_tools:
|
|
133
139
|
agent_config["allowed_tools"] = allowed_tools
|
|
@@ -182,6 +188,7 @@ async def run_single_task(
|
|
|
182
188
|
agent_type,
|
|
183
189
|
model=model,
|
|
184
190
|
allowed_tools=allowed_tools,
|
|
191
|
+
verbose=verbose,
|
|
185
192
|
)
|
|
186
193
|
design.info(task.prompt)
|
|
187
194
|
result = await agent.run(task, max_steps=max_steps)
|
|
@@ -199,6 +206,7 @@ async def run_full_dataset(
|
|
|
199
206
|
parallel: bool = False,
|
|
200
207
|
max_workers: int | None = None,
|
|
201
208
|
max_concurrent_per_worker: int = 25,
|
|
209
|
+
verbose: bool = False,
|
|
202
210
|
) -> list[Any]:
|
|
203
211
|
"""Run evaluation across the entire dataset.
|
|
204
212
|
|
|
@@ -211,7 +219,7 @@ async def run_full_dataset(
|
|
|
211
219
|
except ImportError as e:
|
|
212
220
|
design.error(
|
|
213
221
|
"Dataset dependencies are not installed. "
|
|
214
|
-
"Please install with: pip install 'hud-python[agent]'"
|
|
222
|
+
"Please install with: pip install 'hud-python[[agent]]'"
|
|
215
223
|
)
|
|
216
224
|
raise typer.Exit(1) from e
|
|
217
225
|
|
|
@@ -245,7 +253,7 @@ async def run_full_dataset(
|
|
|
245
253
|
)
|
|
246
254
|
raise typer.Exit(1) from e
|
|
247
255
|
|
|
248
|
-
agent_config: dict[str, Any] = {}
|
|
256
|
+
agent_config: dict[str, Any] = {"verbose": verbose}
|
|
249
257
|
if allowed_tools:
|
|
250
258
|
agent_config["allowed_tools"] = allowed_tools
|
|
251
259
|
|
|
@@ -263,6 +271,7 @@ async def run_full_dataset(
|
|
|
263
271
|
|
|
264
272
|
agent_config = {
|
|
265
273
|
"model": model or "claude-sonnet-4-20250514",
|
|
274
|
+
"verbose": verbose,
|
|
266
275
|
}
|
|
267
276
|
if allowed_tools:
|
|
268
277
|
agent_config["allowed_tools"] = allowed_tools
|
|
@@ -360,6 +369,11 @@ def eval_command(
|
|
|
360
369
|
"--max-concurrent-per-worker",
|
|
361
370
|
help="Maximum concurrent tasks per worker in parallel mode",
|
|
362
371
|
),
|
|
372
|
+
verbose: bool = typer.Option(
|
|
373
|
+
False,
|
|
374
|
+
"--verbose",
|
|
375
|
+
help="Enable verbose output from the agent",
|
|
376
|
+
),
|
|
363
377
|
) -> None:
|
|
364
378
|
"""🚀 Run evaluation on datasets or individual tasks with agents.
|
|
365
379
|
|
|
@@ -387,6 +401,9 @@ def eval_command(
|
|
|
387
401
|
|
|
388
402
|
# Run with OpenAI Operator agent
|
|
389
403
|
hud eval hud-evals/OSWorld-Gold-Beta --agent openai
|
|
404
|
+
|
|
405
|
+
# Run with verbose output for debugging
|
|
406
|
+
hud eval task.json --verbose
|
|
390
407
|
"""
|
|
391
408
|
from hud.settings import settings
|
|
392
409
|
|
|
@@ -428,6 +445,7 @@ def eval_command(
|
|
|
428
445
|
parallel=parallel,
|
|
429
446
|
max_workers=max_workers,
|
|
430
447
|
max_concurrent_per_worker=max_concurrent_per_worker,
|
|
448
|
+
verbose=verbose,
|
|
431
449
|
)
|
|
432
450
|
)
|
|
433
451
|
else:
|
|
@@ -438,5 +456,6 @@ def eval_command(
|
|
|
438
456
|
model=model,
|
|
439
457
|
allowed_tools=allowed_tools_list,
|
|
440
458
|
max_steps=max_steps,
|
|
459
|
+
verbose=verbose,
|
|
441
460
|
)
|
|
442
461
|
)
|
hud/cli/init.py
CHANGED
|
@@ -182,15 +182,15 @@ async def run_task(task_data: dict):
|
|
|
182
182
|
await client.initialize()
|
|
183
183
|
|
|
184
184
|
result = await client.call_tool(task.setup_tool) # type: ignore
|
|
185
|
-
print(f"✅ Setup: {
|
|
185
|
+
print(f"✅ Setup: {result.content}")
|
|
186
186
|
|
|
187
187
|
print("\\n🔄 Performing actions:")
|
|
188
188
|
for _ in range(10):
|
|
189
|
-
result = await client.call_tool(name="act", arguments={
|
|
190
|
-
print(f" {
|
|
189
|
+
result = await client.call_tool(name="act", arguments={})
|
|
190
|
+
print(f" {result.content}")
|
|
191
191
|
|
|
192
192
|
result = await client.call_tool(task.evaluate_tool) # type: ignore
|
|
193
|
-
print(f"\\n📊 Evaluation: {
|
|
193
|
+
print(f"\\n📊 Evaluation: {result.content}")
|
|
194
194
|
|
|
195
195
|
return result.content
|
|
196
196
|
except Exception as e:
|
hud/cli/push.py
CHANGED
|
@@ -11,7 +11,6 @@ import requests
|
|
|
11
11
|
import typer
|
|
12
12
|
import yaml
|
|
13
13
|
|
|
14
|
-
from hud.settings import settings
|
|
15
14
|
from hud.utils.design import HUDDesign
|
|
16
15
|
|
|
17
16
|
|
|
@@ -127,6 +126,9 @@ def push_environment(
|
|
|
127
126
|
design = HUDDesign()
|
|
128
127
|
design.header("HUD Environment Push")
|
|
129
128
|
|
|
129
|
+
# Import settings lazily after any environment setup
|
|
130
|
+
from hud.settings import settings
|
|
131
|
+
|
|
130
132
|
# Find hud.lock.yaml in specified directory
|
|
131
133
|
env_dir = Path(directory)
|
|
132
134
|
lock_path = env_dir / "hud.lock.yaml"
|
hud/cli/tests/test_push.py
CHANGED
|
@@ -123,7 +123,7 @@ class TestPushEnvironment:
|
|
|
123
123
|
mock_design.error.assert_called()
|
|
124
124
|
|
|
125
125
|
@mock.patch("hud.cli.push.HUDDesign")
|
|
126
|
-
@mock.patch("hud.
|
|
126
|
+
@mock.patch("hud.settings.settings")
|
|
127
127
|
def test_push_no_api_key(self, mock_settings, mock_design_class, tmp_path):
|
|
128
128
|
"""Test pushing without API key."""
|
|
129
129
|
mock_design = mock.Mock()
|
|
@@ -143,7 +143,7 @@ class TestPushEnvironment:
|
|
|
143
143
|
@mock.patch("subprocess.Popen")
|
|
144
144
|
@mock.patch("subprocess.run")
|
|
145
145
|
@mock.patch("hud.cli.push.get_docker_username")
|
|
146
|
-
@mock.patch("hud.
|
|
146
|
+
@mock.patch("hud.settings.settings")
|
|
147
147
|
@mock.patch("hud.cli.push.HUDDesign")
|
|
148
148
|
def test_push_auto_detect_username(
|
|
149
149
|
self,
|
|
@@ -205,7 +205,7 @@ class TestPushEnvironment:
|
|
|
205
205
|
assert "testuser/image%3A0.1.0" in call_args[0][0]
|
|
206
206
|
|
|
207
207
|
@mock.patch("subprocess.run")
|
|
208
|
-
@mock.patch("hud.
|
|
208
|
+
@mock.patch("hud.settings.settings")
|
|
209
209
|
@mock.patch("hud.cli.push.HUDDesign")
|
|
210
210
|
def test_push_explicit_image(self, mock_design_class, mock_settings, mock_run, tmp_path):
|
|
211
211
|
"""Test pushing with explicit image name."""
|
|
@@ -226,7 +226,7 @@ class TestPushEnvironment:
|
|
|
226
226
|
|
|
227
227
|
@mock.patch("subprocess.Popen")
|
|
228
228
|
@mock.patch("subprocess.run")
|
|
229
|
-
@mock.patch("hud.
|
|
229
|
+
@mock.patch("hud.settings.settings")
|
|
230
230
|
@mock.patch("hud.cli.push.HUDDesign")
|
|
231
231
|
def test_push_with_tag(self, mock_design_class, mock_settings, mock_run, mock_popen, tmp_path):
|
|
232
232
|
"""Test pushing with explicit tag."""
|
|
@@ -282,7 +282,7 @@ class TestPushEnvironment:
|
|
|
282
282
|
mock_process.returncode = 1
|
|
283
283
|
mock_popen.return_value = mock_process
|
|
284
284
|
|
|
285
|
-
with mock.patch("hud.
|
|
285
|
+
with mock.patch("hud.settings.settings") as mock_settings:
|
|
286
286
|
mock_settings.api_key = "test-key"
|
|
287
287
|
with (
|
|
288
288
|
mock.patch("subprocess.run"),
|
|
@@ -292,7 +292,7 @@ class TestPushEnvironment:
|
|
|
292
292
|
|
|
293
293
|
@mock.patch("hud.cli.push.get_docker_image_labels")
|
|
294
294
|
@mock.patch("subprocess.run")
|
|
295
|
-
@mock.patch("hud.
|
|
295
|
+
@mock.patch("hud.settings.settings")
|
|
296
296
|
@mock.patch("hud.cli.push.HUDDesign")
|
|
297
297
|
def test_push_with_labels(
|
|
298
298
|
self, mock_design_class, mock_settings, mock_run, mock_get_labels, tmp_path
|
hud/cli/utils/interactive.py
CHANGED
hud/clients/__init__.py
CHANGED
|
@@ -4,9 +4,10 @@ from __future__ import annotations
|
|
|
4
4
|
|
|
5
5
|
from .base import AgentMCPClient, BaseHUDClient
|
|
6
6
|
from .fastmcp import FastMCPHUDClient
|
|
7
|
+
from .mcp_use import MCPUseHUDClient
|
|
7
8
|
|
|
8
|
-
# Default to
|
|
9
|
-
MCPClient =
|
|
9
|
+
# Default to MCP-use for new features
|
|
10
|
+
MCPClient = MCPUseHUDClient
|
|
10
11
|
|
|
11
12
|
__all__ = [
|
|
12
13
|
"AgentMCPClient",
|
hud/clients/base.py
CHANGED
|
@@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Any, Protocol, overload, runtime_checkable
|
|
|
9
9
|
|
|
10
10
|
from mcp.types import Implementation
|
|
11
11
|
|
|
12
|
+
from hud.shared.exceptions import HudAuthenticationError, HudException
|
|
12
13
|
from hud.types import MCPToolCall, MCPToolResult
|
|
13
14
|
from hud.utils.mcp import setup_hud_telemetry
|
|
14
15
|
from hud.version import __version__ as hud_version
|
|
@@ -120,8 +121,10 @@ class BaseHUDClient(AgentMCPClient):
|
|
|
120
121
|
|
|
121
122
|
self._mcp_config = mcp_config or self._mcp_config
|
|
122
123
|
if self._mcp_config is None:
|
|
123
|
-
|
|
124
|
-
|
|
124
|
+
from hud.shared.exceptions import HudConfigError
|
|
125
|
+
|
|
126
|
+
raise HudConfigError(
|
|
127
|
+
"An MCP server configuration is required. "
|
|
125
128
|
"Either pass it to the constructor or call initialize with a configuration"
|
|
126
129
|
)
|
|
127
130
|
|
|
@@ -135,14 +138,18 @@ class BaseHUDClient(AgentMCPClient):
|
|
|
135
138
|
url = server_config.get("url", "")
|
|
136
139
|
headers = server_config.get("headers", {})
|
|
137
140
|
if "mcp.hud.so" in url and len(headers.get("Authorization", "")) < 10:
|
|
138
|
-
raise
|
|
139
|
-
|
|
140
|
-
"
|
|
141
|
+
raise HudAuthenticationError(
|
|
142
|
+
f'Sending authorization "{headers.get("Authorization", "")}", which may'
|
|
143
|
+
" be incomplete. Ensure HUD_API_KEY environment variable is set or send it"
|
|
144
|
+
" as a header. You can get an API key at https://app.hud.so"
|
|
141
145
|
)
|
|
142
146
|
# Subclasses implement connection
|
|
143
147
|
await self._connect(self._mcp_config)
|
|
148
|
+
except HudException:
|
|
149
|
+
raise
|
|
144
150
|
except Exception as e:
|
|
145
|
-
|
|
151
|
+
# Auto-converts to appropriate HUD exception type with hints
|
|
152
|
+
raise HudException from e
|
|
146
153
|
|
|
147
154
|
# Common hud behavior - fetch telemetry
|
|
148
155
|
await self._fetch_telemetry()
|
|
@@ -168,7 +175,7 @@ class BaseHUDClient(AgentMCPClient):
|
|
|
168
175
|
self._initialized = False
|
|
169
176
|
logger.info("Client disconnected")
|
|
170
177
|
else:
|
|
171
|
-
logger.
|
|
178
|
+
logger.debug("Client was not initialized, skipping disconnect")
|
|
172
179
|
|
|
173
180
|
@overload
|
|
174
181
|
async def call_tool(self, tool_call: MCPToolCall, /) -> MCPToolResult: ...
|
|
@@ -236,7 +243,9 @@ class BaseHUDClient(AgentMCPClient):
|
|
|
236
243
|
def mcp_config(self) -> dict[str, dict[str, Any]]:
|
|
237
244
|
"""Get the MCP config."""
|
|
238
245
|
if self._mcp_config is None:
|
|
239
|
-
|
|
246
|
+
from hud.shared.exceptions import HudConfigError
|
|
247
|
+
|
|
248
|
+
raise HudConfigError("Please initialize the client with a valid MCP config")
|
|
240
249
|
return self._mcp_config
|
|
241
250
|
|
|
242
251
|
async def __aenter__(self: Any) -> Any:
|
|
@@ -305,7 +314,9 @@ class BaseHUDClient(AgentMCPClient):
|
|
|
305
314
|
- metadata: Environment metadata
|
|
306
315
|
"""
|
|
307
316
|
if not self._initialized:
|
|
308
|
-
|
|
317
|
+
from hud.shared.exceptions import HudClientError
|
|
318
|
+
|
|
319
|
+
raise HudClientError("Client must be initialized before analyzing the environment")
|
|
309
320
|
|
|
310
321
|
analysis: dict[str, Any] = {
|
|
311
322
|
"tools": [],
|
hud/clients/mcp_use.py
CHANGED
|
@@ -3,10 +3,12 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
import logging
|
|
6
|
-
from typing import
|
|
6
|
+
from typing import Any
|
|
7
7
|
|
|
8
|
-
from mcp import Implementation
|
|
8
|
+
from mcp import Implementation, types
|
|
9
9
|
from mcp.shared.exceptions import McpError
|
|
10
|
+
from mcp_use.client import MCPClient as MCPUseClient
|
|
11
|
+
from mcp_use.session import MCPSession as MCPUseSession
|
|
10
12
|
from pydantic import AnyUrl
|
|
11
13
|
|
|
12
14
|
from hud.types import MCPToolCall, MCPToolResult
|
|
@@ -14,18 +16,6 @@ from hud.version import __version__ as hud_version
|
|
|
14
16
|
|
|
15
17
|
from .base import BaseHUDClient
|
|
16
18
|
|
|
17
|
-
if TYPE_CHECKING:
|
|
18
|
-
from mcp import types
|
|
19
|
-
from mcp_use.client import MCPClient as MCPUseClient # type: ignore[attr-defined]
|
|
20
|
-
from mcp_use.session import MCPSession as MCPUseSession # type: ignore[attr-defined]
|
|
21
|
-
|
|
22
|
-
try:
|
|
23
|
-
from mcp_use.client import MCPClient as MCPUseClient # type: ignore[attr-defined]
|
|
24
|
-
from mcp_use.session import MCPSession as MCPUseSession # type: ignore[attr-defined]
|
|
25
|
-
except ImportError:
|
|
26
|
-
MCPUseClient = None # type: ignore[misc, assignment]
|
|
27
|
-
MCPUseSession = None # type: ignore[misc, assignment]
|
|
28
|
-
|
|
29
19
|
logger = logging.getLogger(__name__)
|
|
30
20
|
|
|
31
21
|
|
|
@@ -53,7 +43,9 @@ class MCPUseHUDClient(BaseHUDClient):
|
|
|
53
43
|
)
|
|
54
44
|
|
|
55
45
|
self._sessions: dict[str, Any] = {} # Will be MCPUseSession when available
|
|
56
|
-
self._tool_map: dict[
|
|
46
|
+
self._tool_map: dict[
|
|
47
|
+
str, tuple[str, types.Tool, types.Tool]
|
|
48
|
+
] = {} # server_name, original_tool, prefixed_tool
|
|
57
49
|
self._client: Any | None = None # Will be MCPUseClient when available
|
|
58
50
|
|
|
59
51
|
async def _connect(self, mcp_config: dict[str, dict[str, Any]]) -> None:
|
|
@@ -106,14 +98,23 @@ class MCPUseHUDClient(BaseHUDClient):
|
|
|
106
98
|
logger.info("Check that the MCP server is running and accessible")
|
|
107
99
|
raise
|
|
108
100
|
|
|
101
|
+
# Populate tool map during initialization
|
|
102
|
+
await self.list_tools()
|
|
103
|
+
|
|
109
104
|
async def list_tools(self) -> list[types.Tool]:
|
|
110
105
|
"""List all available tools from all sessions."""
|
|
111
106
|
if self._client is None or not self._sessions:
|
|
112
107
|
raise ValueError("Client is not connected, call initialize() first")
|
|
113
108
|
|
|
109
|
+
if self._tool_map:
|
|
110
|
+
return [tool[2] for tool in self._tool_map.values()]
|
|
111
|
+
|
|
114
112
|
all_tools = []
|
|
115
113
|
self._tool_map = {}
|
|
116
114
|
|
|
115
|
+
# Check if we need to prefix (more than one server)
|
|
116
|
+
use_prefix = len(self._sessions) > 1
|
|
117
|
+
|
|
117
118
|
for server_name, session in self._sessions.items():
|
|
118
119
|
try:
|
|
119
120
|
# Ensure session is initialized
|
|
@@ -136,10 +137,26 @@ class MCPUseHUDClient(BaseHUDClient):
|
|
|
136
137
|
[tool.name for tool in tools_result.tools],
|
|
137
138
|
)
|
|
138
139
|
|
|
139
|
-
# Add to collections
|
|
140
|
+
# Add to collections with optional prefix
|
|
140
141
|
for tool in tools_result.tools:
|
|
141
|
-
|
|
142
|
-
|
|
142
|
+
if use_prefix:
|
|
143
|
+
# Create a new tool with prefixed name
|
|
144
|
+
prefixed_name = f"{server_name}_{tool.name}"
|
|
145
|
+
# Create a new tool instance with prefixed name
|
|
146
|
+
from mcp import types as mcp_types
|
|
147
|
+
|
|
148
|
+
prefixed_tool = mcp_types.Tool(
|
|
149
|
+
name=prefixed_name,
|
|
150
|
+
description=tool.description,
|
|
151
|
+
inputSchema=tool.inputSchema,
|
|
152
|
+
)
|
|
153
|
+
all_tools.append(prefixed_tool)
|
|
154
|
+
# Map prefixed name to (server_name, original_tool)
|
|
155
|
+
self._tool_map[prefixed_name] = (server_name, tool, prefixed_tool)
|
|
156
|
+
else:
|
|
157
|
+
# Single server - no prefix needed
|
|
158
|
+
all_tools.append(tool)
|
|
159
|
+
self._tool_map[tool.name] = (server_name, tool, tool)
|
|
143
160
|
|
|
144
161
|
# Log detailed tool info in verbose mode
|
|
145
162
|
if self.verbose:
|
|
@@ -164,15 +181,20 @@ class MCPUseHUDClient(BaseHUDClient):
|
|
|
164
181
|
raise ValueError("Client is not connected, call initialize() first")
|
|
165
182
|
|
|
166
183
|
if tool_call.name not in self._tool_map:
|
|
167
|
-
|
|
184
|
+
return MCPToolResult(
|
|
185
|
+
content=[types.TextContent(type="text", text=f"Tool '{tool_call.name}' not found")],
|
|
186
|
+
isError=True,
|
|
187
|
+
structuredContent=None,
|
|
188
|
+
)
|
|
168
189
|
|
|
169
|
-
server_name, _ = self._tool_map[tool_call.name]
|
|
190
|
+
server_name, original_tool, _ = self._tool_map[tool_call.name]
|
|
170
191
|
session = self._sessions[server_name]
|
|
171
192
|
|
|
172
193
|
if self.verbose:
|
|
173
194
|
logger.debug(
|
|
174
|
-
"Calling tool '%s' on server '%s' with arguments: %s",
|
|
195
|
+
"Calling tool '%s' (original: '%s') on server '%s' with arguments: %s",
|
|
175
196
|
tool_call.name,
|
|
197
|
+
original_tool.name,
|
|
176
198
|
server_name,
|
|
177
199
|
tool_call.arguments,
|
|
178
200
|
)
|
|
@@ -181,7 +203,7 @@ class MCPUseHUDClient(BaseHUDClient):
|
|
|
181
203
|
raise ValueError(f"Client session not initialized for {server_name}")
|
|
182
204
|
|
|
183
205
|
result = await session.connector.client_session.call_tool(
|
|
184
|
-
name=
|
|
206
|
+
name=original_tool.name, # Use original tool name, not prefixed
|
|
185
207
|
arguments=tool_call.arguments or {},
|
|
186
208
|
)
|
|
187
209
|
|
hud/datasets/task.py
CHANGED
|
@@ -51,7 +51,9 @@ class Task(BaseModel):
|
|
|
51
51
|
try:
|
|
52
52
|
return json.loads(v)
|
|
53
53
|
except json.JSONDecodeError as e:
|
|
54
|
-
|
|
54
|
+
from hud.shared.exceptions import HudConfigError
|
|
55
|
+
|
|
56
|
+
raise HudConfigError(f"Invalid JSON string: {e}") from e
|
|
55
57
|
return v
|
|
56
58
|
|
|
57
59
|
@field_validator("setup_tool", "evaluate_tool", mode="before")
|
|
@@ -66,7 +68,9 @@ class Task(BaseModel):
|
|
|
66
68
|
try:
|
|
67
69
|
v = json.loads(v)
|
|
68
70
|
except json.JSONDecodeError as e:
|
|
69
|
-
|
|
71
|
+
from hud.shared.exceptions import HudConfigError
|
|
72
|
+
|
|
73
|
+
raise HudConfigError(f"Invalid JSON string: {e}") from e
|
|
70
74
|
|
|
71
75
|
if isinstance(v, dict):
|
|
72
76
|
return MCPToolCall(**v)
|