hud-python 0.4.20__py3-none-any.whl → 0.4.22__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of hud-python might be problematic. Click here for more details.

Files changed (54) hide show
  1. hud/__init__.py +7 -0
  2. hud/agents/base.py +42 -10
  3. hud/agents/claude.py +24 -14
  4. hud/agents/grounded_openai.py +280 -0
  5. hud/agents/tests/test_client.py +11 -27
  6. hud/agents/tests/test_grounded_openai_agent.py +155 -0
  7. hud/cli/__init__.py +50 -20
  8. hud/cli/build.py +3 -44
  9. hud/cli/eval.py +25 -6
  10. hud/cli/init.py +4 -4
  11. hud/cli/push.py +3 -1
  12. hud/cli/tests/test_push.py +6 -6
  13. hud/cli/utils/interactive.py +1 -1
  14. hud/clients/__init__.py +3 -2
  15. hud/clients/base.py +20 -9
  16. hud/clients/mcp_use.py +44 -22
  17. hud/datasets/task.py +6 -2
  18. hud/native/__init__.py +6 -0
  19. hud/native/comparator.py +546 -0
  20. hud/native/tests/__init__.py +1 -0
  21. hud/native/tests/test_comparator.py +539 -0
  22. hud/native/tests/test_native_init.py +79 -0
  23. hud/otel/instrumentation.py +0 -2
  24. hud/server/server.py +9 -2
  25. hud/settings.py +6 -0
  26. hud/shared/exceptions.py +204 -31
  27. hud/shared/hints.py +177 -0
  28. hud/shared/requests.py +15 -3
  29. hud/shared/tests/test_exceptions.py +385 -144
  30. hud/tools/__init__.py +2 -0
  31. hud/tools/executors/tests/test_base_executor.py +1 -1
  32. hud/tools/executors/xdo.py +1 -1
  33. hud/tools/grounding/__init__.py +13 -0
  34. hud/tools/grounding/config.py +54 -0
  35. hud/tools/grounding/grounded_tool.py +314 -0
  36. hud/tools/grounding/grounder.py +301 -0
  37. hud/tools/grounding/tests/__init__.py +1 -0
  38. hud/tools/grounding/tests/test_grounded_tool.py +196 -0
  39. hud/tools/submit.py +66 -0
  40. hud/tools/tests/test_playwright_tool.py +1 -1
  41. hud/tools/tests/test_tools_init.py +1 -1
  42. hud/tools/tests/test_utils.py +2 -2
  43. hud/types.py +33 -5
  44. hud/utils/agent_factories.py +86 -0
  45. hud/utils/design.py +57 -0
  46. hud/utils/mcp.py +6 -0
  47. hud/utils/pretty_errors.py +68 -0
  48. hud/utils/tests/test_version.py +1 -1
  49. hud/version.py +1 -1
  50. {hud_python-0.4.20.dist-info → hud_python-0.4.22.dist-info}/METADATA +2 -4
  51. {hud_python-0.4.20.dist-info → hud_python-0.4.22.dist-info}/RECORD +54 -37
  52. {hud_python-0.4.20.dist-info → hud_python-0.4.22.dist-info}/WHEEL +0 -0
  53. {hud_python-0.4.20.dist-info → hud_python-0.4.22.dist-info}/entry_points.txt +0 -0
  54. {hud_python-0.4.20.dist-info → hud_python-0.4.22.dist-info}/licenses/LICENSE +0 -0
hud/cli/__init__.py CHANGED
@@ -43,6 +43,12 @@ app = typer.Typer(
43
43
 
44
44
  console = Console()
45
45
 
46
+ # Standard support hint appended to error outputs
47
+ SUPPORT_HINT = (
48
+ "If this looks like an issue with the sdk, please make a github issue at "
49
+ "https://github.com/hud-evals/hud-python/issues"
50
+ )
51
+
46
52
 
47
53
  # Capture IMAGE and any following Docker args as a single variadic argument list.
48
54
  @app.command(context_settings={"allow_extra_args": True, "ignore_unknown_options": True})
@@ -818,6 +824,11 @@ def eval(
818
824
  "--max-concurrent-per-worker",
819
825
  help="Maximum concurrent tasks per worker in parallel mode",
820
826
  ),
827
+ verbose: bool = typer.Option(
828
+ False,
829
+ "--verbose",
830
+ help="Enable verbose output from the agent",
831
+ ),
821
832
  ) -> None:
822
833
  """🚀 Run evaluation on datasets or individual tasks with agents."""
823
834
  from hud.utils.design import HUDDesign
@@ -912,6 +923,7 @@ def eval(
912
923
  parallel=parallel,
913
924
  max_workers=max_workers,
914
925
  max_concurrent_per_worker=max_concurrent_per_worker,
926
+ verbose=verbose,
915
927
  )
916
928
 
917
929
 
@@ -950,27 +962,45 @@ def hf(
950
962
 
951
963
  def main() -> None:
952
964
  """Main entry point for the CLI."""
953
- # Show header for main help
954
- if len(sys.argv) == 1 or (len(sys.argv) == 2 and sys.argv[1] in ["--help", "-h"]):
955
- console.print(
956
- Panel.fit(
957
- "[bold cyan]🚀 HUD CLI[/bold cyan]\nMCP Environment Analysis & Debugging",
958
- border_style="cyan",
965
+ try:
966
+ # Show header for main help
967
+ if len(sys.argv) == 1 or (len(sys.argv) == 2 and sys.argv[1] in ["--help", "-h"]):
968
+ console.print(
969
+ Panel.fit(
970
+ "[bold cyan]🚀 HUD CLI[/bold cyan]\nMCP Environment Analysis & Debugging",
971
+ border_style="cyan",
972
+ )
959
973
  )
960
- )
961
- console.print("\n[yellow]Quick Start:[/yellow]")
962
- console.print(" 1. Create a new environment: [cyan]hud init my-env && cd my-env[/cyan]")
963
- console.print(" 2. Develop with hot-reload: [cyan]hud dev --interactive[/cyan]")
964
- console.print(" 3. Build for production: [cyan]hud build[/cyan]")
965
- console.print(" 4. Share your environment: [cyan]hud push[/cyan]")
966
- console.print(" 5. Get shared environments: [cyan]hud pull <org/name:tag>[/cyan]")
967
- console.print(" 6. Run and test: [cyan]hud run <image>[/cyan]")
968
- console.print("\n[yellow]RL Training:[/yellow]")
969
- console.print(" 1. Generate config: [cyan]hud rl init my-env:latest[/cyan]")
970
- console.print(" 2. Create dataset: [cyan]hud hf tasks.json --name my-org/my-tasks[/cyan]")
971
- console.print(" 3. Start training: [cyan]hud rl --model Qwen/Qwen2.5-3B[/cyan]\n")
972
-
973
- app()
974
+ console.print("\n[yellow]Quick Start:[/yellow]")
975
+ console.print(
976
+ " 1. Create a new environment: [cyan]hud init my-env && cd my-env[/cyan]"
977
+ )
978
+ console.print(" 2. Develop with hot-reload: [cyan]hud dev --interactive[/cyan]")
979
+ console.print(" 3. Build for production: [cyan]hud build[/cyan]")
980
+ console.print(" 4. Share your environment: [cyan]hud push[/cyan]")
981
+ console.print(" 5. Get shared environments: [cyan]hud pull <org/name:tag>[/cyan]")
982
+ console.print(" 6. Run and test: [cyan]hud run <image>[/cyan]")
983
+ console.print("\n[yellow]RL Training:[/yellow]")
984
+ console.print(" 1. Generate config: [cyan]hud rl init my-env:latest[/cyan]")
985
+ console.print(
986
+ " 2. Create dataset: [cyan]hud hf tasks.json --name my-org/my-tasks[/cyan]"
987
+ )
988
+ console.print(" 3. Start training: [cyan]hud rl --model Qwen/Qwen2.5-3B[/cyan]\n")
989
+
990
+ app()
991
+ except typer.Exit as e:
992
+ # Append SDK support hint for non-zero exits
993
+ try:
994
+ exit_code = getattr(e, "exit_code", 0)
995
+ except Exception:
996
+ exit_code = 1
997
+ if exit_code != 0:
998
+ from hud.utils.design import design
999
+
1000
+ design.info(SUPPORT_HINT)
1001
+ raise
1002
+ except Exception:
1003
+ raise
974
1004
 
975
1005
 
976
1006
  if __name__ == "__main__":
hud/cli/build.py CHANGED
@@ -204,30 +204,10 @@ async def analyze_mcp_environment(
204
204
  "success": True,
205
205
  }
206
206
  except Exception as e:
207
- import traceback
207
+ from hud.shared.exceptions import HudException
208
208
 
209
- error_msg = str(e)
210
- if verbose:
211
- design.error(f"Failed to analyze environment: {error_msg}")
212
- design.error(f"Traceback:\n{traceback.format_exc()}")
213
-
214
- # Common issues
215
- if "Connection reset" in error_msg or "EOF" in error_msg:
216
- design.warning(
217
- "The MCP server may have crashed on startup. Check your server.py for errors."
218
- )
219
- elif "timeout" in error_msg:
220
- design.warning(
221
- "The MCP server took too long to initialize. It might need more startup time."
222
- )
223
-
224
- return {
225
- "initializeMs": 0,
226
- "toolCount": 0,
227
- "tools": [],
228
- "success": False,
229
- "error": error_msg,
230
- }
209
+ # Convert to HudException for better error messages and hints
210
+ raise HudException from e
231
211
  finally:
232
212
  # Only shutdown if we successfully initialized
233
213
  if initialized:
@@ -340,27 +320,6 @@ def build_environment(
340
320
  finally:
341
321
  loop.close()
342
322
 
343
- if not analysis["success"]:
344
- design.error("Failed to analyze MCP environment")
345
- if "error" in analysis:
346
- design.error(f"Error: {analysis['error']}")
347
-
348
- # Provide helpful debugging tips
349
- design.section_title("Debugging Tips")
350
- design.info("1. Debug your environment build:")
351
- design.command_example("hud debug . --build")
352
- design.dim_info(" This will", "test MCP server connection and show detailed logs")
353
- design.info("")
354
- design.info("2. Check for common issues:")
355
- design.info(" - Server crashes on startup")
356
- design.info(" - Missing dependencies")
357
- design.info(" - Syntax errors in server.py")
358
- design.info("")
359
- design.info("3. Run with verbose mode:")
360
- design.command_example("hud build . --verbose")
361
-
362
- raise typer.Exit(1)
363
-
364
323
  design.success(f"Analyzed environment: {analysis['toolCount']} tools found")
365
324
 
366
325
  # Extract environment variables from Dockerfile
hud/cli/eval.py CHANGED
@@ -22,6 +22,7 @@ def build_agent(
22
22
  *,
23
23
  model: str | None = None,
24
24
  allowed_tools: list[str] | None = None,
25
+ verbose: bool = False,
25
26
  ) -> Any:
26
27
  """Create and return the requested agent type."""
27
28
 
@@ -39,9 +40,10 @@ def build_agent(
39
40
  if allowed_tools:
40
41
  return OperatorAgent(
41
42
  allowed_tools=allowed_tools,
43
+ verbose=verbose,
42
44
  )
43
45
  else:
44
- return OperatorAgent()
46
+ return OperatorAgent(verbose=verbose)
45
47
 
46
48
  # Fallback Claude agent (Anthropic)
47
49
  try:
@@ -59,10 +61,12 @@ def build_agent(
59
61
  return ClaudeAgent(
60
62
  model=model,
61
63
  allowed_tools=allowed_tools,
64
+ verbose=verbose,
62
65
  )
63
66
  else:
64
67
  return ClaudeAgent(
65
68
  model=model,
69
+ verbose=verbose,
66
70
  )
67
71
 
68
72
 
@@ -73,6 +77,7 @@ async def run_single_task(
73
77
  model: str | None = None,
74
78
  allowed_tools: list[str] | None = None,
75
79
  max_steps: int = 10,
80
+ verbose: bool = False,
76
81
  ) -> None:
77
82
  """Load one task and execute it, or detect if JSON contains a list and run as dataset."""
78
83
 
@@ -82,7 +87,7 @@ async def run_single_task(
82
87
  except ImportError as e:
83
88
  design.error(
84
89
  "Dataset dependencies are not installed. "
85
- "Please install with: pip install 'hud-python[agent]'"
90
+ "Please install with: pip install 'hud-python\u27e6agent\u27e7'"
86
91
  )
87
92
  raise typer.Exit(1) from e
88
93
 
@@ -106,11 +111,11 @@ async def run_single_task(
106
111
  except ImportError as e:
107
112
  design.error(
108
113
  "OpenAI agent dependencies are not installed. "
109
- "Please install with: pip install 'hud-python[agent]'"
114
+ "Please install with: pip install 'hud-python\u27e6agent\u27e7'"
110
115
  )
111
116
  raise typer.Exit(1) from e
112
117
 
113
- agent_config: dict[str, Any] = {}
118
+ agent_config: dict[str, Any] = {"verbose": verbose}
114
119
  if allowed_tools:
115
120
  agent_config["allowed_tools"] = allowed_tools
116
121
 
@@ -128,6 +133,7 @@ async def run_single_task(
128
133
 
129
134
  agent_config = {
130
135
  "model": model or "claude-sonnet-4-20250514",
136
+ "verbose": verbose,
131
137
  }
132
138
  if allowed_tools:
133
139
  agent_config["allowed_tools"] = allowed_tools
@@ -182,6 +188,7 @@ async def run_single_task(
182
188
  agent_type,
183
189
  model=model,
184
190
  allowed_tools=allowed_tools,
191
+ verbose=verbose,
185
192
  )
186
193
  design.info(task.prompt)
187
194
  result = await agent.run(task, max_steps=max_steps)
@@ -199,6 +206,7 @@ async def run_full_dataset(
199
206
  parallel: bool = False,
200
207
  max_workers: int | None = None,
201
208
  max_concurrent_per_worker: int = 25,
209
+ verbose: bool = False,
202
210
  ) -> list[Any]:
203
211
  """Run evaluation across the entire dataset.
204
212
 
@@ -211,7 +219,7 @@ async def run_full_dataset(
211
219
  except ImportError as e:
212
220
  design.error(
213
221
  "Dataset dependencies are not installed. "
214
- "Please install with: pip install 'hud-python[agent]'"
222
+ "Please install with: pip install 'hud-python[[agent]]'"
215
223
  )
216
224
  raise typer.Exit(1) from e
217
225
 
@@ -245,7 +253,7 @@ async def run_full_dataset(
245
253
  )
246
254
  raise typer.Exit(1) from e
247
255
 
248
- agent_config: dict[str, Any] = {}
256
+ agent_config: dict[str, Any] = {"verbose": verbose}
249
257
  if allowed_tools:
250
258
  agent_config["allowed_tools"] = allowed_tools
251
259
 
@@ -263,6 +271,7 @@ async def run_full_dataset(
263
271
 
264
272
  agent_config = {
265
273
  "model": model or "claude-sonnet-4-20250514",
274
+ "verbose": verbose,
266
275
  }
267
276
  if allowed_tools:
268
277
  agent_config["allowed_tools"] = allowed_tools
@@ -360,6 +369,11 @@ def eval_command(
360
369
  "--max-concurrent-per-worker",
361
370
  help="Maximum concurrent tasks per worker in parallel mode",
362
371
  ),
372
+ verbose: bool = typer.Option(
373
+ False,
374
+ "--verbose",
375
+ help="Enable verbose output from the agent",
376
+ ),
363
377
  ) -> None:
364
378
  """🚀 Run evaluation on datasets or individual tasks with agents.
365
379
 
@@ -387,6 +401,9 @@ def eval_command(
387
401
 
388
402
  # Run with OpenAI Operator agent
389
403
  hud eval hud-evals/OSWorld-Gold-Beta --agent openai
404
+
405
+ # Run with verbose output for debugging
406
+ hud eval task.json --verbose
390
407
  """
391
408
  from hud.settings import settings
392
409
 
@@ -428,6 +445,7 @@ def eval_command(
428
445
  parallel=parallel,
429
446
  max_workers=max_workers,
430
447
  max_concurrent_per_worker=max_concurrent_per_worker,
448
+ verbose=verbose,
431
449
  )
432
450
  )
433
451
  else:
@@ -438,5 +456,6 @@ def eval_command(
438
456
  model=model,
439
457
  allowed_tools=allowed_tools_list,
440
458
  max_steps=max_steps,
459
+ verbose=verbose,
441
460
  )
442
461
  )
hud/cli/init.py CHANGED
@@ -182,15 +182,15 @@ async def run_task(task_data: dict):
182
182
  await client.initialize()
183
183
 
184
184
  result = await client.call_tool(task.setup_tool) # type: ignore
185
- print(f"✅ Setup: {{result.content}}")
185
+ print(f"✅ Setup: {result.content}")
186
186
 
187
187
  print("\\n🔄 Performing actions:")
188
188
  for _ in range(10):
189
- result = await client.call_tool(name="act", arguments={{}})
190
- print(f" {{result.content}}")
189
+ result = await client.call_tool(name="act", arguments={})
190
+ print(f" {result.content}")
191
191
 
192
192
  result = await client.call_tool(task.evaluate_tool) # type: ignore
193
- print(f"\\n📊 Evaluation: {{result.content}}")
193
+ print(f"\\n📊 Evaluation: {result.content}")
194
194
 
195
195
  return result.content
196
196
  except Exception as e:
hud/cli/push.py CHANGED
@@ -11,7 +11,6 @@ import requests
11
11
  import typer
12
12
  import yaml
13
13
 
14
- from hud.settings import settings
15
14
  from hud.utils.design import HUDDesign
16
15
 
17
16
 
@@ -127,6 +126,9 @@ def push_environment(
127
126
  design = HUDDesign()
128
127
  design.header("HUD Environment Push")
129
128
 
129
+ # Import settings lazily after any environment setup
130
+ from hud.settings import settings
131
+
130
132
  # Find hud.lock.yaml in specified directory
131
133
  env_dir = Path(directory)
132
134
  lock_path = env_dir / "hud.lock.yaml"
@@ -123,7 +123,7 @@ class TestPushEnvironment:
123
123
  mock_design.error.assert_called()
124
124
 
125
125
  @mock.patch("hud.cli.push.HUDDesign")
126
- @mock.patch("hud.cli.push.settings")
126
+ @mock.patch("hud.settings.settings")
127
127
  def test_push_no_api_key(self, mock_settings, mock_design_class, tmp_path):
128
128
  """Test pushing without API key."""
129
129
  mock_design = mock.Mock()
@@ -143,7 +143,7 @@ class TestPushEnvironment:
143
143
  @mock.patch("subprocess.Popen")
144
144
  @mock.patch("subprocess.run")
145
145
  @mock.patch("hud.cli.push.get_docker_username")
146
- @mock.patch("hud.cli.push.settings")
146
+ @mock.patch("hud.settings.settings")
147
147
  @mock.patch("hud.cli.push.HUDDesign")
148
148
  def test_push_auto_detect_username(
149
149
  self,
@@ -205,7 +205,7 @@ class TestPushEnvironment:
205
205
  assert "testuser/image%3A0.1.0" in call_args[0][0]
206
206
 
207
207
  @mock.patch("subprocess.run")
208
- @mock.patch("hud.cli.push.settings")
208
+ @mock.patch("hud.settings.settings")
209
209
  @mock.patch("hud.cli.push.HUDDesign")
210
210
  def test_push_explicit_image(self, mock_design_class, mock_settings, mock_run, tmp_path):
211
211
  """Test pushing with explicit image name."""
@@ -226,7 +226,7 @@ class TestPushEnvironment:
226
226
 
227
227
  @mock.patch("subprocess.Popen")
228
228
  @mock.patch("subprocess.run")
229
- @mock.patch("hud.cli.push.settings")
229
+ @mock.patch("hud.settings.settings")
230
230
  @mock.patch("hud.cli.push.HUDDesign")
231
231
  def test_push_with_tag(self, mock_design_class, mock_settings, mock_run, mock_popen, tmp_path):
232
232
  """Test pushing with explicit tag."""
@@ -282,7 +282,7 @@ class TestPushEnvironment:
282
282
  mock_process.returncode = 1
283
283
  mock_popen.return_value = mock_process
284
284
 
285
- with mock.patch("hud.cli.push.settings") as mock_settings:
285
+ with mock.patch("hud.settings.settings") as mock_settings:
286
286
  mock_settings.api_key = "test-key"
287
287
  with (
288
288
  mock.patch("subprocess.run"),
@@ -292,7 +292,7 @@ class TestPushEnvironment:
292
292
 
293
293
  @mock.patch("hud.cli.push.get_docker_image_labels")
294
294
  @mock.patch("subprocess.run")
295
- @mock.patch("hud.cli.push.settings")
295
+ @mock.patch("hud.settings.settings")
296
296
  @mock.patch("hud.cli.push.HUDDesign")
297
297
  def test_push_with_labels(
298
298
  self, mock_design_class, mock_settings, mock_run, mock_get_labels, tmp_path
@@ -74,7 +74,7 @@ class InteractiveMCPTester:
74
74
 
75
75
  for tool in self.tools:
76
76
  if "/" in tool.name:
77
- hub, name = tool.name.split("/", 1)
77
+ hub, _ = tool.name.split("/", 1)
78
78
  if hub not in hub_tools:
79
79
  hub_tools[hub] = []
80
80
  hub_tools[hub].append(tool)
hud/clients/__init__.py CHANGED
@@ -4,9 +4,10 @@ from __future__ import annotations
4
4
 
5
5
  from .base import AgentMCPClient, BaseHUDClient
6
6
  from .fastmcp import FastMCPHUDClient
7
+ from .mcp_use import MCPUseHUDClient
7
8
 
8
- # Default to FastMCP for new features
9
- MCPClient = FastMCPHUDClient
9
+ # Default to MCP-use for new features
10
+ MCPClient = MCPUseHUDClient
10
11
 
11
12
  __all__ = [
12
13
  "AgentMCPClient",
hud/clients/base.py CHANGED
@@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Any, Protocol, overload, runtime_checkable
9
9
 
10
10
  from mcp.types import Implementation
11
11
 
12
+ from hud.shared.exceptions import HudAuthenticationError, HudException
12
13
  from hud.types import MCPToolCall, MCPToolResult
13
14
  from hud.utils.mcp import setup_hud_telemetry
14
15
  from hud.version import __version__ as hud_version
@@ -120,8 +121,10 @@ class BaseHUDClient(AgentMCPClient):
120
121
 
121
122
  self._mcp_config = mcp_config or self._mcp_config
122
123
  if self._mcp_config is None:
123
- raise ValueError(
124
- "An MCP server configuration is required"
124
+ from hud.shared.exceptions import HudConfigError
125
+
126
+ raise HudConfigError(
127
+ "An MCP server configuration is required. "
125
128
  "Either pass it to the constructor or call initialize with a configuration"
126
129
  )
127
130
 
@@ -135,14 +138,18 @@ class BaseHUDClient(AgentMCPClient):
135
138
  url = server_config.get("url", "")
136
139
  headers = server_config.get("headers", {})
137
140
  if "mcp.hud.so" in url and len(headers.get("Authorization", "")) < 10:
138
- raise RuntimeError(
139
- "Please ensure your HUD_API_KEY environment variable is set correctly."
140
- "You can get an API key at https://app.hud.so"
141
+ raise HudAuthenticationError(
142
+ f'Sending authorization "{headers.get("Authorization", "")}", which may'
143
+ " be incomplete. Ensure HUD_API_KEY environment variable is set or send it"
144
+ " as a header. You can get an API key at https://app.hud.so"
141
145
  )
142
146
  # Subclasses implement connection
143
147
  await self._connect(self._mcp_config)
148
+ except HudException:
149
+ raise
144
150
  except Exception as e:
145
- raise e
151
+ # Auto-converts to appropriate HUD exception type with hints
152
+ raise HudException from e
146
153
 
147
154
  # Common hud behavior - fetch telemetry
148
155
  await self._fetch_telemetry()
@@ -168,7 +175,7 @@ class BaseHUDClient(AgentMCPClient):
168
175
  self._initialized = False
169
176
  logger.info("Client disconnected")
170
177
  else:
171
- logger.warning("Client is not running, cannot disconnect")
178
+ logger.debug("Client was not initialized, skipping disconnect")
172
179
 
173
180
  @overload
174
181
  async def call_tool(self, tool_call: MCPToolCall, /) -> MCPToolResult: ...
@@ -236,7 +243,9 @@ class BaseHUDClient(AgentMCPClient):
236
243
  def mcp_config(self) -> dict[str, dict[str, Any]]:
237
244
  """Get the MCP config."""
238
245
  if self._mcp_config is None:
239
- raise ValueError("Please initialize the client with a valid MCP config")
246
+ from hud.shared.exceptions import HudConfigError
247
+
248
+ raise HudConfigError("Please initialize the client with a valid MCP config")
240
249
  return self._mcp_config
241
250
 
242
251
  async def __aenter__(self: Any) -> Any:
@@ -305,7 +314,9 @@ class BaseHUDClient(AgentMCPClient):
305
314
  - metadata: Environment metadata
306
315
  """
307
316
  if not self._initialized:
308
- raise ValueError("Client must be initialized before analyzing the environment")
317
+ from hud.shared.exceptions import HudClientError
318
+
319
+ raise HudClientError("Client must be initialized before analyzing the environment")
309
320
 
310
321
  analysis: dict[str, Any] = {
311
322
  "tools": [],
hud/clients/mcp_use.py CHANGED
@@ -3,10 +3,12 @@
3
3
  from __future__ import annotations
4
4
 
5
5
  import logging
6
- from typing import TYPE_CHECKING, Any
6
+ from typing import Any
7
7
 
8
- from mcp import Implementation
8
+ from mcp import Implementation, types
9
9
  from mcp.shared.exceptions import McpError
10
+ from mcp_use.client import MCPClient as MCPUseClient
11
+ from mcp_use.session import MCPSession as MCPUseSession
10
12
  from pydantic import AnyUrl
11
13
 
12
14
  from hud.types import MCPToolCall, MCPToolResult
@@ -14,18 +16,6 @@ from hud.version import __version__ as hud_version
14
16
 
15
17
  from .base import BaseHUDClient
16
18
 
17
- if TYPE_CHECKING:
18
- from mcp import types
19
- from mcp_use.client import MCPClient as MCPUseClient # type: ignore[attr-defined]
20
- from mcp_use.session import MCPSession as MCPUseSession # type: ignore[attr-defined]
21
-
22
- try:
23
- from mcp_use.client import MCPClient as MCPUseClient # type: ignore[attr-defined]
24
- from mcp_use.session import MCPSession as MCPUseSession # type: ignore[attr-defined]
25
- except ImportError:
26
- MCPUseClient = None # type: ignore[misc, assignment]
27
- MCPUseSession = None # type: ignore[misc, assignment]
28
-
29
19
  logger = logging.getLogger(__name__)
30
20
 
31
21
 
@@ -53,7 +43,9 @@ class MCPUseHUDClient(BaseHUDClient):
53
43
  )
54
44
 
55
45
  self._sessions: dict[str, Any] = {} # Will be MCPUseSession when available
56
- self._tool_map: dict[str, tuple[str, types.Tool]] = {}
46
+ self._tool_map: dict[
47
+ str, tuple[str, types.Tool, types.Tool]
48
+ ] = {} # server_name, original_tool, prefixed_tool
57
49
  self._client: Any | None = None # Will be MCPUseClient when available
58
50
 
59
51
  async def _connect(self, mcp_config: dict[str, dict[str, Any]]) -> None:
@@ -106,14 +98,23 @@ class MCPUseHUDClient(BaseHUDClient):
106
98
  logger.info("Check that the MCP server is running and accessible")
107
99
  raise
108
100
 
101
+ # Populate tool map during initialization
102
+ await self.list_tools()
103
+
109
104
  async def list_tools(self) -> list[types.Tool]:
110
105
  """List all available tools from all sessions."""
111
106
  if self._client is None or not self._sessions:
112
107
  raise ValueError("Client is not connected, call initialize() first")
113
108
 
109
+ if self._tool_map:
110
+ return [tool[2] for tool in self._tool_map.values()]
111
+
114
112
  all_tools = []
115
113
  self._tool_map = {}
116
114
 
115
+ # Check if we need to prefix (more than one server)
116
+ use_prefix = len(self._sessions) > 1
117
+
117
118
  for server_name, session in self._sessions.items():
118
119
  try:
119
120
  # Ensure session is initialized
@@ -136,10 +137,26 @@ class MCPUseHUDClient(BaseHUDClient):
136
137
  [tool.name for tool in tools_result.tools],
137
138
  )
138
139
 
139
- # Add to collections
140
+ # Add to collections with optional prefix
140
141
  for tool in tools_result.tools:
141
- all_tools.append(tool)
142
- self._tool_map[tool.name] = (server_name, tool)
142
+ if use_prefix:
143
+ # Create a new tool with prefixed name
144
+ prefixed_name = f"{server_name}_{tool.name}"
145
+ # Create a new tool instance with prefixed name
146
+ from mcp import types as mcp_types
147
+
148
+ prefixed_tool = mcp_types.Tool(
149
+ name=prefixed_name,
150
+ description=tool.description,
151
+ inputSchema=tool.inputSchema,
152
+ )
153
+ all_tools.append(prefixed_tool)
154
+ # Map prefixed name to (server_name, original_tool)
155
+ self._tool_map[prefixed_name] = (server_name, tool, prefixed_tool)
156
+ else:
157
+ # Single server - no prefix needed
158
+ all_tools.append(tool)
159
+ self._tool_map[tool.name] = (server_name, tool, tool)
143
160
 
144
161
  # Log detailed tool info in verbose mode
145
162
  if self.verbose:
@@ -164,15 +181,20 @@ class MCPUseHUDClient(BaseHUDClient):
164
181
  raise ValueError("Client is not connected, call initialize() first")
165
182
 
166
183
  if tool_call.name not in self._tool_map:
167
- raise ValueError(f"Tool '{tool_call.name}' not found")
184
+ return MCPToolResult(
185
+ content=[types.TextContent(type="text", text=f"Tool '{tool_call.name}' not found")],
186
+ isError=True,
187
+ structuredContent=None,
188
+ )
168
189
 
169
- server_name, _ = self._tool_map[tool_call.name]
190
+ server_name, original_tool, _ = self._tool_map[tool_call.name]
170
191
  session = self._sessions[server_name]
171
192
 
172
193
  if self.verbose:
173
194
  logger.debug(
174
- "Calling tool '%s' on server '%s' with arguments: %s",
195
+ "Calling tool '%s' (original: '%s') on server '%s' with arguments: %s",
175
196
  tool_call.name,
197
+ original_tool.name,
176
198
  server_name,
177
199
  tool_call.arguments,
178
200
  )
@@ -181,7 +203,7 @@ class MCPUseHUDClient(BaseHUDClient):
181
203
  raise ValueError(f"Client session not initialized for {server_name}")
182
204
 
183
205
  result = await session.connector.client_session.call_tool(
184
- name=tool_call.name,
206
+ name=original_tool.name, # Use original tool name, not prefixed
185
207
  arguments=tool_call.arguments or {},
186
208
  )
187
209
 
hud/datasets/task.py CHANGED
@@ -51,7 +51,9 @@ class Task(BaseModel):
51
51
  try:
52
52
  return json.loads(v)
53
53
  except json.JSONDecodeError as e:
54
- raise ValueError(f"Invalid JSON string: {e}") from e
54
+ from hud.shared.exceptions import HudConfigError
55
+
56
+ raise HudConfigError(f"Invalid JSON string: {e}") from e
55
57
  return v
56
58
 
57
59
  @field_validator("setup_tool", "evaluate_tool", mode="before")
@@ -66,7 +68,9 @@ class Task(BaseModel):
66
68
  try:
67
69
  v = json.loads(v)
68
70
  except json.JSONDecodeError as e:
69
- raise ValueError(f"Invalid JSON string: {e}") from e
71
+ from hud.shared.exceptions import HudConfigError
72
+
73
+ raise HudConfigError(f"Invalid JSON string: {e}") from e
70
74
 
71
75
  if isinstance(v, dict):
72
76
  return MCPToolCall(**v)
hud/native/__init__.py ADDED
@@ -0,0 +1,6 @@
1
+ """Native Python MCP servers for HUD.
2
+
3
+ These servers run as pure Python processes without containerization.
4
+ They can be run standalone or mounted into other servers, providing
5
+ lightweight evaluation and comparison capabilities.
6
+ """