hud-python 0.5.1__py3-none-any.whl → 0.5.13__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hud/__init__.py +1 -1
- hud/agents/__init__.py +65 -6
- hud/agents/base.py +33 -15
- hud/agents/claude.py +60 -31
- hud/agents/gateway.py +42 -0
- hud/agents/gemini.py +15 -26
- hud/agents/gemini_cua.py +6 -17
- hud/agents/misc/response_agent.py +7 -0
- hud/agents/openai.py +16 -29
- hud/agents/openai_chat.py +3 -19
- hud/agents/operator.py +5 -17
- hud/agents/resolver.py +70 -0
- hud/agents/tests/test_claude.py +2 -4
- hud/agents/tests/test_openai.py +2 -1
- hud/agents/tests/test_resolver.py +192 -0
- hud/agents/types.py +148 -0
- hud/cli/__init__.py +34 -3
- hud/cli/build.py +37 -5
- hud/cli/dev.py +11 -2
- hud/cli/eval.py +51 -39
- hud/cli/flows/init.py +1 -1
- hud/cli/pull.py +1 -1
- hud/cli/push.py +9 -2
- hud/cli/tests/test_build.py +2 -2
- hud/cli/tests/test_push.py +1 -1
- hud/cli/utils/metadata.py +1 -1
- hud/cli/utils/tests/test_metadata.py +1 -1
- hud/clients/mcp_use.py +6 -1
- hud/datasets/loader.py +17 -18
- hud/datasets/runner.py +16 -10
- hud/datasets/tests/test_loader.py +15 -15
- hud/environment/__init__.py +5 -3
- hud/environment/connection.py +58 -6
- hud/environment/connectors/mcp_config.py +29 -1
- hud/environment/environment.py +218 -77
- hud/environment/router.py +175 -24
- hud/environment/scenarios.py +313 -186
- hud/environment/tests/test_connectors.py +10 -23
- hud/environment/tests/test_environment.py +432 -0
- hud/environment/tests/test_local_connectors.py +81 -40
- hud/environment/tests/test_scenarios.py +820 -14
- hud/eval/context.py +63 -10
- hud/eval/instrument.py +4 -2
- hud/eval/manager.py +79 -12
- hud/eval/task.py +36 -4
- hud/eval/tests/test_eval.py +1 -1
- hud/eval/tests/test_task.py +147 -1
- hud/eval/types.py +2 -0
- hud/eval/utils.py +14 -3
- hud/patches/mcp_patches.py +178 -21
- hud/telemetry/instrument.py +8 -1
- hud/telemetry/tests/test_eval_telemetry.py +8 -8
- hud/tools/__init__.py +2 -0
- hud/tools/agent.py +223 -0
- hud/tools/computer/__init__.py +34 -5
- hud/tools/shell.py +3 -3
- hud/tools/tests/test_agent_tool.py +355 -0
- hud/types.py +62 -34
- hud/utils/hud_console.py +30 -17
- hud/utils/strict_schema.py +1 -1
- hud/utils/tests/test_version.py +1 -1
- hud/version.py +1 -1
- {hud_python-0.5.1.dist-info → hud_python-0.5.13.dist-info}/METADATA +2 -2
- {hud_python-0.5.1.dist-info → hud_python-0.5.13.dist-info}/RECORD +67 -61
- {hud_python-0.5.1.dist-info → hud_python-0.5.13.dist-info}/WHEEL +0 -0
- {hud_python-0.5.1.dist-info → hud_python-0.5.13.dist-info}/entry_points.txt +0 -0
- {hud_python-0.5.1.dist-info → hud_python-0.5.13.dist-info}/licenses/LICENSE +0 -0
hud/environment/router.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""
|
|
1
|
+
"""MCP routing for Environment - tools, prompts, and resources."""
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
@@ -12,7 +12,7 @@ if TYPE_CHECKING:
|
|
|
12
12
|
|
|
13
13
|
from hud.environment.connection import Connector
|
|
14
14
|
|
|
15
|
-
__all__ = ["LOCAL_CONNECTION", "ConflictResolution", "ToolRouter"]
|
|
15
|
+
__all__ = ["LOCAL_CONNECTION", "ConflictResolution", "MCPRouter", "ToolRouter"]
|
|
16
16
|
|
|
17
17
|
logger = logging.getLogger(__name__)
|
|
18
18
|
|
|
@@ -20,7 +20,7 @@ LOCAL_CONNECTION = "__local__"
|
|
|
20
20
|
|
|
21
21
|
|
|
22
22
|
class ConflictResolution(str, Enum):
|
|
23
|
-
"""Strategy for resolving
|
|
23
|
+
"""Strategy for resolving name conflicts."""
|
|
24
24
|
|
|
25
25
|
PREFIX = "prefix" # Add connection name as prefix
|
|
26
26
|
FIRST_WINS = "first_wins" # First connection wins
|
|
@@ -29,30 +29,89 @@ class ConflictResolution(str, Enum):
|
|
|
29
29
|
|
|
30
30
|
|
|
31
31
|
@dataclass
|
|
32
|
-
class
|
|
33
|
-
"""Routes
|
|
32
|
+
class MCPRouter:
|
|
33
|
+
"""Routes tools, prompts, and resources to local or remote handlers.
|
|
34
|
+
|
|
35
|
+
Builds routing tables during Environment.__aenter__ from local registrations
|
|
36
|
+
and connection caches. Provides get_*_connection() methods to find which
|
|
37
|
+
connection serves a given tool/prompt/resource.
|
|
38
|
+
"""
|
|
34
39
|
|
|
35
40
|
conflict_resolution: ConflictResolution = ConflictResolution.PREFIX
|
|
41
|
+
|
|
42
|
+
# Tool routing
|
|
36
43
|
_tools: list[mcp_types.Tool] = field(default_factory=list)
|
|
37
|
-
|
|
38
|
-
|
|
44
|
+
_tool_routing: dict[str, str] = field(default_factory=dict) # name -> connection
|
|
45
|
+
_local_tool_names: set[str] = field(default_factory=set)
|
|
46
|
+
|
|
47
|
+
# Prompt routing
|
|
48
|
+
_prompts: list[mcp_types.Prompt] = field(default_factory=list)
|
|
49
|
+
_prompt_routing: dict[str, str] = field(default_factory=dict) # name -> connection
|
|
50
|
+
|
|
51
|
+
# Resource routing
|
|
52
|
+
_resources: list[mcp_types.Resource] = field(default_factory=list)
|
|
53
|
+
_resource_routing: dict[str, str] = field(default_factory=dict) # uri -> connection
|
|
54
|
+
|
|
55
|
+
# =========================================================================
|
|
56
|
+
# Tool routing (backwards compatible)
|
|
57
|
+
# =========================================================================
|
|
39
58
|
|
|
40
59
|
@property
|
|
41
60
|
def tools(self) -> list[mcp_types.Tool]:
|
|
42
61
|
return self._tools
|
|
43
62
|
|
|
44
63
|
def is_local(self, name: str) -> bool:
|
|
45
|
-
|
|
64
|
+
"""Check if tool is local (backwards compat)."""
|
|
65
|
+
return name in self._local_tool_names
|
|
46
66
|
|
|
47
67
|
def get_connection(self, name: str) -> str | None:
|
|
68
|
+
"""Get connection name for tool, None if local or not found (backwards compat)."""
|
|
69
|
+
return self.get_tool_connection(name)
|
|
70
|
+
|
|
71
|
+
def get_tool_connection(self, name: str) -> str | None:
|
|
48
72
|
"""Get connection name for tool, None if local or not found."""
|
|
49
|
-
conn = self.
|
|
73
|
+
conn = self._tool_routing.get(name)
|
|
74
|
+
return None if conn == LOCAL_CONNECTION else conn
|
|
75
|
+
|
|
76
|
+
# =========================================================================
|
|
77
|
+
# Prompt routing
|
|
78
|
+
# =========================================================================
|
|
79
|
+
|
|
80
|
+
@property
|
|
81
|
+
def prompts(self) -> list[mcp_types.Prompt]:
|
|
82
|
+
return self._prompts
|
|
83
|
+
|
|
84
|
+
def get_prompt_connection(self, name: str) -> str | None:
|
|
85
|
+
"""Get connection name for prompt, None if local or not found."""
|
|
86
|
+
conn = self._prompt_routing.get(name)
|
|
87
|
+
return None if conn == LOCAL_CONNECTION else conn
|
|
88
|
+
|
|
89
|
+
# =========================================================================
|
|
90
|
+
# Resource routing
|
|
91
|
+
# =========================================================================
|
|
92
|
+
|
|
93
|
+
@property
|
|
94
|
+
def resources(self) -> list[mcp_types.Resource]:
|
|
95
|
+
return self._resources
|
|
96
|
+
|
|
97
|
+
def get_resource_connection(self, uri: str) -> str | None:
|
|
98
|
+
"""Get connection name for resource, None if local or not found."""
|
|
99
|
+
conn = self._resource_routing.get(uri)
|
|
50
100
|
return None if conn == LOCAL_CONNECTION else conn
|
|
51
101
|
|
|
102
|
+
# =========================================================================
|
|
103
|
+
# Building routes
|
|
104
|
+
# =========================================================================
|
|
105
|
+
|
|
52
106
|
def clear(self) -> None:
|
|
107
|
+
"""Clear all routing tables."""
|
|
53
108
|
self._tools.clear()
|
|
54
|
-
self.
|
|
55
|
-
self.
|
|
109
|
+
self._tool_routing.clear()
|
|
110
|
+
self._local_tool_names.clear()
|
|
111
|
+
self._prompts.clear()
|
|
112
|
+
self._prompt_routing.clear()
|
|
113
|
+
self._resources.clear()
|
|
114
|
+
self._resource_routing.clear()
|
|
56
115
|
|
|
57
116
|
def build(
|
|
58
117
|
self,
|
|
@@ -60,22 +119,24 @@ class ToolRouter:
|
|
|
60
119
|
connections: dict[str, Connector],
|
|
61
120
|
connection_order: list[str],
|
|
62
121
|
) -> None:
|
|
63
|
-
"""Build routing from local tools and connection caches.
|
|
122
|
+
"""Build tool routing from local tools and connection caches.
|
|
64
123
|
|
|
65
124
|
Local tools always have priority over remote tools.
|
|
66
125
|
Tools starting with '_' are internal and hidden from listing
|
|
67
126
|
(but still callable directly).
|
|
68
127
|
"""
|
|
69
|
-
|
|
128
|
+
# Clear tool routing only (prompts/resources built separately)
|
|
129
|
+
self._tools.clear()
|
|
130
|
+
self._tool_routing.clear()
|
|
131
|
+
self._local_tool_names.clear()
|
|
132
|
+
|
|
70
133
|
seen: dict[str, str] = {}
|
|
71
134
|
|
|
72
135
|
# Local tools first (always priority)
|
|
73
136
|
for tool in local_tools:
|
|
74
|
-
# Always add to routing (so tool is callable)
|
|
75
137
|
seen[tool.name] = LOCAL_CONNECTION
|
|
76
|
-
self.
|
|
77
|
-
self.
|
|
78
|
-
# Only add to visible list if not internal (underscore prefix)
|
|
138
|
+
self._tool_routing[tool.name] = LOCAL_CONNECTION
|
|
139
|
+
self._local_tool_names.add(tool.name)
|
|
79
140
|
if not tool.name.startswith("_"):
|
|
80
141
|
self._tools.append(tool)
|
|
81
142
|
|
|
@@ -88,25 +149,115 @@ class ToolRouter:
|
|
|
88
149
|
if name in seen:
|
|
89
150
|
existing = seen[name]
|
|
90
151
|
if existing == LOCAL_CONNECTION:
|
|
91
|
-
continue
|
|
152
|
+
continue
|
|
92
153
|
if not self._handle_conflict(name, existing, conn_name):
|
|
93
154
|
continue
|
|
94
155
|
self._tools = [t for t in self._tools if t.name != name]
|
|
95
156
|
|
|
96
|
-
# Always add to routing (so tool is callable)
|
|
97
157
|
seen[name] = conn_name
|
|
98
|
-
self.
|
|
99
|
-
# Only add to visible list if not internal (underscore prefix)
|
|
158
|
+
self._tool_routing[name] = conn_name
|
|
100
159
|
if not name.startswith("_"):
|
|
101
160
|
self._tools.append(tool)
|
|
102
161
|
|
|
103
|
-
logger.debug("Router: %d tools (%d local)", len(self._tools), len(self.
|
|
162
|
+
logger.debug("Router: %d tools (%d local)", len(self._tools), len(self._local_tool_names))
|
|
163
|
+
|
|
164
|
+
def build_prompts(
|
|
165
|
+
self,
|
|
166
|
+
local_prompts: list[mcp_types.Prompt],
|
|
167
|
+
connections: dict[str, Connector],
|
|
168
|
+
) -> None:
|
|
169
|
+
"""Build prompt routing from local prompts and connections.
|
|
170
|
+
|
|
171
|
+
Uses cached prompts from connections (populated during __aenter__).
|
|
172
|
+
"""
|
|
173
|
+
self._prompts.clear()
|
|
174
|
+
self._prompt_routing.clear()
|
|
175
|
+
|
|
176
|
+
seen: dict[str, str] = {}
|
|
177
|
+
|
|
178
|
+
# Local prompts first (always priority)
|
|
179
|
+
for prompt in local_prompts:
|
|
180
|
+
seen[prompt.name] = LOCAL_CONNECTION
|
|
181
|
+
self._prompt_routing[prompt.name] = LOCAL_CONNECTION
|
|
182
|
+
self._prompts.append(prompt)
|
|
183
|
+
|
|
184
|
+
# Use cached prompts from each connection (populated during __aenter__)
|
|
185
|
+
results: list[tuple[str, list[mcp_types.Prompt]]] = [
|
|
186
|
+
(conn_name, conn.cached_prompts) for conn_name, conn in connections.items()
|
|
187
|
+
]
|
|
188
|
+
|
|
189
|
+
# Process results in connection order (dict preserves insertion order)
|
|
190
|
+
for conn_name, remote_prompts in results:
|
|
191
|
+
for prompt in remote_prompts:
|
|
192
|
+
name = prompt.name
|
|
193
|
+
if name in seen:
|
|
194
|
+
existing = seen[name]
|
|
195
|
+
if existing == LOCAL_CONNECTION:
|
|
196
|
+
continue # Local always wins
|
|
197
|
+
if not self._handle_conflict(name, existing, conn_name):
|
|
198
|
+
continue
|
|
199
|
+
# Remove old prompt from list
|
|
200
|
+
self._prompts = [p for p in self._prompts if p.name != name]
|
|
201
|
+
|
|
202
|
+
seen[name] = conn_name
|
|
203
|
+
self._prompt_routing[name] = conn_name
|
|
204
|
+
self._prompts.append(prompt)
|
|
205
|
+
|
|
206
|
+
logger.debug("Router: %d prompts", len(self._prompts))
|
|
207
|
+
|
|
208
|
+
def build_resources(
|
|
209
|
+
self,
|
|
210
|
+
local_resources: list[mcp_types.Resource],
|
|
211
|
+
connections: dict[str, Connector],
|
|
212
|
+
) -> None:
|
|
213
|
+
"""Build resource routing from local resources and connections.
|
|
214
|
+
|
|
215
|
+
Uses cached resources from connections (populated during __aenter__).
|
|
216
|
+
"""
|
|
217
|
+
self._resources.clear()
|
|
218
|
+
self._resource_routing.clear()
|
|
219
|
+
|
|
220
|
+
seen: dict[str, str] = {}
|
|
221
|
+
|
|
222
|
+
# Local resources first (always priority)
|
|
223
|
+
for resource in local_resources:
|
|
224
|
+
uri = str(resource.uri)
|
|
225
|
+
seen[uri] = LOCAL_CONNECTION
|
|
226
|
+
self._resource_routing[uri] = LOCAL_CONNECTION
|
|
227
|
+
self._resources.append(resource)
|
|
228
|
+
|
|
229
|
+
# Use cached resources from each connection (populated during __aenter__)
|
|
230
|
+
results: list[tuple[str, list[mcp_types.Resource]]] = [
|
|
231
|
+
(conn_name, conn.cached_resources) for conn_name, conn in connections.items()
|
|
232
|
+
]
|
|
233
|
+
|
|
234
|
+
# Process results in connection order (dict preserves insertion order)
|
|
235
|
+
for conn_name, remote_resources in results:
|
|
236
|
+
for resource in remote_resources:
|
|
237
|
+
uri = str(resource.uri)
|
|
238
|
+
if uri in seen:
|
|
239
|
+
existing = seen[uri]
|
|
240
|
+
if existing == LOCAL_CONNECTION:
|
|
241
|
+
continue # Local always wins
|
|
242
|
+
if not self._handle_conflict(uri, existing, conn_name):
|
|
243
|
+
continue
|
|
244
|
+
# Remove old resource from list
|
|
245
|
+
self._resources = [r for r in self._resources if str(r.uri) != uri]
|
|
246
|
+
|
|
247
|
+
seen[uri] = conn_name
|
|
248
|
+
self._resource_routing[uri] = conn_name
|
|
249
|
+
self._resources.append(resource)
|
|
250
|
+
|
|
251
|
+
logger.debug("Router: %d resources", len(self._resources))
|
|
104
252
|
|
|
105
253
|
def _handle_conflict(self, name: str, existing: str, new: str) -> bool:
|
|
106
254
|
"""Handle remote-to-remote conflict. Returns True to replace existing."""
|
|
107
255
|
if self.conflict_resolution == ConflictResolution.ERROR:
|
|
108
|
-
raise ValueError(f"
|
|
256
|
+
raise ValueError(f"Conflict: '{name}' in '{existing}' and '{new}'")
|
|
109
257
|
if self.conflict_resolution == ConflictResolution.FIRST_WINS:
|
|
110
258
|
return False
|
|
111
|
-
# LAST_WINS returns True, PREFIX (shouldn't conflict) returns False
|
|
112
259
|
return self.conflict_resolution == ConflictResolution.LAST_WINS
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
# Backwards compatibility alias
|
|
263
|
+
ToolRouter = MCPRouter
|