hud-python 0.4.52__py3-none-any.whl → 0.4.54__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of hud-python might be problematic. Click here for more details.
- hud/agents/base.py +9 -2
- hud/agents/openai_chat_generic.py +15 -3
- hud/agents/tests/test_base.py +15 -0
- hud/agents/tests/test_base_runtime.py +164 -0
- hud/cli/__init__.py +20 -12
- hud/cli/build.py +35 -27
- hud/cli/dev.py +13 -31
- hud/cli/eval.py +85 -84
- hud/cli/tests/test_analyze_module.py +120 -0
- hud/cli/tests/test_build.py +24 -2
- hud/cli/tests/test_build_failure.py +41 -0
- hud/cli/tests/test_build_module.py +50 -0
- hud/cli/tests/test_cli_more_wrappers.py +30 -0
- hud/cli/tests/test_cli_root.py +134 -0
- hud/cli/tests/test_eval.py +6 -6
- hud/cli/tests/test_mcp_server.py +8 -7
- hud/cli/tests/test_push_happy.py +74 -0
- hud/cli/tests/test_push_wrapper.py +23 -0
- hud/cli/utils/docker.py +120 -1
- hud/cli/utils/runner.py +1 -1
- hud/cli/utils/tests/__init__.py +0 -0
- hud/cli/utils/tests/test_config.py +58 -0
- hud/cli/utils/tests/test_docker.py +93 -0
- hud/cli/utils/tests/test_docker_hints.py +71 -0
- hud/cli/utils/tests/test_env_check.py +74 -0
- hud/cli/utils/tests/test_environment.py +42 -0
- hud/cli/utils/tests/test_interactive_module.py +60 -0
- hud/cli/utils/tests/test_local_runner.py +50 -0
- hud/cli/utils/tests/test_logging_utils.py +23 -0
- hud/cli/utils/tests/test_metadata.py +49 -0
- hud/cli/utils/tests/test_package_runner.py +35 -0
- hud/cli/utils/tests/test_registry_utils.py +49 -0
- hud/cli/utils/tests/test_remote_runner.py +25 -0
- hud/cli/utils/tests/test_runner_modules.py +52 -0
- hud/cli/utils/tests/test_source_hash.py +36 -0
- hud/cli/utils/tests/test_tasks.py +80 -0
- hud/cli/utils/version_check.py +2 -2
- hud/datasets/tests/__init__.py +0 -0
- hud/datasets/tests/test_runner.py +106 -0
- hud/datasets/tests/test_utils.py +228 -0
- hud/otel/tests/__init__.py +0 -1
- hud/otel/tests/test_instrumentation.py +207 -0
- hud/server/tests/test_server_extra.py +2 -0
- hud/shared/exceptions.py +35 -4
- hud/shared/hints.py +25 -0
- hud/shared/requests.py +15 -3
- hud/shared/tests/test_exceptions.py +31 -23
- hud/shared/tests/test_hints.py +167 -0
- hud/telemetry/tests/test_async_context.py +242 -0
- hud/telemetry/tests/test_instrument.py +414 -0
- hud/telemetry/tests/test_job.py +609 -0
- hud/telemetry/tests/test_trace.py +183 -5
- hud/tools/computer/settings.py +2 -2
- hud/tools/tests/test_submit.py +85 -0
- hud/tools/tests/test_types.py +193 -0
- hud/types.py +17 -1
- hud/utils/agent_factories.py +1 -3
- hud/utils/mcp.py +1 -1
- hud/utils/tests/test_agent_factories.py +60 -0
- hud/utils/tests/test_mcp.py +4 -6
- hud/utils/tests/test_pretty_errors.py +186 -0
- hud/utils/tests/test_tasks.py +187 -0
- hud/utils/tests/test_tool_shorthand.py +154 -0
- hud/utils/tests/test_version.py +1 -1
- hud/version.py +1 -1
- {hud_python-0.4.52.dist-info → hud_python-0.4.54.dist-info}/METADATA +49 -49
- {hud_python-0.4.52.dist-info → hud_python-0.4.54.dist-info}/RECORD +70 -32
- {hud_python-0.4.52.dist-info → hud_python-0.4.54.dist-info}/WHEEL +0 -0
- {hud_python-0.4.52.dist-info → hud_python-0.4.54.dist-info}/entry_points.txt +0 -0
- {hud_python-0.4.52.dist-info → hud_python-0.4.54.dist-info}/licenses/LICENSE +0 -0
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from unittest.mock import AsyncMock, MagicMock, patch
|
|
4
|
+
|
|
5
|
+
import pytest
|
|
6
|
+
|
|
7
|
+
from hud.datasets.runner import _flush_telemetry
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@pytest.mark.asyncio
|
|
11
|
+
async def test_flush_telemetry():
|
|
12
|
+
"""Test _flush_telemetry function."""
|
|
13
|
+
with (
|
|
14
|
+
patch("hud.otel.config.is_telemetry_configured", return_value=True),
|
|
15
|
+
patch("hud.utils.hud_console.hud_console"),
|
|
16
|
+
patch("hud.utils.task_tracking.wait_all_tasks", new_callable=AsyncMock) as mock_wait,
|
|
17
|
+
patch("opentelemetry.trace.get_tracer_provider") as mock_get_provider,
|
|
18
|
+
):
|
|
19
|
+
from opentelemetry.sdk.trace import TracerProvider
|
|
20
|
+
|
|
21
|
+
mock_provider = MagicMock(spec=TracerProvider)
|
|
22
|
+
mock_provider.force_flush.return_value = True
|
|
23
|
+
mock_get_provider.return_value = mock_provider
|
|
24
|
+
|
|
25
|
+
mock_wait.return_value = 5
|
|
26
|
+
|
|
27
|
+
await _flush_telemetry()
|
|
28
|
+
|
|
29
|
+
mock_wait.assert_called_once()
|
|
30
|
+
mock_provider.force_flush.assert_called_once_with(timeout_millis=20000)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@pytest.mark.asyncio
|
|
34
|
+
async def test_flush_telemetry_no_telemetry():
|
|
35
|
+
"""Test _flush_telemetry when telemetry is not configured."""
|
|
36
|
+
with (
|
|
37
|
+
patch("hud.otel.config.is_telemetry_configured", return_value=False),
|
|
38
|
+
patch("hud.utils.hud_console.hud_console"),
|
|
39
|
+
patch("hud.utils.task_tracking.wait_all_tasks", new_callable=AsyncMock) as mock_wait,
|
|
40
|
+
patch("opentelemetry.trace.get_tracer_provider"),
|
|
41
|
+
):
|
|
42
|
+
mock_wait.return_value = 0
|
|
43
|
+
|
|
44
|
+
await _flush_telemetry()
|
|
45
|
+
|
|
46
|
+
mock_wait.assert_called_once()
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@pytest.mark.asyncio
|
|
50
|
+
async def test_flush_telemetry_exception():
|
|
51
|
+
"""Test _flush_telemetry handles exceptions gracefully."""
|
|
52
|
+
with (
|
|
53
|
+
patch("hud.otel.config.is_telemetry_configured", return_value=True),
|
|
54
|
+
patch("hud.utils.hud_console.hud_console"),
|
|
55
|
+
patch("hud.utils.task_tracking.wait_all_tasks", new_callable=AsyncMock) as mock_wait,
|
|
56
|
+
patch("opentelemetry.trace.get_tracer_provider") as mock_get_provider,
|
|
57
|
+
):
|
|
58
|
+
from opentelemetry.sdk.trace import TracerProvider
|
|
59
|
+
|
|
60
|
+
mock_provider = MagicMock(spec=TracerProvider)
|
|
61
|
+
mock_provider.force_flush.side_effect = Exception("Flush failed")
|
|
62
|
+
mock_get_provider.return_value = mock_provider
|
|
63
|
+
|
|
64
|
+
mock_wait.return_value = 3
|
|
65
|
+
|
|
66
|
+
# Should not raise
|
|
67
|
+
await _flush_telemetry()
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@pytest.mark.asyncio
|
|
71
|
+
async def test_flush_telemetry_no_completed_tasks():
|
|
72
|
+
"""Test _flush_telemetry when no tasks were completed."""
|
|
73
|
+
with (
|
|
74
|
+
patch("hud.otel.config.is_telemetry_configured", return_value=True),
|
|
75
|
+
patch("hud.utils.hud_console.hud_console"),
|
|
76
|
+
patch("hud.utils.task_tracking.wait_all_tasks", new_callable=AsyncMock) as mock_wait,
|
|
77
|
+
patch("opentelemetry.trace.get_tracer_provider") as mock_get_provider,
|
|
78
|
+
):
|
|
79
|
+
from opentelemetry.sdk.trace import TracerProvider
|
|
80
|
+
|
|
81
|
+
mock_provider = MagicMock(spec=TracerProvider)
|
|
82
|
+
mock_get_provider.return_value = mock_provider
|
|
83
|
+
|
|
84
|
+
mock_wait.return_value = 0
|
|
85
|
+
|
|
86
|
+
await _flush_telemetry()
|
|
87
|
+
|
|
88
|
+
mock_provider.force_flush.assert_called_once()
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
@pytest.mark.asyncio
|
|
92
|
+
async def test_flush_telemetry_non_sdk_provider():
|
|
93
|
+
"""Test _flush_telemetry with non-SDK TracerProvider."""
|
|
94
|
+
with (
|
|
95
|
+
patch("hud.otel.config.is_telemetry_configured", return_value=True),
|
|
96
|
+
patch("hud.utils.hud_console.hud_console"),
|
|
97
|
+
patch("hud.utils.task_tracking.wait_all_tasks", new_callable=AsyncMock) as mock_wait,
|
|
98
|
+
patch("opentelemetry.trace.get_tracer_provider") as mock_get_provider,
|
|
99
|
+
):
|
|
100
|
+
# Return a non-TracerProvider object
|
|
101
|
+
mock_get_provider.return_value = MagicMock(spec=object)
|
|
102
|
+
|
|
103
|
+
mock_wait.return_value = 2
|
|
104
|
+
|
|
105
|
+
# Should not raise
|
|
106
|
+
await _flush_telemetry()
|
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from unittest.mock import MagicMock, mock_open, patch
|
|
4
|
+
|
|
5
|
+
import pytest
|
|
6
|
+
|
|
7
|
+
from hud.datasets.utils import fetch_system_prompt_from_dataset, save_tasks
|
|
8
|
+
from hud.types import Task
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@pytest.mark.asyncio
|
|
12
|
+
async def test_fetch_system_prompt_success():
|
|
13
|
+
"""Test successful fetch of system prompt."""
|
|
14
|
+
with patch("huggingface_hub.hf_hub_download") as mock_download:
|
|
15
|
+
mock_download.return_value = "/tmp/system_prompt.txt"
|
|
16
|
+
with patch("builtins.open", mock_open(read_data="Test system prompt")):
|
|
17
|
+
result = await fetch_system_prompt_from_dataset("test/dataset")
|
|
18
|
+
assert result == "Test system prompt"
|
|
19
|
+
mock_download.assert_called_once()
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@pytest.mark.asyncio
|
|
23
|
+
async def test_fetch_system_prompt_empty_file():
|
|
24
|
+
"""Test fetch when file is empty."""
|
|
25
|
+
with patch("huggingface_hub.hf_hub_download") as mock_download:
|
|
26
|
+
mock_download.return_value = "/tmp/system_prompt.txt"
|
|
27
|
+
with patch("builtins.open", mock_open(read_data=" \n ")):
|
|
28
|
+
result = await fetch_system_prompt_from_dataset("test/dataset")
|
|
29
|
+
assert result is None
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@pytest.mark.asyncio
|
|
33
|
+
async def test_fetch_system_prompt_file_not_found():
|
|
34
|
+
"""Test fetch when file doesn't exist."""
|
|
35
|
+
with patch("huggingface_hub.hf_hub_download") as mock_download:
|
|
36
|
+
from huggingface_hub.errors import EntryNotFoundError
|
|
37
|
+
|
|
38
|
+
mock_download.side_effect = EntryNotFoundError("File not found")
|
|
39
|
+
result = await fetch_system_prompt_from_dataset("test/dataset")
|
|
40
|
+
assert result is None
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@pytest.mark.asyncio
|
|
44
|
+
async def test_fetch_system_prompt_import_error():
|
|
45
|
+
"""Test fetch when huggingface_hub is not installed."""
|
|
46
|
+
# Mock the import itself to raise ImportError
|
|
47
|
+
import sys
|
|
48
|
+
|
|
49
|
+
with patch.dict(sys.modules, {"huggingface_hub": None}):
|
|
50
|
+
result = await fetch_system_prompt_from_dataset("test/dataset")
|
|
51
|
+
assert result is None
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@pytest.mark.asyncio
|
|
55
|
+
async def test_fetch_system_prompt_general_exception():
|
|
56
|
+
"""Test fetch with general exception."""
|
|
57
|
+
with patch("huggingface_hub.hf_hub_download") as mock_download:
|
|
58
|
+
mock_download.side_effect = Exception("Network error")
|
|
59
|
+
result = await fetch_system_prompt_from_dataset("test/dataset")
|
|
60
|
+
assert result is None
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def test_save_tasks_basic():
|
|
64
|
+
"""Test basic save_tasks functionality."""
|
|
65
|
+
tasks = [
|
|
66
|
+
{"id": "1", "prompt": "test", "mcp_config": {"key": "value"}},
|
|
67
|
+
{"id": "2", "prompt": "test2", "mcp_config": {"key2": "value2"}},
|
|
68
|
+
]
|
|
69
|
+
|
|
70
|
+
with patch("hud.datasets.utils.Dataset") as mock_dataset_class:
|
|
71
|
+
mock_dataset = MagicMock()
|
|
72
|
+
mock_dataset_class.from_list.return_value = mock_dataset
|
|
73
|
+
|
|
74
|
+
save_tasks(tasks, "test/repo")
|
|
75
|
+
|
|
76
|
+
mock_dataset_class.from_list.assert_called_once()
|
|
77
|
+
call_args = mock_dataset_class.from_list.call_args[0][0]
|
|
78
|
+
assert len(call_args) == 2
|
|
79
|
+
# Check that mcp_config was JSON serialized
|
|
80
|
+
assert isinstance(call_args[0]["mcp_config"], str)
|
|
81
|
+
mock_dataset.push_to_hub.assert_called_once_with("test/repo")
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def test_save_tasks_with_specific_fields():
|
|
85
|
+
"""Test save_tasks with specific fields."""
|
|
86
|
+
tasks = [
|
|
87
|
+
{"id": "1", "prompt": "test", "mcp_config": {"key": "value"}, "extra": "data"},
|
|
88
|
+
]
|
|
89
|
+
|
|
90
|
+
with patch("hud.datasets.utils.Dataset") as mock_dataset_class:
|
|
91
|
+
mock_dataset = MagicMock()
|
|
92
|
+
mock_dataset_class.from_list.return_value = mock_dataset
|
|
93
|
+
|
|
94
|
+
save_tasks(tasks, "test/repo", fields=["id", "prompt"])
|
|
95
|
+
|
|
96
|
+
call_args = mock_dataset_class.from_list.call_args[0][0]
|
|
97
|
+
assert "id" in call_args[0]
|
|
98
|
+
assert "prompt" in call_args[0]
|
|
99
|
+
assert "extra" not in call_args[0]
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def test_save_tasks_with_list_field():
|
|
103
|
+
"""Test save_tasks serializes list fields."""
|
|
104
|
+
tasks = [
|
|
105
|
+
{"id": "1", "tags": ["tag1", "tag2"], "count": 5},
|
|
106
|
+
]
|
|
107
|
+
|
|
108
|
+
with patch("hud.datasets.utils.Dataset") as mock_dataset_class:
|
|
109
|
+
mock_dataset = MagicMock()
|
|
110
|
+
mock_dataset_class.from_list.return_value = mock_dataset
|
|
111
|
+
|
|
112
|
+
save_tasks(tasks, "test/repo")
|
|
113
|
+
|
|
114
|
+
call_args = mock_dataset_class.from_list.call_args[0][0]
|
|
115
|
+
# List should be JSON serialized
|
|
116
|
+
assert isinstance(call_args[0]["tags"], str)
|
|
117
|
+
assert '"tag1"' in call_args[0]["tags"]
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def test_save_tasks_with_primitive_types():
|
|
121
|
+
"""Test save_tasks handles various primitive types."""
|
|
122
|
+
tasks = [
|
|
123
|
+
{
|
|
124
|
+
"string": "text",
|
|
125
|
+
"integer": 42,
|
|
126
|
+
"float": 3.14,
|
|
127
|
+
"boolean": True,
|
|
128
|
+
"none": None,
|
|
129
|
+
},
|
|
130
|
+
]
|
|
131
|
+
|
|
132
|
+
with patch("hud.datasets.utils.Dataset") as mock_dataset_class:
|
|
133
|
+
mock_dataset = MagicMock()
|
|
134
|
+
mock_dataset_class.from_list.return_value = mock_dataset
|
|
135
|
+
|
|
136
|
+
save_tasks(tasks, "test/repo")
|
|
137
|
+
|
|
138
|
+
call_args = mock_dataset_class.from_list.call_args[0][0]
|
|
139
|
+
assert call_args[0]["string"] == "text"
|
|
140
|
+
assert call_args[0]["integer"] == 42
|
|
141
|
+
assert call_args[0]["float"] == 3.14
|
|
142
|
+
assert call_args[0]["boolean"] is True
|
|
143
|
+
assert call_args[0]["none"] == "" # None becomes empty string
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def test_save_tasks_with_other_type():
|
|
147
|
+
"""Test save_tasks converts other types to string."""
|
|
148
|
+
|
|
149
|
+
class CustomObj:
|
|
150
|
+
def __str__(self):
|
|
151
|
+
return "custom_value"
|
|
152
|
+
|
|
153
|
+
tasks = [
|
|
154
|
+
{"id": "1", "custom": CustomObj()},
|
|
155
|
+
]
|
|
156
|
+
|
|
157
|
+
with patch("hud.datasets.utils.Dataset") as mock_dataset_class:
|
|
158
|
+
mock_dataset = MagicMock()
|
|
159
|
+
mock_dataset_class.from_list.return_value = mock_dataset
|
|
160
|
+
|
|
161
|
+
save_tasks(tasks, "test/repo")
|
|
162
|
+
|
|
163
|
+
call_args = mock_dataset_class.from_list.call_args[0][0]
|
|
164
|
+
assert call_args[0]["custom"] == "custom_value"
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def test_save_tasks_rejects_task_objects():
|
|
168
|
+
"""Test save_tasks raises error for Task objects."""
|
|
169
|
+
task = Task(prompt="test", mcp_config={})
|
|
170
|
+
|
|
171
|
+
with pytest.raises(ValueError, match="expects dictionaries, not Task objects"):
|
|
172
|
+
save_tasks([task], "test/repo") # type: ignore
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def test_save_tasks_rejects_task_objects_in_list():
|
|
176
|
+
"""Test save_tasks raises error when Task object is in the list."""
|
|
177
|
+
tasks = [
|
|
178
|
+
{"id": "1", "prompt": "test", "mcp_config": {}},
|
|
179
|
+
Task(prompt="test2", mcp_config={}), # Task object
|
|
180
|
+
]
|
|
181
|
+
|
|
182
|
+
with pytest.raises(ValueError, match="Item 1 is a Task object"):
|
|
183
|
+
save_tasks(tasks, "test/repo") # type: ignore
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def test_save_tasks_with_kwargs():
|
|
187
|
+
"""Test save_tasks passes kwargs to push_to_hub."""
|
|
188
|
+
tasks = [{"id": "1", "prompt": "test"}]
|
|
189
|
+
|
|
190
|
+
with patch("hud.datasets.utils.Dataset") as mock_dataset_class:
|
|
191
|
+
mock_dataset = MagicMock()
|
|
192
|
+
mock_dataset_class.from_list.return_value = mock_dataset
|
|
193
|
+
|
|
194
|
+
save_tasks(tasks, "test/repo", private=True, commit_message="Test commit")
|
|
195
|
+
|
|
196
|
+
mock_dataset.push_to_hub.assert_called_once_with(
|
|
197
|
+
"test/repo", private=True, commit_message="Test commit"
|
|
198
|
+
)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def test_save_tasks_field_not_in_dict():
|
|
202
|
+
"""Test save_tasks handles missing fields gracefully."""
|
|
203
|
+
tasks = [
|
|
204
|
+
{"id": "1", "prompt": "test"},
|
|
205
|
+
]
|
|
206
|
+
|
|
207
|
+
with patch("hud.datasets.utils.Dataset") as mock_dataset_class:
|
|
208
|
+
mock_dataset = MagicMock()
|
|
209
|
+
mock_dataset_class.from_list.return_value = mock_dataset
|
|
210
|
+
|
|
211
|
+
# Request fields that don't exist
|
|
212
|
+
save_tasks(tasks, "test/repo", fields=["id", "missing_field"])
|
|
213
|
+
|
|
214
|
+
call_args = mock_dataset_class.from_list.call_args[0][0]
|
|
215
|
+
assert "id" in call_args[0]
|
|
216
|
+
assert "missing_field" not in call_args[0]
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def test_save_tasks_empty_list():
|
|
220
|
+
"""Test save_tasks with empty list."""
|
|
221
|
+
with patch("hud.datasets.utils.Dataset") as mock_dataset_class:
|
|
222
|
+
mock_dataset = MagicMock()
|
|
223
|
+
mock_dataset_class.from_list.return_value = mock_dataset
|
|
224
|
+
|
|
225
|
+
save_tasks([], "test/repo")
|
|
226
|
+
|
|
227
|
+
mock_dataset_class.from_list.assert_called_once_with([])
|
|
228
|
+
mock_dataset.push_to_hub.assert_called_once()
|
hud/otel/tests/__init__.py
CHANGED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
"""Tests for OpenTelemetry integration."""
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from unittest.mock import MagicMock, patch
|
|
4
|
+
|
|
5
|
+
import pytest
|
|
6
|
+
|
|
7
|
+
from hud.otel.instrumentation import (
|
|
8
|
+
_patch_get_error_type,
|
|
9
|
+
_patch_mcp_instrumentation,
|
|
10
|
+
install_mcp_instrumentation,
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def test_install_mcp_instrumentation_success():
|
|
15
|
+
"""Test successful installation of MCP instrumentation."""
|
|
16
|
+
mock_provider = MagicMock()
|
|
17
|
+
|
|
18
|
+
with (
|
|
19
|
+
patch("opentelemetry.instrumentation.mcp.instrumentation"),
|
|
20
|
+
patch(
|
|
21
|
+
"opentelemetry.instrumentation.mcp.instrumentation.McpInstrumentor"
|
|
22
|
+
) as mock_instrumentor_class,
|
|
23
|
+
patch("hud.otel.instrumentation._patch_mcp_instrumentation"),
|
|
24
|
+
):
|
|
25
|
+
mock_instrumentor = MagicMock()
|
|
26
|
+
mock_instrumentor_class.return_value = mock_instrumentor
|
|
27
|
+
|
|
28
|
+
install_mcp_instrumentation(mock_provider)
|
|
29
|
+
|
|
30
|
+
mock_instrumentor.instrument.assert_called_once_with(tracer_provider=mock_provider)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def test_install_mcp_instrumentation_import_error():
|
|
34
|
+
"""Test installation handles ImportError gracefully."""
|
|
35
|
+
mock_provider = MagicMock()
|
|
36
|
+
|
|
37
|
+
# Mock the import to raise ImportError
|
|
38
|
+
import sys
|
|
39
|
+
|
|
40
|
+
with patch.dict(sys.modules, {"opentelemetry.instrumentation.mcp.instrumentation": None}):
|
|
41
|
+
# Should not raise
|
|
42
|
+
install_mcp_instrumentation(mock_provider)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def test_install_mcp_instrumentation_general_exception():
|
|
46
|
+
"""Test installation handles general exceptions gracefully."""
|
|
47
|
+
mock_provider = MagicMock()
|
|
48
|
+
|
|
49
|
+
with (
|
|
50
|
+
patch("opentelemetry.instrumentation.mcp.instrumentation"),
|
|
51
|
+
patch(
|
|
52
|
+
"opentelemetry.instrumentation.mcp.instrumentation.McpInstrumentor"
|
|
53
|
+
) as mock_instrumentor_class,
|
|
54
|
+
):
|
|
55
|
+
mock_instrumentor_class.side_effect = Exception("Unexpected error")
|
|
56
|
+
|
|
57
|
+
# Should not raise
|
|
58
|
+
install_mcp_instrumentation(mock_provider)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def test_patch_mcp_instrumentation_success():
|
|
62
|
+
"""Test successful patching of MCP instrumentation."""
|
|
63
|
+
with (
|
|
64
|
+
patch("opentelemetry.instrumentation.mcp.instrumentation.McpInstrumentor") as mock_class,
|
|
65
|
+
patch("hud.otel.instrumentation._patch_get_error_type"),
|
|
66
|
+
):
|
|
67
|
+
mock_class._transport_wrapper = None
|
|
68
|
+
|
|
69
|
+
_patch_mcp_instrumentation()
|
|
70
|
+
|
|
71
|
+
# Should have set the _transport_wrapper
|
|
72
|
+
assert mock_class._transport_wrapper is not None
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def test_patch_mcp_instrumentation_exception():
|
|
76
|
+
"""Test patching handles exceptions gracefully."""
|
|
77
|
+
with patch(
|
|
78
|
+
"opentelemetry.instrumentation.mcp.instrumentation.McpInstrumentor",
|
|
79
|
+
side_effect=Exception("Error"),
|
|
80
|
+
):
|
|
81
|
+
# Should not raise
|
|
82
|
+
_patch_mcp_instrumentation()
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def test_patch_get_error_type_success():
|
|
86
|
+
"""Test successful patching of get_error_type."""
|
|
87
|
+
with patch("opentelemetry.instrumentation.mcp.instrumentation") as mock_mcp_inst:
|
|
88
|
+
mock_mcp_inst.get_error_type = None
|
|
89
|
+
|
|
90
|
+
_patch_get_error_type()
|
|
91
|
+
|
|
92
|
+
# Should have set get_error_type
|
|
93
|
+
assert mock_mcp_inst.get_error_type is not None
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def test_patch_get_error_type_exception():
|
|
97
|
+
"""Test patching get_error_type handles exceptions."""
|
|
98
|
+
with patch(
|
|
99
|
+
"opentelemetry.instrumentation.mcp.instrumentation", side_effect=ImportError("Not found")
|
|
100
|
+
):
|
|
101
|
+
# Should not raise
|
|
102
|
+
_patch_get_error_type()
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def test_patched_get_error_type_valid_4xx():
|
|
106
|
+
"""Test patched get_error_type with valid 4xx status code."""
|
|
107
|
+
with patch("opentelemetry.instrumentation.mcp.instrumentation") as mock_mcp_inst:
|
|
108
|
+
_patch_get_error_type()
|
|
109
|
+
|
|
110
|
+
patched_func = mock_mcp_inst.get_error_type
|
|
111
|
+
|
|
112
|
+
# Test with a valid 4xx error
|
|
113
|
+
result = patched_func("Error 404 not found")
|
|
114
|
+
assert result == "NOT_FOUND"
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def test_patched_get_error_type_valid_5xx():
|
|
118
|
+
"""Test patched get_error_type with valid 5xx status code."""
|
|
119
|
+
with patch("opentelemetry.instrumentation.mcp.instrumentation") as mock_mcp_inst:
|
|
120
|
+
_patch_get_error_type()
|
|
121
|
+
|
|
122
|
+
patched_func = mock_mcp_inst.get_error_type
|
|
123
|
+
|
|
124
|
+
# Test with a valid 5xx error
|
|
125
|
+
result = patched_func("Error 500 internal server error")
|
|
126
|
+
assert result == "INTERNAL_SERVER_ERROR"
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def test_patched_get_error_type_invalid_status():
|
|
130
|
+
"""Test patched get_error_type with invalid status code."""
|
|
131
|
+
with patch("opentelemetry.instrumentation.mcp.instrumentation") as mock_mcp_inst:
|
|
132
|
+
_patch_get_error_type()
|
|
133
|
+
|
|
134
|
+
patched_func = mock_mcp_inst.get_error_type
|
|
135
|
+
|
|
136
|
+
# Test with an invalid HTTP status code (e.g., 499 doesn't exist in HTTPStatus)
|
|
137
|
+
result = patched_func("Error 499 custom error")
|
|
138
|
+
# Should return the name even if it's not a standard HTTPStatus
|
|
139
|
+
assert result is None or isinstance(result, str)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def test_patched_get_error_type_no_status():
|
|
143
|
+
"""Test patched get_error_type with no status code."""
|
|
144
|
+
with patch("opentelemetry.instrumentation.mcp.instrumentation") as mock_mcp_inst:
|
|
145
|
+
_patch_get_error_type()
|
|
146
|
+
|
|
147
|
+
patched_func = mock_mcp_inst.get_error_type
|
|
148
|
+
|
|
149
|
+
result = patched_func("Error message without status code")
|
|
150
|
+
assert result is None
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def test_patched_get_error_type_non_string():
|
|
154
|
+
"""Test patched get_error_type with non-string input."""
|
|
155
|
+
with patch("opentelemetry.instrumentation.mcp.instrumentation") as mock_mcp_inst:
|
|
156
|
+
_patch_get_error_type()
|
|
157
|
+
|
|
158
|
+
patched_func = mock_mcp_inst.get_error_type
|
|
159
|
+
|
|
160
|
+
result = patched_func(None)
|
|
161
|
+
assert result is None
|
|
162
|
+
|
|
163
|
+
result = patched_func(123)
|
|
164
|
+
assert result is None
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def test_patched_get_error_type_3xx_ignored():
|
|
168
|
+
"""Test patched get_error_type ignores 3xx codes."""
|
|
169
|
+
with patch("opentelemetry.instrumentation.mcp.instrumentation") as mock_mcp_inst:
|
|
170
|
+
_patch_get_error_type()
|
|
171
|
+
|
|
172
|
+
patched_func = mock_mcp_inst.get_error_type
|
|
173
|
+
|
|
174
|
+
result = patched_func("Error 301 moved")
|
|
175
|
+
assert result is None
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
@pytest.mark.asyncio
|
|
179
|
+
async def test_transport_wrapper_three_values():
|
|
180
|
+
"""Test transport wrapper handles 3-value tuple."""
|
|
181
|
+
with (
|
|
182
|
+
patch("opentelemetry.instrumentation.mcp.instrumentation.McpInstrumentor") as mock_class,
|
|
183
|
+
patch("hud.otel.instrumentation._patch_get_error_type"),
|
|
184
|
+
):
|
|
185
|
+
mock_class._transport_wrapper = None
|
|
186
|
+
|
|
187
|
+
_patch_mcp_instrumentation()
|
|
188
|
+
|
|
189
|
+
# Get the patched wrapper
|
|
190
|
+
wrapper_func = mock_class._transport_wrapper
|
|
191
|
+
assert wrapper_func is not None
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
@pytest.mark.asyncio
|
|
195
|
+
async def test_transport_wrapper_two_values():
|
|
196
|
+
"""Test transport wrapper handles 2-value tuple."""
|
|
197
|
+
with (
|
|
198
|
+
patch("opentelemetry.instrumentation.mcp.instrumentation.McpInstrumentor") as mock_class,
|
|
199
|
+
patch("hud.otel.instrumentation._patch_get_error_type"),
|
|
200
|
+
):
|
|
201
|
+
mock_class._transport_wrapper = None
|
|
202
|
+
|
|
203
|
+
_patch_mcp_instrumentation()
|
|
204
|
+
|
|
205
|
+
# Get the patched wrapper
|
|
206
|
+
wrapper_func = mock_class._transport_wrapper
|
|
207
|
+
assert wrapper_func is not None
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
from __future__ import annotations
|
|
3
3
|
|
|
4
4
|
import asyncio
|
|
5
|
+
import sys
|
|
5
6
|
from contextlib import asynccontextmanager, suppress
|
|
6
7
|
|
|
7
8
|
import anyio
|
|
@@ -98,6 +99,7 @@ async def test_last_shutdown_handler_wins(patch_stdio):
|
|
|
98
99
|
server_mod._sigterm_received = False # type: ignore[attr-defined]
|
|
99
100
|
|
|
100
101
|
|
|
102
|
+
@pytest.mark.skipif(sys.platform == "win32", reason="asyncio.add_signal_handler is Unix-only")
|
|
101
103
|
def test__run_with_sigterm_registers_handlers_when_enabled(monkeypatch: pytest.MonkeyPatch):
|
|
102
104
|
"""
|
|
103
105
|
Verify that _run_with_sigterm attempts to register SIGTERM/SIGINT handlers
|
hud/shared/exceptions.py
CHANGED
|
@@ -185,11 +185,42 @@ class HudRequestError(HudException):
|
|
|
185
185
|
self.response_text = response_text
|
|
186
186
|
self.response_headers = response_headers
|
|
187
187
|
# Compute default hints from status code if none provided
|
|
188
|
-
if hints is None and status_code in (401, 403, 429):
|
|
188
|
+
if hints is None and status_code in (401, 402, 403, 429):
|
|
189
189
|
try:
|
|
190
|
-
from hud.shared.hints import
|
|
191
|
-
|
|
192
|
-
|
|
190
|
+
from hud.shared.hints import ( # type: ignore
|
|
191
|
+
CREDITS_EXHAUSTED,
|
|
192
|
+
HUD_API_KEY_MISSING,
|
|
193
|
+
PRO_PLAN_REQUIRED,
|
|
194
|
+
RATE_LIMIT_HIT,
|
|
195
|
+
)
|
|
196
|
+
|
|
197
|
+
if status_code == 402:
|
|
198
|
+
hints = [CREDITS_EXHAUSTED]
|
|
199
|
+
elif status_code == 403:
|
|
200
|
+
# Default 403 to auth unless the message clearly indicates Pro plan
|
|
201
|
+
combined_text = (message or "").lower()
|
|
202
|
+
try:
|
|
203
|
+
if response_text:
|
|
204
|
+
combined_text += "\n" + str(response_text).lower()
|
|
205
|
+
except Exception: # noqa: S110
|
|
206
|
+
pass
|
|
207
|
+
try:
|
|
208
|
+
if response_json and isinstance(response_json, dict):
|
|
209
|
+
detail = response_json.get("detail")
|
|
210
|
+
if isinstance(detail, str):
|
|
211
|
+
combined_text += "\n" + detail.lower()
|
|
212
|
+
except Exception: # noqa: S110
|
|
213
|
+
pass
|
|
214
|
+
|
|
215
|
+
mentions_pro = (
|
|
216
|
+
"pro plan" in combined_text
|
|
217
|
+
or "requires pro" in combined_text
|
|
218
|
+
or "pro mode" in combined_text
|
|
219
|
+
or combined_text.strip().startswith("pro ")
|
|
220
|
+
)
|
|
221
|
+
|
|
222
|
+
hints = [PRO_PLAN_REQUIRED] if mentions_pro else [HUD_API_KEY_MISSING]
|
|
223
|
+
elif status_code == 401:
|
|
193
224
|
hints = [HUD_API_KEY_MISSING]
|
|
194
225
|
elif status_code == 429:
|
|
195
226
|
hints = [RATE_LIMIT_HIT]
|
hud/shared/hints.py
CHANGED
|
@@ -61,6 +61,31 @@ RATE_LIMIT_HIT = Hint(
|
|
|
61
61
|
context=["network"],
|
|
62
62
|
)
|
|
63
63
|
|
|
64
|
+
# Billing / plan upgrade
|
|
65
|
+
PRO_PLAN_REQUIRED = Hint(
|
|
66
|
+
title="Pro plan required",
|
|
67
|
+
message="This feature requires Pro.",
|
|
68
|
+
tips=[
|
|
69
|
+
"Upgrade your plan to continue",
|
|
70
|
+
],
|
|
71
|
+
docs_url="https://hud.so/project/billing",
|
|
72
|
+
command_examples=None,
|
|
73
|
+
code="PRO_PLAN_REQUIRED",
|
|
74
|
+
context=["billing", "plan"],
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
CREDITS_EXHAUSTED = Hint(
|
|
78
|
+
title="Credits exhausted",
|
|
79
|
+
message="Your credits are exhausted.",
|
|
80
|
+
tips=[
|
|
81
|
+
"Top up credits or upgrade your plan",
|
|
82
|
+
],
|
|
83
|
+
docs_url="https://hud.so/project/billing",
|
|
84
|
+
command_examples=None,
|
|
85
|
+
code="CREDITS_EXHAUSTED",
|
|
86
|
+
context=["billing", "credits"],
|
|
87
|
+
)
|
|
88
|
+
|
|
64
89
|
TOOL_NOT_FOUND = Hint(
|
|
65
90
|
title="Tool not found",
|
|
66
91
|
message="Requested tool doesn't exist.",
|
hud/shared/requests.py
CHANGED
|
@@ -18,7 +18,11 @@ from hud.shared.exceptions import (
|
|
|
18
18
|
HudRequestError,
|
|
19
19
|
HudTimeoutError,
|
|
20
20
|
)
|
|
21
|
-
from hud.shared.hints import
|
|
21
|
+
from hud.shared.hints import (
|
|
22
|
+
CREDITS_EXHAUSTED,
|
|
23
|
+
HUD_API_KEY_MISSING,
|
|
24
|
+
RATE_LIMIT_HIT,
|
|
25
|
+
)
|
|
22
26
|
|
|
23
27
|
# Set up logger
|
|
24
28
|
logger = logging.getLogger("hud.http")
|
|
@@ -137,9 +141,13 @@ async def make_request(
|
|
|
137
141
|
raise HudTimeoutError(f"Request timed out: {e!s}") from None
|
|
138
142
|
except httpx.HTTPStatusError as e:
|
|
139
143
|
err = HudRequestError.from_httpx_error(e)
|
|
140
|
-
|
|
144
|
+
code = getattr(err, "status_code", None)
|
|
145
|
+
if code == 429 and RATE_LIMIT_HIT not in err.hints:
|
|
141
146
|
logger.debug("Attaching RATE_LIMIT hint to 429 error")
|
|
142
147
|
err.hints.append(RATE_LIMIT_HIT)
|
|
148
|
+
elif code == 402 and CREDITS_EXHAUSTED not in err.hints:
|
|
149
|
+
logger.debug("Attaching CREDITS_EXHAUSTED hint to 402 error")
|
|
150
|
+
err.hints.append(CREDITS_EXHAUSTED)
|
|
143
151
|
raise err from None
|
|
144
152
|
except httpx.RequestError as e:
|
|
145
153
|
if attempt <= max_retries:
|
|
@@ -234,9 +242,13 @@ def make_request_sync(
|
|
|
234
242
|
raise HudTimeoutError(f"Request timed out: {e!s}") from None
|
|
235
243
|
except httpx.HTTPStatusError as e:
|
|
236
244
|
err = HudRequestError.from_httpx_error(e)
|
|
237
|
-
|
|
245
|
+
code = getattr(err, "status_code", None)
|
|
246
|
+
if code == 429 and RATE_LIMIT_HIT not in err.hints:
|
|
238
247
|
logger.debug("Attaching RATE_LIMIT hint to 429 error")
|
|
239
248
|
err.hints.append(RATE_LIMIT_HIT)
|
|
249
|
+
elif code == 402 and CREDITS_EXHAUSTED not in err.hints:
|
|
250
|
+
logger.debug("Attaching CREDITS_EXHAUSTED hint to 402 error")
|
|
251
|
+
err.hints.append(CREDITS_EXHAUSTED)
|
|
240
252
|
raise err from None
|
|
241
253
|
except httpx.RequestError as e:
|
|
242
254
|
if attempt <= max_retries:
|