agentprobe-toolkit 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentprobe/__init__.py +42 -0
- agentprobe/agents/__init__.py +5 -0
- agentprobe/agents/autogen_agent.py +308 -0
- agentprobe/agents/base.py +97 -0
- agentprobe/agents/langchain_agent.py +140 -0
- agentprobe/agents/langgraph_agent.py +203 -0
- agentprobe/benchmark/__init__.py +67 -0
- agentprobe/benchmark/grader.py +162 -0
- agentprobe/benchmark/metadata.py +214 -0
- agentprobe/benchmark/metrics.py +247 -0
- agentprobe/benchmark/runner.py +434 -0
- agentprobe/benchmark/statistics.py +459 -0
- agentprobe/benchmark/tasks/__init__.py +75 -0
- agentprobe/benchmark/tasks/multi_agent.py +191 -0
- agentprobe/benchmark/tasks/rag.py +309 -0
- agentprobe/benchmark/tasks/tool_use.py +228 -0
- agentprobe/config.py +67 -0
- agentprobe/injector/__init__.py +66 -0
- agentprobe/injector/context_failure.py +328 -0
- agentprobe/injector/latency_failure.py +275 -0
- agentprobe/injector/orchestration_failure.py +332 -0
- agentprobe/injector/tool_failure.py +404 -0
- agentprobe/observer/__init__.py +5 -0
- agentprobe/observer/classifier.py +320 -0
- agentprobe/observer/dashboard.py +216 -0
- agentprobe/observer/tracer.py +137 -0
- agentprobe_toolkit-1.2.0.dist-info/METADATA +405 -0
- agentprobe_toolkit-1.2.0.dist-info/RECORD +30 -0
- agentprobe_toolkit-1.2.0.dist-info/WHEEL +4 -0
- agentprobe_toolkit-1.2.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,404 @@
|
|
|
1
|
+
"""Tool call failure injection for agentic LLM systems.
|
|
2
|
+
|
|
3
|
+
Failure taxonomy — Tool Call Failures (Category 1):
|
|
4
|
+
- TIMEOUT : Tool call exceeds wall-clock deadline
|
|
5
|
+
- MALFORMED_OUTPUT: Tool returns non-parseable or corrupt output
|
|
6
|
+
- RATE_LIMIT : Tool API returns HTTP 429 / rate-limit error
|
|
7
|
+
- API_ERROR : Tool API returns HTTP 5xx / generic server error
|
|
8
|
+
|
|
9
|
+
Usage — decorator::
|
|
10
|
+
|
|
11
|
+
injector = ToolFailureInjector(
|
|
12
|
+
config=ToolFailureConfig(
|
|
13
|
+
failure_type=ToolFailureType.TIMEOUT,
|
|
14
|
+
failure_probability=0.5,
|
|
15
|
+
timeout_seconds=5.0,
|
|
16
|
+
)
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
@injector.wrap
|
|
20
|
+
def web_search(query: str) -> str:
|
|
21
|
+
return _real_search(query)
|
|
22
|
+
|
|
23
|
+
Usage — explicit call::
|
|
24
|
+
|
|
25
|
+
result = injector.call(web_search, "climate change")
|
|
26
|
+
|
|
27
|
+
Usage — LangChain tool wrapping::
|
|
28
|
+
|
|
29
|
+
from langchain_core.tools import tool
|
|
30
|
+
from agentprobe.injector.tool_failure import inject_langchain_tool
|
|
31
|
+
|
|
32
|
+
@tool
|
|
33
|
+
def web_search(query: str) -> str:
|
|
34
|
+
\"\"\"Search the web.\"\"\"
|
|
35
|
+
return _real_search(query)
|
|
36
|
+
|
|
37
|
+
faulty_tool = inject_langchain_tool(web_search, injector)
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
from __future__ import annotations
|
|
41
|
+
|
|
42
|
+
import asyncio
|
|
43
|
+
import functools
|
|
44
|
+
import random
|
|
45
|
+
import time
|
|
46
|
+
from collections.abc import Callable
|
|
47
|
+
from dataclasses import dataclass, field
|
|
48
|
+
from enum import StrEnum
|
|
49
|
+
from typing import Any, TypeVar
|
|
50
|
+
|
|
51
|
+
from opentelemetry.trace import Status, StatusCode
|
|
52
|
+
|
|
53
|
+
from agentprobe.observer.tracer import get_tracer
|
|
54
|
+
|
|
55
|
+
T = TypeVar("T")
|
|
56
|
+
|
|
57
|
+
_tracer = get_tracer(__name__)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
# ---------------------------------------------------------------------------
|
|
61
|
+
# Exception hierarchy
|
|
62
|
+
# ---------------------------------------------------------------------------
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class ToolCallError(Exception):
|
|
66
|
+
"""Base class for injected tool call errors."""
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class ToolCallTimeout(ToolCallError):
|
|
70
|
+
"""Raised when a tool call exceeds its timeout deadline."""
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class ToolCallMalformedOutput(ToolCallError):
|
|
74
|
+
"""Raised when a tool returns unparseable or corrupt output."""
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class ToolCallRateLimited(ToolCallError):
|
|
78
|
+
"""Raised to simulate HTTP 429 rate-limit responses."""
|
|
79
|
+
|
|
80
|
+
def __init__(self, retry_after: float = 60.0) -> None:
|
|
81
|
+
self.retry_after = retry_after
|
|
82
|
+
super().__init__(f"Rate limit exceeded. Retry-After: {retry_after}s (HTTP 429)")
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class ToolCallAPIError(ToolCallError):
|
|
86
|
+
"""Raised to simulate HTTP 5xx server-side API errors."""
|
|
87
|
+
|
|
88
|
+
def __init__(self, status_code: int = 500) -> None:
|
|
89
|
+
self.status_code = status_code
|
|
90
|
+
super().__init__(f"Tool API returned HTTP {status_code}")
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
# ---------------------------------------------------------------------------
|
|
94
|
+
# Enums and config
|
|
95
|
+
# ---------------------------------------------------------------------------
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
class ToolFailureType(StrEnum):
|
|
99
|
+
NONE = "none"
|
|
100
|
+
TIMEOUT = "timeout"
|
|
101
|
+
MALFORMED_OUTPUT = "malformed_output"
|
|
102
|
+
RATE_LIMIT = "rate_limit"
|
|
103
|
+
API_ERROR = "api_error"
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
@dataclass
|
|
107
|
+
class ToolFailureConfig:
|
|
108
|
+
"""Configuration for tool failure injection.
|
|
109
|
+
|
|
110
|
+
Attributes:
|
|
111
|
+
failure_type: Which failure mode to inject.
|
|
112
|
+
failure_probability: Probability [0, 1] of injecting on each call.
|
|
113
|
+
timeout_seconds: Simulated wall-clock timeout (TIMEOUT mode).
|
|
114
|
+
malformed_payload: Custom bad payload string (MALFORMED_OUTPUT mode).
|
|
115
|
+
If None, a random malformed string is chosen.
|
|
116
|
+
retry_after_seconds: Retry-After value in seconds (RATE_LIMIT mode).
|
|
117
|
+
error_code: HTTP status code to simulate (API_ERROR mode).
|
|
118
|
+
seed: RNG seed for reproducible experiments.
|
|
119
|
+
"""
|
|
120
|
+
|
|
121
|
+
failure_type: ToolFailureType = ToolFailureType.NONE
|
|
122
|
+
failure_probability: float = 1.0
|
|
123
|
+
timeout_seconds: float = 30.0
|
|
124
|
+
malformed_payload: str | None = None
|
|
125
|
+
retry_after_seconds: float = 60.0
|
|
126
|
+
error_code: int = 500
|
|
127
|
+
seed: int | None = None
|
|
128
|
+
|
|
129
|
+
def __post_init__(self) -> None:
|
|
130
|
+
if not 0.0 <= self.failure_probability <= 1.0:
|
|
131
|
+
raise ValueError("failure_probability must be in [0, 1]")
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
# ---------------------------------------------------------------------------
|
|
135
|
+
# Injection result
|
|
136
|
+
# ---------------------------------------------------------------------------
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
@dataclass
|
|
140
|
+
class InjectionResult:
|
|
141
|
+
"""Record of a single tool call invocation under injection."""
|
|
142
|
+
|
|
143
|
+
tool_name: str
|
|
144
|
+
failure_type: ToolFailureType
|
|
145
|
+
injected: bool
|
|
146
|
+
exception: Exception | None = None
|
|
147
|
+
latency_ms: float = 0.0
|
|
148
|
+
span_id: str | None = None
|
|
149
|
+
call_index: int = 0
|
|
150
|
+
call_args: tuple = ()
|
|
151
|
+
call_kwargs: dict = field(default_factory=dict)
|
|
152
|
+
return_value: Any = None
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
# ---------------------------------------------------------------------------
|
|
156
|
+
# Injector
|
|
157
|
+
# ---------------------------------------------------------------------------
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
class ToolFailureInjector:
|
|
161
|
+
"""Injects tool call failures into LangGraph / LangChain / AutoGen agents.
|
|
162
|
+
|
|
163
|
+
Thread-safe for read operations on :attr:`injection_log`; writes are
|
|
164
|
+
append-only and safe for single-threaded benchmark runs.
|
|
165
|
+
"""
|
|
166
|
+
|
|
167
|
+
def __init__(self, config: ToolFailureConfig | None = None) -> None:
|
|
168
|
+
self.config = config or ToolFailureConfig()
|
|
169
|
+
self._rng = random.Random(self.config.seed)
|
|
170
|
+
self._log: list[InjectionResult] = []
|
|
171
|
+
self._call_count = 0
|
|
172
|
+
|
|
173
|
+
# ------------------------------------------------------------------
|
|
174
|
+
# Decorator API
|
|
175
|
+
# ------------------------------------------------------------------
|
|
176
|
+
|
|
177
|
+
def wrap(self, fn: Callable[..., T]) -> Callable[..., T]:
|
|
178
|
+
"""Wrap a synchronous tool function with failure injection."""
|
|
179
|
+
|
|
180
|
+
@functools.wraps(fn)
|
|
181
|
+
def wrapper(*args: Any, **kwargs: Any) -> T:
|
|
182
|
+
return self._execute_sync(fn, fn.__name__, *args, **kwargs) # type: ignore[return-value]
|
|
183
|
+
|
|
184
|
+
return wrapper # type: ignore[return-value]
|
|
185
|
+
|
|
186
|
+
def wrap_async(self, fn: Callable[..., T]) -> Callable[..., T]:
|
|
187
|
+
"""Wrap an async tool function with failure injection."""
|
|
188
|
+
|
|
189
|
+
@functools.wraps(fn)
|
|
190
|
+
async def wrapper(*args: Any, **kwargs: Any) -> T:
|
|
191
|
+
return await self._execute_async(fn, fn.__name__, *args, **kwargs) # type: ignore[return-value]
|
|
192
|
+
|
|
193
|
+
return wrapper # type: ignore[return-value]
|
|
194
|
+
|
|
195
|
+
def call(
|
|
196
|
+
self, fn: Callable[..., T], *args: Any, tool_name: str | None = None, **kwargs: Any
|
|
197
|
+
) -> T:
|
|
198
|
+
"""Directly invoke a function with failure injection (no decoration needed).
|
|
199
|
+
|
|
200
|
+
Args:
|
|
201
|
+
fn: The tool function to call.
|
|
202
|
+
*args: Positional arguments forwarded to *fn*.
|
|
203
|
+
tool_name: Override the recorded tool name (defaults to ``fn.__name__``).
|
|
204
|
+
**kwargs: Keyword arguments forwarded to *fn*.
|
|
205
|
+
"""
|
|
206
|
+
return self._execute_sync(fn, tool_name or fn.__name__, *args, **kwargs) # type: ignore[return-value]
|
|
207
|
+
|
|
208
|
+
async def call_async(
|
|
209
|
+
self, fn: Callable[..., T], *args: Any, tool_name: str | None = None, **kwargs: Any
|
|
210
|
+
) -> T:
|
|
211
|
+
"""Directly invoke an async function with failure injection."""
|
|
212
|
+
return await self._execute_async(fn, tool_name or fn.__name__, *args, **kwargs) # type: ignore[return-value]
|
|
213
|
+
|
|
214
|
+
# ------------------------------------------------------------------
|
|
215
|
+
# LangChain integration
|
|
216
|
+
# ------------------------------------------------------------------
|
|
217
|
+
|
|
218
|
+
def inject_langchain_tool(self, tool: Any) -> Any:
|
|
219
|
+
"""Wrap a LangChain BaseTool's _run/_arun with failure injection.
|
|
220
|
+
|
|
221
|
+
Returns the same tool object with monkey-patched run methods.
|
|
222
|
+
This preserves the tool's schema and metadata for the agent.
|
|
223
|
+
"""
|
|
224
|
+
original_run = tool._run
|
|
225
|
+
original_arun = tool._arun
|
|
226
|
+
|
|
227
|
+
@functools.wraps(original_run)
|
|
228
|
+
def patched_run(*args: Any, **kwargs: Any) -> Any:
|
|
229
|
+
return self._execute_sync(original_run, tool.name, *args, **kwargs)
|
|
230
|
+
|
|
231
|
+
@functools.wraps(original_arun)
|
|
232
|
+
async def patched_arun(*args: Any, **kwargs: Any) -> Any:
|
|
233
|
+
return await self._execute_async(original_arun, tool.name, *args, **kwargs)
|
|
234
|
+
|
|
235
|
+
tool._run = patched_run
|
|
236
|
+
tool._arun = patched_arun
|
|
237
|
+
return tool
|
|
238
|
+
|
|
239
|
+
# ------------------------------------------------------------------
|
|
240
|
+
# Observation
|
|
241
|
+
# ------------------------------------------------------------------
|
|
242
|
+
|
|
243
|
+
@property
|
|
244
|
+
def injection_log(self) -> list[InjectionResult]:
|
|
245
|
+
return list(self._log)
|
|
246
|
+
|
|
247
|
+
def reset(self) -> None:
|
|
248
|
+
self._log.clear()
|
|
249
|
+
self._call_count = 0
|
|
250
|
+
self._rng = random.Random(self.config.seed)
|
|
251
|
+
|
|
252
|
+
def summary(self) -> dict[str, Any]:
|
|
253
|
+
"""Aggregate statistics over all recorded calls."""
|
|
254
|
+
total = len(self._log)
|
|
255
|
+
injected = [r for r in self._log if r.injected]
|
|
256
|
+
latencies = [r.latency_ms for r in self._log]
|
|
257
|
+
by_type: dict[str, int] = {}
|
|
258
|
+
for r in injected:
|
|
259
|
+
key = r.failure_type.value
|
|
260
|
+
by_type[key] = by_type.get(key, 0) + 1
|
|
261
|
+
|
|
262
|
+
return {
|
|
263
|
+
"total_calls": total,
|
|
264
|
+
"injected_failures": len(injected),
|
|
265
|
+
"injection_rate": len(injected) / total if total > 0 else 0.0,
|
|
266
|
+
"failures_by_type": by_type,
|
|
267
|
+
"mean_latency_ms": sum(latencies) / len(latencies) if latencies else 0.0,
|
|
268
|
+
"max_latency_ms": max(latencies) if latencies else 0.0,
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
# ------------------------------------------------------------------
|
|
272
|
+
# Internal
|
|
273
|
+
# ------------------------------------------------------------------
|
|
274
|
+
|
|
275
|
+
def _should_inject(self) -> bool:
|
|
276
|
+
if self.config.failure_type == ToolFailureType.NONE:
|
|
277
|
+
return False
|
|
278
|
+
return self._rng.random() < self.config.failure_probability
|
|
279
|
+
|
|
280
|
+
def _execute_sync(self, fn: Callable, name: str, *args: Any, **kwargs: Any) -> Any:
|
|
281
|
+
self._call_count += 1
|
|
282
|
+
idx = self._call_count
|
|
283
|
+
inject = self._should_inject()
|
|
284
|
+
start = time.perf_counter()
|
|
285
|
+
|
|
286
|
+
with _tracer.start_as_current_span(
|
|
287
|
+
f"tool.call.{name}",
|
|
288
|
+
attributes={
|
|
289
|
+
"tool.name": name,
|
|
290
|
+
"tool.call_index": idx,
|
|
291
|
+
"failure.injected": inject,
|
|
292
|
+
"failure.type": self.config.failure_type.value if inject else "none",
|
|
293
|
+
},
|
|
294
|
+
) as span:
|
|
295
|
+
rec = InjectionResult(
|
|
296
|
+
tool_name=name,
|
|
297
|
+
failure_type=self.config.failure_type if inject else ToolFailureType.NONE,
|
|
298
|
+
injected=inject,
|
|
299
|
+
call_index=idx,
|
|
300
|
+
span_id=format(span.get_span_context().span_id, "016x"),
|
|
301
|
+
call_args=args,
|
|
302
|
+
call_kwargs=dict(kwargs),
|
|
303
|
+
)
|
|
304
|
+
try:
|
|
305
|
+
if inject:
|
|
306
|
+
if self.config.failure_type == ToolFailureType.TIMEOUT:
|
|
307
|
+
time.sleep(self.config.timeout_seconds)
|
|
308
|
+
exc = self._build_exception()
|
|
309
|
+
span.set_status(Status(StatusCode.ERROR, str(exc)))
|
|
310
|
+
rec.exception = exc
|
|
311
|
+
raise exc
|
|
312
|
+
result = fn(*args, **kwargs)
|
|
313
|
+
rec.return_value = result
|
|
314
|
+
return result
|
|
315
|
+
except ToolCallError:
|
|
316
|
+
raise
|
|
317
|
+
except Exception as exc:
|
|
318
|
+
# Real (non-injected) error — record it
|
|
319
|
+
span.set_status(Status(StatusCode.ERROR, str(exc)))
|
|
320
|
+
rec.exception = exc
|
|
321
|
+
raise
|
|
322
|
+
finally:
|
|
323
|
+
rec.latency_ms = (time.perf_counter() - start) * 1000
|
|
324
|
+
self._log.append(rec)
|
|
325
|
+
|
|
326
|
+
async def _execute_async(self, fn: Callable, name: str, *args: Any, **kwargs: Any) -> Any:
|
|
327
|
+
self._call_count += 1
|
|
328
|
+
idx = self._call_count
|
|
329
|
+
inject = self._should_inject()
|
|
330
|
+
start = time.perf_counter()
|
|
331
|
+
|
|
332
|
+
with _tracer.start_as_current_span(
|
|
333
|
+
f"tool.call.{name}",
|
|
334
|
+
attributes={
|
|
335
|
+
"tool.name": name,
|
|
336
|
+
"tool.call_index": idx,
|
|
337
|
+
"failure.injected": inject,
|
|
338
|
+
"failure.type": self.config.failure_type.value if inject else "none",
|
|
339
|
+
},
|
|
340
|
+
) as span:
|
|
341
|
+
rec = InjectionResult(
|
|
342
|
+
tool_name=name,
|
|
343
|
+
failure_type=self.config.failure_type if inject else ToolFailureType.NONE,
|
|
344
|
+
injected=inject,
|
|
345
|
+
call_index=idx,
|
|
346
|
+
span_id=format(span.get_span_context().span_id, "016x"),
|
|
347
|
+
call_args=args,
|
|
348
|
+
call_kwargs=dict(kwargs),
|
|
349
|
+
)
|
|
350
|
+
try:
|
|
351
|
+
if inject:
|
|
352
|
+
if self.config.failure_type == ToolFailureType.TIMEOUT:
|
|
353
|
+
# Simulate actual async hang before raising
|
|
354
|
+
await asyncio.sleep(self.config.timeout_seconds)
|
|
355
|
+
exc = self._build_exception()
|
|
356
|
+
span.set_status(Status(StatusCode.ERROR, str(exc)))
|
|
357
|
+
rec.exception = exc
|
|
358
|
+
raise exc
|
|
359
|
+
result = await fn(*args, **kwargs)
|
|
360
|
+
rec.return_value = result
|
|
361
|
+
return result
|
|
362
|
+
except ToolCallError:
|
|
363
|
+
raise
|
|
364
|
+
except Exception as exc:
|
|
365
|
+
span.set_status(Status(StatusCode.ERROR, str(exc)))
|
|
366
|
+
rec.exception = exc
|
|
367
|
+
raise
|
|
368
|
+
finally:
|
|
369
|
+
rec.latency_ms = (time.perf_counter() - start) * 1000
|
|
370
|
+
self._log.append(rec)
|
|
371
|
+
|
|
372
|
+
def _build_exception(self) -> ToolCallError:
|
|
373
|
+
match self.config.failure_type:
|
|
374
|
+
case ToolFailureType.TIMEOUT:
|
|
375
|
+
return ToolCallTimeout(f"Tool call timed out after {self.config.timeout_seconds}s")
|
|
376
|
+
case ToolFailureType.MALFORMED_OUTPUT:
|
|
377
|
+
payload = self.config.malformed_payload or _random_malformed_payload()
|
|
378
|
+
return ToolCallMalformedOutput(f"Tool returned unparseable output: {payload!r}")
|
|
379
|
+
case ToolFailureType.RATE_LIMIT:
|
|
380
|
+
return ToolCallRateLimited(retry_after=self.config.retry_after_seconds)
|
|
381
|
+
case ToolFailureType.API_ERROR:
|
|
382
|
+
return ToolCallAPIError(status_code=self.config.error_code)
|
|
383
|
+
case _:
|
|
384
|
+
return ToolCallError("Unknown tool failure type")
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
# ---------------------------------------------------------------------------
|
|
388
|
+
# Helpers
|
|
389
|
+
# ---------------------------------------------------------------------------
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
_MALFORMED_PAYLOADS = [
|
|
393
|
+
'{"result": ', # truncated JSON
|
|
394
|
+
"not json at all", # plain string
|
|
395
|
+
'{"result": null,}', # trailing comma (invalid JSON)
|
|
396
|
+
"\x00\x01\x02\xff", # binary garbage
|
|
397
|
+
"undefined", # JavaScript undefined leak
|
|
398
|
+
"<html>502 Bad Gateway</html>", # HTML error page instead of JSON
|
|
399
|
+
"", # empty response
|
|
400
|
+
]
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
def _random_malformed_payload() -> str:
|
|
404
|
+
return random.choice(_MALFORMED_PAYLOADS)
|