agentprobe-toolkit 1.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,404 @@
1
+ """Tool call failure injection for agentic LLM systems.
2
+
3
+ Failure taxonomy — Tool Call Failures (Category 1):
4
+ - TIMEOUT : Tool call exceeds wall-clock deadline
5
+ - MALFORMED_OUTPUT: Tool returns non-parseable or corrupt output
6
+ - RATE_LIMIT : Tool API returns HTTP 429 / rate-limit error
7
+ - API_ERROR : Tool API returns HTTP 5xx / generic server error
8
+
9
+ Usage — decorator::
10
+
11
+ injector = ToolFailureInjector(
12
+ config=ToolFailureConfig(
13
+ failure_type=ToolFailureType.TIMEOUT,
14
+ failure_probability=0.5,
15
+ timeout_seconds=5.0,
16
+ )
17
+ )
18
+
19
+ @injector.wrap
20
+ def web_search(query: str) -> str:
21
+ return _real_search(query)
22
+
23
+ Usage — explicit call::
24
+
25
+ result = injector.call(web_search, "climate change")
26
+
27
+ Usage — LangChain tool wrapping::
28
+
29
+ from langchain_core.tools import tool
30
+ from agentprobe.injector.tool_failure import inject_langchain_tool
31
+
32
+ @tool
33
+ def web_search(query: str) -> str:
34
+ \"\"\"Search the web.\"\"\"
35
+ return _real_search(query)
36
+
37
+ faulty_tool = inject_langchain_tool(web_search, injector)
38
+ """
39
+
40
+ from __future__ import annotations
41
+
42
+ import asyncio
43
+ import functools
44
+ import random
45
+ import time
46
+ from collections.abc import Callable
47
+ from dataclasses import dataclass, field
48
+ from enum import StrEnum
49
+ from typing import Any, TypeVar
50
+
51
+ from opentelemetry.trace import Status, StatusCode
52
+
53
+ from agentprobe.observer.tracer import get_tracer
54
+
55
+ T = TypeVar("T")
56
+
57
+ _tracer = get_tracer(__name__)
58
+
59
+
60
+ # ---------------------------------------------------------------------------
61
+ # Exception hierarchy
62
+ # ---------------------------------------------------------------------------
63
+
64
+
65
+ class ToolCallError(Exception):
66
+ """Base class for injected tool call errors."""
67
+
68
+
69
+ class ToolCallTimeout(ToolCallError):
70
+ """Raised when a tool call exceeds its timeout deadline."""
71
+
72
+
73
+ class ToolCallMalformedOutput(ToolCallError):
74
+ """Raised when a tool returns unparseable or corrupt output."""
75
+
76
+
77
+ class ToolCallRateLimited(ToolCallError):
78
+ """Raised to simulate HTTP 429 rate-limit responses."""
79
+
80
+ def __init__(self, retry_after: float = 60.0) -> None:
81
+ self.retry_after = retry_after
82
+ super().__init__(f"Rate limit exceeded. Retry-After: {retry_after}s (HTTP 429)")
83
+
84
+
85
+ class ToolCallAPIError(ToolCallError):
86
+ """Raised to simulate HTTP 5xx server-side API errors."""
87
+
88
+ def __init__(self, status_code: int = 500) -> None:
89
+ self.status_code = status_code
90
+ super().__init__(f"Tool API returned HTTP {status_code}")
91
+
92
+
93
+ # ---------------------------------------------------------------------------
94
+ # Enums and config
95
+ # ---------------------------------------------------------------------------
96
+
97
+
98
+ class ToolFailureType(StrEnum):
99
+ NONE = "none"
100
+ TIMEOUT = "timeout"
101
+ MALFORMED_OUTPUT = "malformed_output"
102
+ RATE_LIMIT = "rate_limit"
103
+ API_ERROR = "api_error"
104
+
105
+
106
+ @dataclass
107
+ class ToolFailureConfig:
108
+ """Configuration for tool failure injection.
109
+
110
+ Attributes:
111
+ failure_type: Which failure mode to inject.
112
+ failure_probability: Probability [0, 1] of injecting on each call.
113
+ timeout_seconds: Simulated wall-clock timeout (TIMEOUT mode).
114
+ malformed_payload: Custom bad payload string (MALFORMED_OUTPUT mode).
115
+ If None, a random malformed string is chosen.
116
+ retry_after_seconds: Retry-After value in seconds (RATE_LIMIT mode).
117
+ error_code: HTTP status code to simulate (API_ERROR mode).
118
+ seed: RNG seed for reproducible experiments.
119
+ """
120
+
121
+ failure_type: ToolFailureType = ToolFailureType.NONE
122
+ failure_probability: float = 1.0
123
+ timeout_seconds: float = 30.0
124
+ malformed_payload: str | None = None
125
+ retry_after_seconds: float = 60.0
126
+ error_code: int = 500
127
+ seed: int | None = None
128
+
129
+ def __post_init__(self) -> None:
130
+ if not 0.0 <= self.failure_probability <= 1.0:
131
+ raise ValueError("failure_probability must be in [0, 1]")
132
+
133
+
134
+ # ---------------------------------------------------------------------------
135
+ # Injection result
136
+ # ---------------------------------------------------------------------------
137
+
138
+
139
+ @dataclass
140
+ class InjectionResult:
141
+ """Record of a single tool call invocation under injection."""
142
+
143
+ tool_name: str
144
+ failure_type: ToolFailureType
145
+ injected: bool
146
+ exception: Exception | None = None
147
+ latency_ms: float = 0.0
148
+ span_id: str | None = None
149
+ call_index: int = 0
150
+ call_args: tuple = ()
151
+ call_kwargs: dict = field(default_factory=dict)
152
+ return_value: Any = None
153
+
154
+
155
+ # ---------------------------------------------------------------------------
156
+ # Injector
157
+ # ---------------------------------------------------------------------------
158
+
159
+
160
+ class ToolFailureInjector:
161
+ """Injects tool call failures into LangGraph / LangChain / AutoGen agents.
162
+
163
+ Thread-safe for read operations on :attr:`injection_log`; writes are
164
+ append-only and safe for single-threaded benchmark runs.
165
+ """
166
+
167
+ def __init__(self, config: ToolFailureConfig | None = None) -> None:
168
+ self.config = config or ToolFailureConfig()
169
+ self._rng = random.Random(self.config.seed)
170
+ self._log: list[InjectionResult] = []
171
+ self._call_count = 0
172
+
173
+ # ------------------------------------------------------------------
174
+ # Decorator API
175
+ # ------------------------------------------------------------------
176
+
177
+ def wrap(self, fn: Callable[..., T]) -> Callable[..., T]:
178
+ """Wrap a synchronous tool function with failure injection."""
179
+
180
+ @functools.wraps(fn)
181
+ def wrapper(*args: Any, **kwargs: Any) -> T:
182
+ return self._execute_sync(fn, fn.__name__, *args, **kwargs) # type: ignore[return-value]
183
+
184
+ return wrapper # type: ignore[return-value]
185
+
186
+ def wrap_async(self, fn: Callable[..., T]) -> Callable[..., T]:
187
+ """Wrap an async tool function with failure injection."""
188
+
189
+ @functools.wraps(fn)
190
+ async def wrapper(*args: Any, **kwargs: Any) -> T:
191
+ return await self._execute_async(fn, fn.__name__, *args, **kwargs) # type: ignore[return-value]
192
+
193
+ return wrapper # type: ignore[return-value]
194
+
195
+ def call(
196
+ self, fn: Callable[..., T], *args: Any, tool_name: str | None = None, **kwargs: Any
197
+ ) -> T:
198
+ """Directly invoke a function with failure injection (no decoration needed).
199
+
200
+ Args:
201
+ fn: The tool function to call.
202
+ *args: Positional arguments forwarded to *fn*.
203
+ tool_name: Override the recorded tool name (defaults to ``fn.__name__``).
204
+ **kwargs: Keyword arguments forwarded to *fn*.
205
+ """
206
+ return self._execute_sync(fn, tool_name or fn.__name__, *args, **kwargs) # type: ignore[return-value]
207
+
208
+ async def call_async(
209
+ self, fn: Callable[..., T], *args: Any, tool_name: str | None = None, **kwargs: Any
210
+ ) -> T:
211
+ """Directly invoke an async function with failure injection."""
212
+ return await self._execute_async(fn, tool_name or fn.__name__, *args, **kwargs) # type: ignore[return-value]
213
+
214
+ # ------------------------------------------------------------------
215
+ # LangChain integration
216
+ # ------------------------------------------------------------------
217
+
218
+ def inject_langchain_tool(self, tool: Any) -> Any:
219
+ """Wrap a LangChain BaseTool's _run/_arun with failure injection.
220
+
221
+ Returns the same tool object with monkey-patched run methods.
222
+ This preserves the tool's schema and metadata for the agent.
223
+ """
224
+ original_run = tool._run
225
+ original_arun = tool._arun
226
+
227
+ @functools.wraps(original_run)
228
+ def patched_run(*args: Any, **kwargs: Any) -> Any:
229
+ return self._execute_sync(original_run, tool.name, *args, **kwargs)
230
+
231
+ @functools.wraps(original_arun)
232
+ async def patched_arun(*args: Any, **kwargs: Any) -> Any:
233
+ return await self._execute_async(original_arun, tool.name, *args, **kwargs)
234
+
235
+ tool._run = patched_run
236
+ tool._arun = patched_arun
237
+ return tool
238
+
239
+ # ------------------------------------------------------------------
240
+ # Observation
241
+ # ------------------------------------------------------------------
242
+
243
+ @property
244
+ def injection_log(self) -> list[InjectionResult]:
245
+ return list(self._log)
246
+
247
+ def reset(self) -> None:
248
+ self._log.clear()
249
+ self._call_count = 0
250
+ self._rng = random.Random(self.config.seed)
251
+
252
+ def summary(self) -> dict[str, Any]:
253
+ """Aggregate statistics over all recorded calls."""
254
+ total = len(self._log)
255
+ injected = [r for r in self._log if r.injected]
256
+ latencies = [r.latency_ms for r in self._log]
257
+ by_type: dict[str, int] = {}
258
+ for r in injected:
259
+ key = r.failure_type.value
260
+ by_type[key] = by_type.get(key, 0) + 1
261
+
262
+ return {
263
+ "total_calls": total,
264
+ "injected_failures": len(injected),
265
+ "injection_rate": len(injected) / total if total > 0 else 0.0,
266
+ "failures_by_type": by_type,
267
+ "mean_latency_ms": sum(latencies) / len(latencies) if latencies else 0.0,
268
+ "max_latency_ms": max(latencies) if latencies else 0.0,
269
+ }
270
+
271
+ # ------------------------------------------------------------------
272
+ # Internal
273
+ # ------------------------------------------------------------------
274
+
275
+ def _should_inject(self) -> bool:
276
+ if self.config.failure_type == ToolFailureType.NONE:
277
+ return False
278
+ return self._rng.random() < self.config.failure_probability
279
+
280
+ def _execute_sync(self, fn: Callable, name: str, *args: Any, **kwargs: Any) -> Any:
281
+ self._call_count += 1
282
+ idx = self._call_count
283
+ inject = self._should_inject()
284
+ start = time.perf_counter()
285
+
286
+ with _tracer.start_as_current_span(
287
+ f"tool.call.{name}",
288
+ attributes={
289
+ "tool.name": name,
290
+ "tool.call_index": idx,
291
+ "failure.injected": inject,
292
+ "failure.type": self.config.failure_type.value if inject else "none",
293
+ },
294
+ ) as span:
295
+ rec = InjectionResult(
296
+ tool_name=name,
297
+ failure_type=self.config.failure_type if inject else ToolFailureType.NONE,
298
+ injected=inject,
299
+ call_index=idx,
300
+ span_id=format(span.get_span_context().span_id, "016x"),
301
+ call_args=args,
302
+ call_kwargs=dict(kwargs),
303
+ )
304
+ try:
305
+ if inject:
306
+ if self.config.failure_type == ToolFailureType.TIMEOUT:
307
+ time.sleep(self.config.timeout_seconds)
308
+ exc = self._build_exception()
309
+ span.set_status(Status(StatusCode.ERROR, str(exc)))
310
+ rec.exception = exc
311
+ raise exc
312
+ result = fn(*args, **kwargs)
313
+ rec.return_value = result
314
+ return result
315
+ except ToolCallError:
316
+ raise
317
+ except Exception as exc:
318
+ # Real (non-injected) error — record it
319
+ span.set_status(Status(StatusCode.ERROR, str(exc)))
320
+ rec.exception = exc
321
+ raise
322
+ finally:
323
+ rec.latency_ms = (time.perf_counter() - start) * 1000
324
+ self._log.append(rec)
325
+
326
+ async def _execute_async(self, fn: Callable, name: str, *args: Any, **kwargs: Any) -> Any:
327
+ self._call_count += 1
328
+ idx = self._call_count
329
+ inject = self._should_inject()
330
+ start = time.perf_counter()
331
+
332
+ with _tracer.start_as_current_span(
333
+ f"tool.call.{name}",
334
+ attributes={
335
+ "tool.name": name,
336
+ "tool.call_index": idx,
337
+ "failure.injected": inject,
338
+ "failure.type": self.config.failure_type.value if inject else "none",
339
+ },
340
+ ) as span:
341
+ rec = InjectionResult(
342
+ tool_name=name,
343
+ failure_type=self.config.failure_type if inject else ToolFailureType.NONE,
344
+ injected=inject,
345
+ call_index=idx,
346
+ span_id=format(span.get_span_context().span_id, "016x"),
347
+ call_args=args,
348
+ call_kwargs=dict(kwargs),
349
+ )
350
+ try:
351
+ if inject:
352
+ if self.config.failure_type == ToolFailureType.TIMEOUT:
353
+ # Simulate actual async hang before raising
354
+ await asyncio.sleep(self.config.timeout_seconds)
355
+ exc = self._build_exception()
356
+ span.set_status(Status(StatusCode.ERROR, str(exc)))
357
+ rec.exception = exc
358
+ raise exc
359
+ result = await fn(*args, **kwargs)
360
+ rec.return_value = result
361
+ return result
362
+ except ToolCallError:
363
+ raise
364
+ except Exception as exc:
365
+ span.set_status(Status(StatusCode.ERROR, str(exc)))
366
+ rec.exception = exc
367
+ raise
368
+ finally:
369
+ rec.latency_ms = (time.perf_counter() - start) * 1000
370
+ self._log.append(rec)
371
+
372
+ def _build_exception(self) -> ToolCallError:
373
+ match self.config.failure_type:
374
+ case ToolFailureType.TIMEOUT:
375
+ return ToolCallTimeout(f"Tool call timed out after {self.config.timeout_seconds}s")
376
+ case ToolFailureType.MALFORMED_OUTPUT:
377
+ payload = self.config.malformed_payload or _random_malformed_payload()
378
+ return ToolCallMalformedOutput(f"Tool returned unparseable output: {payload!r}")
379
+ case ToolFailureType.RATE_LIMIT:
380
+ return ToolCallRateLimited(retry_after=self.config.retry_after_seconds)
381
+ case ToolFailureType.API_ERROR:
382
+ return ToolCallAPIError(status_code=self.config.error_code)
383
+ case _:
384
+ return ToolCallError("Unknown tool failure type")
385
+
386
+
387
+ # ---------------------------------------------------------------------------
388
+ # Helpers
389
+ # ---------------------------------------------------------------------------
390
+
391
+
392
+ _MALFORMED_PAYLOADS = [
393
+ '{"result": ', # truncated JSON
394
+ "not json at all", # plain string
395
+ '{"result": null,}', # trailing comma (invalid JSON)
396
+ "\x00\x01\x02\xff", # binary garbage
397
+ "undefined", # JavaScript undefined leak
398
+ "<html>502 Bad Gateway</html>", # HTML error page instead of JSON
399
+ "", # empty response
400
+ ]
401
+
402
+
403
+ def _random_malformed_payload() -> str:
404
+ return random.choice(_MALFORMED_PAYLOADS)
@@ -0,0 +1,5 @@
1
+ """Observability modules for AgentProbe."""
2
+
3
+ from agentprobe.observer.tracer import AgentProbeTracer, get_tracer
4
+
5
+ __all__ = ["AgentProbeTracer", "get_tracer"]