agentprobe-toolkit 1.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,275 @@
1
+ """Latency failure injection for agentic LLM systems.
2
+
3
+ Failure taxonomy — Latency Failures (Category 4):
4
+ - P99_SPIKE : Inject a single large latency spike (tail latency simulation)
5
+ - CASCADING_SLOW : Each successive call gets progressively slower
6
+ - COLD_START : First call has large startup delay, subsequent calls are normal
7
+ - JITTER : Add random jitter to every call (simulates network instability)
8
+
9
+ Unlike tool/orchestration failures, latency injectors do NOT raise exceptions.
10
+ They slow down execution to expose how agent frameworks handle slow responses,
11
+ impact downstream P99 latency, and affect user-perceived performance.
12
+
13
+ Usage::
14
+
15
+ injector = LatencyFailureInjector(
16
+ config=LatencyFailureConfig(
17
+ failure_type=LatencyFailureType.P99_SPIKE,
18
+ spike_delay_seconds=5.0,
19
+ spike_probability=0.01,
20
+ )
21
+ )
22
+
23
+ @injector.wrap
24
+ def call_llm(prompt: str) -> str:
25
+ return llm_api.complete(prompt)
26
+ """
27
+
28
+ from __future__ import annotations
29
+
30
+ import asyncio
31
+ import functools
32
+ import random
33
+ import time
34
+ from collections.abc import Callable
35
+ from dataclasses import dataclass
36
+ from enum import StrEnum
37
+ from typing import Any, TypeVar
38
+
39
+ from agentprobe.observer.tracer import get_tracer
40
+
41
+ T = TypeVar("T")
42
+
43
+ _tracer = get_tracer(__name__)
44
+
45
+
46
+ # ---------------------------------------------------------------------------
47
+ # Enums and config
48
+ # ---------------------------------------------------------------------------
49
+
50
+
51
+ class LatencyFailureType(StrEnum):
52
+ NONE = "none"
53
+ P99_SPIKE = "p99_spike"
54
+ CASCADING_SLOW = "cascading_slow"
55
+ COLD_START = "cold_start"
56
+ JITTER = "jitter"
57
+
58
+
59
+ @dataclass
60
+ class LatencyFailureConfig:
61
+ """Configuration for latency failure injection.
62
+
63
+ Attributes:
64
+ failure_type: Which latency pattern to inject.
65
+ spike_delay_seconds: Delay to inject for P99_SPIKE mode.
66
+ spike_probability: Probability of a spike on each call (P99_SPIKE).
67
+ Set to 0.01 to simulate ~P99 tail latency.
68
+ cascade_increment_seconds: Additional delay added per call (CASCADING_SLOW).
69
+ cascade_max_seconds: Maximum total delay cap (CASCADING_SLOW).
70
+ cold_start_delay_seconds: First-call delay (COLD_START).
71
+ jitter_min_seconds: Minimum jitter per call (JITTER).
72
+ jitter_max_seconds: Maximum jitter per call (JITTER).
73
+ seed: RNG seed for reproducible experiments.
74
+ """
75
+
76
+ failure_type: LatencyFailureType = LatencyFailureType.NONE
77
+ spike_delay_seconds: float = 5.0
78
+ spike_probability: float = 0.01
79
+ cascade_increment_seconds: float = 0.5
80
+ cascade_max_seconds: float = 10.0
81
+ cold_start_delay_seconds: float = 3.0
82
+ jitter_min_seconds: float = 0.1
83
+ jitter_max_seconds: float = 2.0
84
+ seed: int | None = None
85
+
86
+
87
+ # ---------------------------------------------------------------------------
88
+ # Latency record
89
+ # ---------------------------------------------------------------------------
90
+
91
+
92
+ @dataclass
93
+ class LatencyRecord:
94
+ call_index: int
95
+ base_latency_ms: float
96
+ injected_delay_ms: float
97
+ total_latency_ms: float
98
+ failure_type: LatencyFailureType
99
+ was_spike: bool = False
100
+ was_cold_start: bool = False
101
+
102
+
103
+ # ---------------------------------------------------------------------------
104
+ # Injector
105
+ # ---------------------------------------------------------------------------
106
+
107
+
108
+ class LatencyFailureInjector:
109
+ """Injects latency failures into agent function calls.
110
+
111
+ Does not raise exceptions — instead adds synchronous or async delays
112
+ to simulate real-world latency distributions including tail latency.
113
+ """
114
+
115
+ def __init__(self, config: LatencyFailureConfig | None = None) -> None:
116
+ self.config = config or LatencyFailureConfig()
117
+ self._rng = random.Random(self.config.seed)
118
+ self._log: list[LatencyRecord] = []
119
+ self._call_count = 0
120
+ self._cold_start_done = False
121
+ self._cascade_delay = 0.0
122
+
123
+ # ------------------------------------------------------------------
124
+ # Decorator API
125
+ # ------------------------------------------------------------------
126
+
127
+ def wrap(self, fn: Callable[..., T]) -> Callable[..., T]:
128
+ """Wrap a synchronous function with latency injection."""
129
+
130
+ @functools.wraps(fn)
131
+ def wrapper(*args: Any, **kwargs: Any) -> T:
132
+ return self._execute_sync(fn, *args, **kwargs) # type: ignore[return-value]
133
+
134
+ return wrapper # type: ignore[return-value]
135
+
136
+ def wrap_async(self, fn: Callable[..., T]) -> Callable[..., T]:
137
+ """Wrap an async function with latency injection."""
138
+
139
+ @functools.wraps(fn)
140
+ async def wrapper(*args: Any, **kwargs: Any) -> T:
141
+ return await self._execute_async(fn, *args, **kwargs) # type: ignore[return-value]
142
+
143
+ return wrapper # type: ignore[return-value]
144
+
145
+ # ------------------------------------------------------------------
146
+ # Observation
147
+ # ------------------------------------------------------------------
148
+
149
+ @property
150
+ def latency_log(self) -> list[LatencyRecord]:
151
+ return list(self._log)
152
+
153
+ def reset(self) -> None:
154
+ self._log.clear()
155
+ self._call_count = 0
156
+ self._cold_start_done = False
157
+ self._cascade_delay = 0.0
158
+ self._rng = random.Random(self.config.seed)
159
+
160
+ def summary(self) -> dict[str, Any]:
161
+ if not self._log:
162
+ return {"total_calls": 0}
163
+ latencies = [r.total_latency_ms for r in self._log]
164
+ sorted_lat = sorted(latencies)
165
+ n = len(sorted_lat)
166
+ return {
167
+ "total_calls": n,
168
+ "mean_latency_ms": sum(sorted_lat) / n,
169
+ "p50_latency_ms": sorted_lat[int(n * 0.50)],
170
+ "p90_latency_ms": sorted_lat[int(n * 0.90)],
171
+ "p99_latency_ms": sorted_lat[min(int(n * 0.99), n - 1)],
172
+ "max_latency_ms": sorted_lat[-1],
173
+ "spikes": sum(1 for r in self._log if r.was_spike),
174
+ }
175
+
176
+ # ------------------------------------------------------------------
177
+ # Internal
178
+ # ------------------------------------------------------------------
179
+
180
+ def _compute_delay(self) -> tuple[float, bool, bool]:
181
+ """Return (delay_seconds, was_spike, was_cold_start)."""
182
+ cfg = self.config
183
+ match cfg.failure_type:
184
+ case LatencyFailureType.NONE:
185
+ return 0.0, False, False
186
+
187
+ case LatencyFailureType.P99_SPIKE:
188
+ is_spike = self._rng.random() < cfg.spike_probability
189
+ return (cfg.spike_delay_seconds if is_spike else 0.0), is_spike, False
190
+
191
+ case LatencyFailureType.CASCADING_SLOW:
192
+ self._cascade_delay = min(
193
+ self._cascade_delay + cfg.cascade_increment_seconds,
194
+ cfg.cascade_max_seconds,
195
+ )
196
+ return self._cascade_delay, False, False
197
+
198
+ case LatencyFailureType.COLD_START:
199
+ if not self._cold_start_done:
200
+ self._cold_start_done = True
201
+ return cfg.cold_start_delay_seconds, False, True
202
+ return 0.0, False, False
203
+
204
+ case LatencyFailureType.JITTER:
205
+ jitter = self._rng.uniform(cfg.jitter_min_seconds, cfg.jitter_max_seconds)
206
+ return jitter, False, False
207
+
208
+ case _:
209
+ return 0.0, False, False
210
+
211
+ def _execute_sync(self, fn: Callable, *args: Any, **kwargs: Any) -> Any:
212
+ self._call_count += 1
213
+ delay, was_spike, was_cold = self._compute_delay()
214
+
215
+ with _tracer.start_as_current_span(
216
+ f"latency.call.{fn.__name__}",
217
+ attributes={
218
+ "latency.injected_delay_ms": delay * 1000,
219
+ "latency.failure_type": self.config.failure_type.value,
220
+ "latency.was_spike": was_spike,
221
+ "latency.was_cold_start": was_cold,
222
+ },
223
+ ):
224
+ start = time.perf_counter()
225
+ if delay > 0:
226
+ time.sleep(delay)
227
+ fn_start = time.perf_counter()
228
+ try:
229
+ return fn(*args, **kwargs)
230
+ finally:
231
+ end = time.perf_counter()
232
+ self._log.append(
233
+ LatencyRecord(
234
+ call_index=self._call_count,
235
+ base_latency_ms=(end - fn_start) * 1000,
236
+ injected_delay_ms=delay * 1000,
237
+ total_latency_ms=(end - start) * 1000,
238
+ failure_type=self.config.failure_type,
239
+ was_spike=was_spike,
240
+ was_cold_start=was_cold,
241
+ )
242
+ )
243
+
244
+ async def _execute_async(self, fn: Callable, *args: Any, **kwargs: Any) -> Any:
245
+ self._call_count += 1
246
+ delay, was_spike, was_cold = self._compute_delay()
247
+
248
+ with _tracer.start_as_current_span(
249
+ f"latency.call.{fn.__name__}",
250
+ attributes={
251
+ "latency.injected_delay_ms": delay * 1000,
252
+ "latency.failure_type": self.config.failure_type.value,
253
+ "latency.was_spike": was_spike,
254
+ "latency.was_cold_start": was_cold,
255
+ },
256
+ ):
257
+ start = time.perf_counter()
258
+ if delay > 0:
259
+ await asyncio.sleep(delay)
260
+ fn_start = time.perf_counter()
261
+ try:
262
+ return await fn(*args, **kwargs)
263
+ finally:
264
+ end = time.perf_counter()
265
+ self._log.append(
266
+ LatencyRecord(
267
+ call_index=self._call_count,
268
+ base_latency_ms=(end - fn_start) * 1000,
269
+ injected_delay_ms=delay * 1000,
270
+ total_latency_ms=(end - start) * 1000,
271
+ failure_type=self.config.failure_type,
272
+ was_spike=was_spike,
273
+ was_cold_start=was_cold,
274
+ )
275
+ )
@@ -0,0 +1,332 @@
1
+ """Orchestration failure injection for agentic LLM systems.
2
+
3
+ Failure taxonomy — Orchestration Failures (Category 2):
4
+ - INFINITE_LOOP : Agent cycles through nodes without terminating
5
+ - WRONG_BRANCH : Agent takes incorrect conditional branch
6
+ - STATE_CORRUPTION : Agent state dict is silently corrupted mid-execution
7
+ - MISSING_HANDOFF : Multi-agent handoff message is dropped
8
+
9
+ These injectors wrap LangGraph StateGraph nodes and edges to simulate
10
+ failures in agent control flow rather than individual tool calls.
11
+
12
+ Usage::
13
+
14
+ injector = OrchestrationFailureInjector(
15
+ config=OrchestrationFailureConfig(
16
+ failure_type=OrchestrationFailureType.INFINITE_LOOP,
17
+ max_loop_iterations=3,
18
+ )
19
+ )
20
+
21
+ # Wrap a LangGraph node function
22
+ safe_node = injector.wrap_node(my_node_fn, node_name="my_node")
23
+
24
+ # Or wrap a conditional edge router
25
+ faulty_router = injector.wrap_router(my_router_fn)
26
+ """
27
+
28
+ from __future__ import annotations
29
+
30
+ import functools
31
+ import random
32
+ import time
33
+ from collections.abc import Callable
34
+ from dataclasses import dataclass
35
+ from enum import StrEnum
36
+ from typing import Any, TypeVar
37
+
38
+ from opentelemetry.trace import Status, StatusCode
39
+
40
+ from agentprobe.observer.tracer import get_tracer
41
+
42
+ T = TypeVar("T")
43
+
44
+ _tracer = get_tracer(__name__)
45
+
46
+
47
+ # ---------------------------------------------------------------------------
48
+ # Exceptions
49
+ # ---------------------------------------------------------------------------
50
+
51
+
52
+ class OrchestrationError(Exception):
53
+ """Base class for injected orchestration errors."""
54
+
55
+
56
+ class InfiniteLoopError(OrchestrationError):
57
+ """Raised when an injected loop exceeds its iteration cap."""
58
+
59
+
60
+ class WrongBranchError(OrchestrationError):
61
+ """Raised to signal incorrect routing in a conditional edge."""
62
+
63
+
64
+ class StateCorruptionError(OrchestrationError):
65
+ """Raised when agent state has been silently corrupted."""
66
+
67
+
68
+ class MissingHandoffError(OrchestrationError):
69
+ """Raised when a multi-agent handoff message is dropped."""
70
+
71
+
72
+ # ---------------------------------------------------------------------------
73
+ # Enums and config
74
+ # ---------------------------------------------------------------------------
75
+
76
+
77
+ class OrchestrationFailureType(StrEnum):
78
+ NONE = "none"
79
+ INFINITE_LOOP = "infinite_loop"
80
+ WRONG_BRANCH = "wrong_branch"
81
+ STATE_CORRUPTION = "state_corruption"
82
+ MISSING_HANDOFF = "missing_handoff"
83
+
84
+
85
+ @dataclass
86
+ class OrchestrationFailureConfig:
87
+ """Configuration for orchestration failure injection.
88
+
89
+ Attributes:
90
+ failure_type: Which orchestration failure mode to inject.
91
+ failure_probability: Probability [0, 1] of injecting on each call.
92
+ max_loop_iterations: How many loop iterations before raising (INFINITE_LOOP).
93
+ corrupt_keys: State keys to corrupt (STATE_CORRUPTION).
94
+ If empty, a random key is selected.
95
+ wrong_branch_target: The incorrect branch label to return (WRONG_BRANCH).
96
+ If None, returns "__end__" prematurely.
97
+ seed: RNG seed for reproducible experiments.
98
+ """
99
+
100
+ failure_type: OrchestrationFailureType = OrchestrationFailureType.NONE
101
+ failure_probability: float = 1.0
102
+ max_loop_iterations: int = 5
103
+ corrupt_keys: list[str] | None = None
104
+ wrong_branch_target: str | None = None
105
+ seed: int | None = None
106
+
107
+ def __post_init__(self) -> None:
108
+ if not 0.0 <= self.failure_probability <= 1.0:
109
+ raise ValueError("failure_probability must be in [0, 1]")
110
+
111
+
112
+ # ---------------------------------------------------------------------------
113
+ # Injection result
114
+ # ---------------------------------------------------------------------------
115
+
116
+
117
+ @dataclass
118
+ class OrchestrationInjectionResult:
119
+ node_name: str
120
+ failure_type: OrchestrationFailureType
121
+ injected: bool
122
+ exception: Exception | None = None
123
+ latency_ms: float = 0.0
124
+ loop_iterations: int = 0
125
+ corrupted_keys: list[str] | None = None
126
+
127
+
128
+ # ---------------------------------------------------------------------------
129
+ # Injector
130
+ # ---------------------------------------------------------------------------
131
+
132
+
133
+ class OrchestrationFailureInjector:
134
+ """Injects orchestration-level failures into LangGraph agent graphs."""
135
+
136
+ def __init__(self, config: OrchestrationFailureConfig | None = None) -> None:
137
+ self.config = config or OrchestrationFailureConfig()
138
+ self._rng = random.Random(self.config.seed)
139
+ self._log: list[OrchestrationInjectionResult] = []
140
+ self._loop_counters: dict[str, int] = {}
141
+
142
+ # ------------------------------------------------------------------
143
+ # Node wrapping
144
+ # ------------------------------------------------------------------
145
+
146
+ def wrap_node(self, fn: Callable, node_name: str = "") -> Callable:
147
+ """Wrap a LangGraph node function with orchestration failure injection.
148
+
149
+ The wrapped function accepts the LangGraph state dict and returns
150
+ a (possibly corrupted) state update dict.
151
+ """
152
+ name = node_name or fn.__name__
153
+
154
+ @functools.wraps(fn)
155
+ def wrapper(state: dict[str, Any]) -> dict[str, Any]:
156
+ return self._execute_node(fn, name, state)
157
+
158
+ return wrapper
159
+
160
+ def wrap_router(self, fn: Callable, node_name: str = "") -> Callable:
161
+ """Wrap a LangGraph conditional edge router with wrong-branch injection."""
162
+ name = node_name or fn.__name__
163
+
164
+ @functools.wraps(fn)
165
+ def wrapper(state: dict[str, Any]) -> str:
166
+ return self._execute_router(fn, name, state)
167
+
168
+ return wrapper
169
+
170
+ # ------------------------------------------------------------------
171
+ # Observation
172
+ # ------------------------------------------------------------------
173
+
174
+ @property
175
+ def injection_log(self) -> list[OrchestrationInjectionResult]:
176
+ return list(self._log)
177
+
178
+ def reset(self) -> None:
179
+ self._log.clear()
180
+ self._loop_counters.clear()
181
+ self._rng = random.Random(self.config.seed)
182
+
183
+ def summary(self) -> dict[str, Any]:
184
+ total = len(self._log)
185
+ injected = sum(1 for r in self._log if r.injected)
186
+ return {
187
+ "total_node_calls": total,
188
+ "injected_failures": injected,
189
+ "injection_rate": injected / total if total > 0 else 0.0,
190
+ }
191
+
192
+ # ------------------------------------------------------------------
193
+ # Internal
194
+ # ------------------------------------------------------------------
195
+
196
+ def _should_inject(self) -> bool:
197
+ if self.config.failure_type == OrchestrationFailureType.NONE:
198
+ return False
199
+ return self._rng.random() < self.config.failure_probability
200
+
201
+ def _execute_node(self, fn: Callable, name: str, state: dict[str, Any]) -> dict[str, Any]:
202
+ inject = self._should_inject()
203
+ start = time.perf_counter()
204
+
205
+ with _tracer.start_as_current_span(
206
+ f"orchestration.node.{name}",
207
+ attributes={
208
+ "node.name": name,
209
+ "failure.injected": inject,
210
+ "failure.type": self.config.failure_type.value if inject else "none",
211
+ },
212
+ ) as span:
213
+ rec = OrchestrationInjectionResult(
214
+ node_name=name,
215
+ failure_type=self.config.failure_type if inject else OrchestrationFailureType.NONE,
216
+ injected=inject,
217
+ )
218
+ try:
219
+ if inject:
220
+ match self.config.failure_type:
221
+ case OrchestrationFailureType.INFINITE_LOOP:
222
+ return self._inject_loop(fn, name, state, span, rec)
223
+ case OrchestrationFailureType.STATE_CORRUPTION:
224
+ return self._inject_state_corruption(fn, name, state, span, rec)
225
+ case OrchestrationFailureType.MISSING_HANDOFF:
226
+ exc = MissingHandoffError(f"Handoff message dropped at node '{name}'")
227
+ span.set_status(Status(StatusCode.ERROR, str(exc)))
228
+ rec.exception = exc
229
+ raise exc
230
+ case _:
231
+ return fn(state)
232
+ return fn(state)
233
+ finally:
234
+ rec.latency_ms = (time.perf_counter() - start) * 1000
235
+ self._log.append(rec)
236
+
237
+ def _execute_router(self, fn: Callable, name: str, state: dict[str, Any]) -> str:
238
+ inject = self._should_inject()
239
+ start = time.perf_counter()
240
+
241
+ with _tracer.start_as_current_span(
242
+ f"orchestration.router.{name}",
243
+ attributes={
244
+ "router.name": name,
245
+ "failure.injected": inject,
246
+ "failure.type": self.config.failure_type.value if inject else "none",
247
+ },
248
+ ) as span:
249
+ rec = OrchestrationInjectionResult(
250
+ node_name=name,
251
+ failure_type=self.config.failure_type if inject else OrchestrationFailureType.NONE,
252
+ injected=inject,
253
+ )
254
+ try:
255
+ if inject and self.config.failure_type == OrchestrationFailureType.WRONG_BRANCH:
256
+ wrong = self.config.wrong_branch_target or "__end__"
257
+ span.set_attribute("router.injected_branch", wrong)
258
+ span.set_attribute("failure.injected", True)
259
+ return wrong
260
+ return fn(state)
261
+ finally:
262
+ rec.latency_ms = (time.perf_counter() - start) * 1000
263
+ self._log.append(rec)
264
+
265
+ def _inject_loop(
266
+ self,
267
+ fn: Callable,
268
+ name: str,
269
+ state: dict[str, Any],
270
+ span: Any,
271
+ rec: OrchestrationInjectionResult,
272
+ ) -> dict[str, Any]:
273
+ """Run the node repeatedly until max_loop_iterations, then raise."""
274
+ counter = self._loop_counters.get(name, 0) + 1
275
+ self._loop_counters[name] = counter
276
+ rec.loop_iterations = counter
277
+
278
+ if counter >= self.config.max_loop_iterations:
279
+ exc = InfiniteLoopError(
280
+ f"Node '{name}' detected as infinite loop after {counter} iterations"
281
+ )
282
+ span.set_status(Status(StatusCode.ERROR, str(exc)))
283
+ rec.exception = exc
284
+ self._loop_counters[name] = 0
285
+ raise exc
286
+
287
+ return fn(state)
288
+
289
+ def _inject_state_corruption(
290
+ self,
291
+ fn: Callable,
292
+ name: str,
293
+ state: dict[str, Any],
294
+ span: Any,
295
+ rec: OrchestrationInjectionResult,
296
+ ) -> dict[str, Any]:
297
+ """Run the node, then silently corrupt keys in the returned state."""
298
+ result = fn(state)
299
+ keys_to_corrupt = self.config.corrupt_keys or (list(result.keys()) or list(state.keys()))
300
+ if not keys_to_corrupt:
301
+ return result
302
+
303
+ # Pick one key to corrupt
304
+ target_key = self._rng.choice(keys_to_corrupt)
305
+ original = result.get(target_key, state.get(target_key))
306
+
307
+ corrupted_result = dict(result)
308
+ corrupted_result[target_key] = _corrupt_value(original)
309
+
310
+ rec.corrupted_keys = [target_key]
311
+ span.set_attribute("corruption.key", target_key)
312
+ return corrupted_result
313
+
314
+
315
+ # ---------------------------------------------------------------------------
316
+ # Helpers
317
+ # ---------------------------------------------------------------------------
318
+
319
+
320
+ def _corrupt_value(value: Any) -> Any:
321
+ """Return a type-preserving but semantically wrong value."""
322
+ if isinstance(value, str):
323
+ return value[::-1] if value else "CORRUPTED"
324
+ if isinstance(value, int):
325
+ return -value - 1
326
+ if isinstance(value, float):
327
+ return float("nan")
328
+ if isinstance(value, list):
329
+ return list(reversed(value))
330
+ if isinstance(value, dict):
331
+ return {k: None for k in value}
332
+ return None