agentprobe-toolkit 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentprobe/__init__.py +42 -0
- agentprobe/agents/__init__.py +5 -0
- agentprobe/agents/autogen_agent.py +308 -0
- agentprobe/agents/base.py +97 -0
- agentprobe/agents/langchain_agent.py +140 -0
- agentprobe/agents/langgraph_agent.py +203 -0
- agentprobe/benchmark/__init__.py +67 -0
- agentprobe/benchmark/grader.py +162 -0
- agentprobe/benchmark/metadata.py +214 -0
- agentprobe/benchmark/metrics.py +247 -0
- agentprobe/benchmark/runner.py +434 -0
- agentprobe/benchmark/statistics.py +459 -0
- agentprobe/benchmark/tasks/__init__.py +75 -0
- agentprobe/benchmark/tasks/multi_agent.py +191 -0
- agentprobe/benchmark/tasks/rag.py +309 -0
- agentprobe/benchmark/tasks/tool_use.py +228 -0
- agentprobe/config.py +67 -0
- agentprobe/injector/__init__.py +66 -0
- agentprobe/injector/context_failure.py +328 -0
- agentprobe/injector/latency_failure.py +275 -0
- agentprobe/injector/orchestration_failure.py +332 -0
- agentprobe/injector/tool_failure.py +404 -0
- agentprobe/observer/__init__.py +5 -0
- agentprobe/observer/classifier.py +320 -0
- agentprobe/observer/dashboard.py +216 -0
- agentprobe/observer/tracer.py +137 -0
- agentprobe_toolkit-1.2.0.dist-info/METADATA +405 -0
- agentprobe_toolkit-1.2.0.dist-info/RECORD +30 -0
- agentprobe_toolkit-1.2.0.dist-info/WHEEL +4 -0
- agentprobe_toolkit-1.2.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
"""Latency failure injection for agentic LLM systems.
|
|
2
|
+
|
|
3
|
+
Failure taxonomy — Latency Failures (Category 4):
|
|
4
|
+
- P99_SPIKE : Inject a single large latency spike (tail latency simulation)
|
|
5
|
+
- CASCADING_SLOW : Each successive call gets progressively slower
|
|
6
|
+
- COLD_START : First call has large startup delay, subsequent calls are normal
|
|
7
|
+
- JITTER : Add random jitter to every call (simulates network instability)
|
|
8
|
+
|
|
9
|
+
Unlike tool/orchestration failures, latency injectors do NOT raise exceptions.
|
|
10
|
+
They slow down execution to expose how agent frameworks handle slow responses,
|
|
11
|
+
impact downstream P99 latency, and affect user-perceived performance.
|
|
12
|
+
|
|
13
|
+
Usage::
|
|
14
|
+
|
|
15
|
+
injector = LatencyFailureInjector(
|
|
16
|
+
config=LatencyFailureConfig(
|
|
17
|
+
failure_type=LatencyFailureType.P99_SPIKE,
|
|
18
|
+
spike_delay_seconds=5.0,
|
|
19
|
+
spike_probability=0.01,
|
|
20
|
+
)
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
@injector.wrap
|
|
24
|
+
def call_llm(prompt: str) -> str:
|
|
25
|
+
return llm_api.complete(prompt)
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import asyncio
|
|
31
|
+
import functools
|
|
32
|
+
import random
|
|
33
|
+
import time
|
|
34
|
+
from collections.abc import Callable
|
|
35
|
+
from dataclasses import dataclass
|
|
36
|
+
from enum import StrEnum
|
|
37
|
+
from typing import Any, TypeVar
|
|
38
|
+
|
|
39
|
+
from agentprobe.observer.tracer import get_tracer
|
|
40
|
+
|
|
41
|
+
T = TypeVar("T")
|
|
42
|
+
|
|
43
|
+
_tracer = get_tracer(__name__)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
# ---------------------------------------------------------------------------
|
|
47
|
+
# Enums and config
|
|
48
|
+
# ---------------------------------------------------------------------------
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class LatencyFailureType(StrEnum):
|
|
52
|
+
NONE = "none"
|
|
53
|
+
P99_SPIKE = "p99_spike"
|
|
54
|
+
CASCADING_SLOW = "cascading_slow"
|
|
55
|
+
COLD_START = "cold_start"
|
|
56
|
+
JITTER = "jitter"
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass
|
|
60
|
+
class LatencyFailureConfig:
|
|
61
|
+
"""Configuration for latency failure injection.
|
|
62
|
+
|
|
63
|
+
Attributes:
|
|
64
|
+
failure_type: Which latency pattern to inject.
|
|
65
|
+
spike_delay_seconds: Delay to inject for P99_SPIKE mode.
|
|
66
|
+
spike_probability: Probability of a spike on each call (P99_SPIKE).
|
|
67
|
+
Set to 0.01 to simulate ~P99 tail latency.
|
|
68
|
+
cascade_increment_seconds: Additional delay added per call (CASCADING_SLOW).
|
|
69
|
+
cascade_max_seconds: Maximum total delay cap (CASCADING_SLOW).
|
|
70
|
+
cold_start_delay_seconds: First-call delay (COLD_START).
|
|
71
|
+
jitter_min_seconds: Minimum jitter per call (JITTER).
|
|
72
|
+
jitter_max_seconds: Maximum jitter per call (JITTER).
|
|
73
|
+
seed: RNG seed for reproducible experiments.
|
|
74
|
+
"""
|
|
75
|
+
|
|
76
|
+
failure_type: LatencyFailureType = LatencyFailureType.NONE
|
|
77
|
+
spike_delay_seconds: float = 5.0
|
|
78
|
+
spike_probability: float = 0.01
|
|
79
|
+
cascade_increment_seconds: float = 0.5
|
|
80
|
+
cascade_max_seconds: float = 10.0
|
|
81
|
+
cold_start_delay_seconds: float = 3.0
|
|
82
|
+
jitter_min_seconds: float = 0.1
|
|
83
|
+
jitter_max_seconds: float = 2.0
|
|
84
|
+
seed: int | None = None
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
# ---------------------------------------------------------------------------
|
|
88
|
+
# Latency record
|
|
89
|
+
# ---------------------------------------------------------------------------
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
@dataclass
|
|
93
|
+
class LatencyRecord:
|
|
94
|
+
call_index: int
|
|
95
|
+
base_latency_ms: float
|
|
96
|
+
injected_delay_ms: float
|
|
97
|
+
total_latency_ms: float
|
|
98
|
+
failure_type: LatencyFailureType
|
|
99
|
+
was_spike: bool = False
|
|
100
|
+
was_cold_start: bool = False
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
# ---------------------------------------------------------------------------
|
|
104
|
+
# Injector
|
|
105
|
+
# ---------------------------------------------------------------------------
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
class LatencyFailureInjector:
|
|
109
|
+
"""Injects latency failures into agent function calls.
|
|
110
|
+
|
|
111
|
+
Does not raise exceptions — instead adds synchronous or async delays
|
|
112
|
+
to simulate real-world latency distributions including tail latency.
|
|
113
|
+
"""
|
|
114
|
+
|
|
115
|
+
def __init__(self, config: LatencyFailureConfig | None = None) -> None:
|
|
116
|
+
self.config = config or LatencyFailureConfig()
|
|
117
|
+
self._rng = random.Random(self.config.seed)
|
|
118
|
+
self._log: list[LatencyRecord] = []
|
|
119
|
+
self._call_count = 0
|
|
120
|
+
self._cold_start_done = False
|
|
121
|
+
self._cascade_delay = 0.0
|
|
122
|
+
|
|
123
|
+
# ------------------------------------------------------------------
|
|
124
|
+
# Decorator API
|
|
125
|
+
# ------------------------------------------------------------------
|
|
126
|
+
|
|
127
|
+
def wrap(self, fn: Callable[..., T]) -> Callable[..., T]:
|
|
128
|
+
"""Wrap a synchronous function with latency injection."""
|
|
129
|
+
|
|
130
|
+
@functools.wraps(fn)
|
|
131
|
+
def wrapper(*args: Any, **kwargs: Any) -> T:
|
|
132
|
+
return self._execute_sync(fn, *args, **kwargs) # type: ignore[return-value]
|
|
133
|
+
|
|
134
|
+
return wrapper # type: ignore[return-value]
|
|
135
|
+
|
|
136
|
+
def wrap_async(self, fn: Callable[..., T]) -> Callable[..., T]:
|
|
137
|
+
"""Wrap an async function with latency injection."""
|
|
138
|
+
|
|
139
|
+
@functools.wraps(fn)
|
|
140
|
+
async def wrapper(*args: Any, **kwargs: Any) -> T:
|
|
141
|
+
return await self._execute_async(fn, *args, **kwargs) # type: ignore[return-value]
|
|
142
|
+
|
|
143
|
+
return wrapper # type: ignore[return-value]
|
|
144
|
+
|
|
145
|
+
# ------------------------------------------------------------------
|
|
146
|
+
# Observation
|
|
147
|
+
# ------------------------------------------------------------------
|
|
148
|
+
|
|
149
|
+
@property
|
|
150
|
+
def latency_log(self) -> list[LatencyRecord]:
|
|
151
|
+
return list(self._log)
|
|
152
|
+
|
|
153
|
+
def reset(self) -> None:
|
|
154
|
+
self._log.clear()
|
|
155
|
+
self._call_count = 0
|
|
156
|
+
self._cold_start_done = False
|
|
157
|
+
self._cascade_delay = 0.0
|
|
158
|
+
self._rng = random.Random(self.config.seed)
|
|
159
|
+
|
|
160
|
+
def summary(self) -> dict[str, Any]:
|
|
161
|
+
if not self._log:
|
|
162
|
+
return {"total_calls": 0}
|
|
163
|
+
latencies = [r.total_latency_ms for r in self._log]
|
|
164
|
+
sorted_lat = sorted(latencies)
|
|
165
|
+
n = len(sorted_lat)
|
|
166
|
+
return {
|
|
167
|
+
"total_calls": n,
|
|
168
|
+
"mean_latency_ms": sum(sorted_lat) / n,
|
|
169
|
+
"p50_latency_ms": sorted_lat[int(n * 0.50)],
|
|
170
|
+
"p90_latency_ms": sorted_lat[int(n * 0.90)],
|
|
171
|
+
"p99_latency_ms": sorted_lat[min(int(n * 0.99), n - 1)],
|
|
172
|
+
"max_latency_ms": sorted_lat[-1],
|
|
173
|
+
"spikes": sum(1 for r in self._log if r.was_spike),
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
# ------------------------------------------------------------------
|
|
177
|
+
# Internal
|
|
178
|
+
# ------------------------------------------------------------------
|
|
179
|
+
|
|
180
|
+
def _compute_delay(self) -> tuple[float, bool, bool]:
|
|
181
|
+
"""Return (delay_seconds, was_spike, was_cold_start)."""
|
|
182
|
+
cfg = self.config
|
|
183
|
+
match cfg.failure_type:
|
|
184
|
+
case LatencyFailureType.NONE:
|
|
185
|
+
return 0.0, False, False
|
|
186
|
+
|
|
187
|
+
case LatencyFailureType.P99_SPIKE:
|
|
188
|
+
is_spike = self._rng.random() < cfg.spike_probability
|
|
189
|
+
return (cfg.spike_delay_seconds if is_spike else 0.0), is_spike, False
|
|
190
|
+
|
|
191
|
+
case LatencyFailureType.CASCADING_SLOW:
|
|
192
|
+
self._cascade_delay = min(
|
|
193
|
+
self._cascade_delay + cfg.cascade_increment_seconds,
|
|
194
|
+
cfg.cascade_max_seconds,
|
|
195
|
+
)
|
|
196
|
+
return self._cascade_delay, False, False
|
|
197
|
+
|
|
198
|
+
case LatencyFailureType.COLD_START:
|
|
199
|
+
if not self._cold_start_done:
|
|
200
|
+
self._cold_start_done = True
|
|
201
|
+
return cfg.cold_start_delay_seconds, False, True
|
|
202
|
+
return 0.0, False, False
|
|
203
|
+
|
|
204
|
+
case LatencyFailureType.JITTER:
|
|
205
|
+
jitter = self._rng.uniform(cfg.jitter_min_seconds, cfg.jitter_max_seconds)
|
|
206
|
+
return jitter, False, False
|
|
207
|
+
|
|
208
|
+
case _:
|
|
209
|
+
return 0.0, False, False
|
|
210
|
+
|
|
211
|
+
def _execute_sync(self, fn: Callable, *args: Any, **kwargs: Any) -> Any:
|
|
212
|
+
self._call_count += 1
|
|
213
|
+
delay, was_spike, was_cold = self._compute_delay()
|
|
214
|
+
|
|
215
|
+
with _tracer.start_as_current_span(
|
|
216
|
+
f"latency.call.{fn.__name__}",
|
|
217
|
+
attributes={
|
|
218
|
+
"latency.injected_delay_ms": delay * 1000,
|
|
219
|
+
"latency.failure_type": self.config.failure_type.value,
|
|
220
|
+
"latency.was_spike": was_spike,
|
|
221
|
+
"latency.was_cold_start": was_cold,
|
|
222
|
+
},
|
|
223
|
+
):
|
|
224
|
+
start = time.perf_counter()
|
|
225
|
+
if delay > 0:
|
|
226
|
+
time.sleep(delay)
|
|
227
|
+
fn_start = time.perf_counter()
|
|
228
|
+
try:
|
|
229
|
+
return fn(*args, **kwargs)
|
|
230
|
+
finally:
|
|
231
|
+
end = time.perf_counter()
|
|
232
|
+
self._log.append(
|
|
233
|
+
LatencyRecord(
|
|
234
|
+
call_index=self._call_count,
|
|
235
|
+
base_latency_ms=(end - fn_start) * 1000,
|
|
236
|
+
injected_delay_ms=delay * 1000,
|
|
237
|
+
total_latency_ms=(end - start) * 1000,
|
|
238
|
+
failure_type=self.config.failure_type,
|
|
239
|
+
was_spike=was_spike,
|
|
240
|
+
was_cold_start=was_cold,
|
|
241
|
+
)
|
|
242
|
+
)
|
|
243
|
+
|
|
244
|
+
async def _execute_async(self, fn: Callable, *args: Any, **kwargs: Any) -> Any:
|
|
245
|
+
self._call_count += 1
|
|
246
|
+
delay, was_spike, was_cold = self._compute_delay()
|
|
247
|
+
|
|
248
|
+
with _tracer.start_as_current_span(
|
|
249
|
+
f"latency.call.{fn.__name__}",
|
|
250
|
+
attributes={
|
|
251
|
+
"latency.injected_delay_ms": delay * 1000,
|
|
252
|
+
"latency.failure_type": self.config.failure_type.value,
|
|
253
|
+
"latency.was_spike": was_spike,
|
|
254
|
+
"latency.was_cold_start": was_cold,
|
|
255
|
+
},
|
|
256
|
+
):
|
|
257
|
+
start = time.perf_counter()
|
|
258
|
+
if delay > 0:
|
|
259
|
+
await asyncio.sleep(delay)
|
|
260
|
+
fn_start = time.perf_counter()
|
|
261
|
+
try:
|
|
262
|
+
return await fn(*args, **kwargs)
|
|
263
|
+
finally:
|
|
264
|
+
end = time.perf_counter()
|
|
265
|
+
self._log.append(
|
|
266
|
+
LatencyRecord(
|
|
267
|
+
call_index=self._call_count,
|
|
268
|
+
base_latency_ms=(end - fn_start) * 1000,
|
|
269
|
+
injected_delay_ms=delay * 1000,
|
|
270
|
+
total_latency_ms=(end - start) * 1000,
|
|
271
|
+
failure_type=self.config.failure_type,
|
|
272
|
+
was_spike=was_spike,
|
|
273
|
+
was_cold_start=was_cold,
|
|
274
|
+
)
|
|
275
|
+
)
|
|
@@ -0,0 +1,332 @@
|
|
|
1
|
+
"""Orchestration failure injection for agentic LLM systems.
|
|
2
|
+
|
|
3
|
+
Failure taxonomy — Orchestration Failures (Category 2):
|
|
4
|
+
- INFINITE_LOOP : Agent cycles through nodes without terminating
|
|
5
|
+
- WRONG_BRANCH : Agent takes incorrect conditional branch
|
|
6
|
+
- STATE_CORRUPTION : Agent state dict is silently corrupted mid-execution
|
|
7
|
+
- MISSING_HANDOFF : Multi-agent handoff message is dropped
|
|
8
|
+
|
|
9
|
+
These injectors wrap LangGraph StateGraph nodes and edges to simulate
|
|
10
|
+
failures in agent control flow rather than individual tool calls.
|
|
11
|
+
|
|
12
|
+
Usage::
|
|
13
|
+
|
|
14
|
+
injector = OrchestrationFailureInjector(
|
|
15
|
+
config=OrchestrationFailureConfig(
|
|
16
|
+
failure_type=OrchestrationFailureType.INFINITE_LOOP,
|
|
17
|
+
max_loop_iterations=3,
|
|
18
|
+
)
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
# Wrap a LangGraph node function
|
|
22
|
+
safe_node = injector.wrap_node(my_node_fn, node_name="my_node")
|
|
23
|
+
|
|
24
|
+
# Or wrap a conditional edge router
|
|
25
|
+
faulty_router = injector.wrap_router(my_router_fn)
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import functools
|
|
31
|
+
import random
|
|
32
|
+
import time
|
|
33
|
+
from collections.abc import Callable
|
|
34
|
+
from dataclasses import dataclass
|
|
35
|
+
from enum import StrEnum
|
|
36
|
+
from typing import Any, TypeVar
|
|
37
|
+
|
|
38
|
+
from opentelemetry.trace import Status, StatusCode
|
|
39
|
+
|
|
40
|
+
from agentprobe.observer.tracer import get_tracer
|
|
41
|
+
|
|
42
|
+
T = TypeVar("T")
|
|
43
|
+
|
|
44
|
+
_tracer = get_tracer(__name__)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
# ---------------------------------------------------------------------------
|
|
48
|
+
# Exceptions
|
|
49
|
+
# ---------------------------------------------------------------------------
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class OrchestrationError(Exception):
|
|
53
|
+
"""Base class for injected orchestration errors."""
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class InfiniteLoopError(OrchestrationError):
|
|
57
|
+
"""Raised when an injected loop exceeds its iteration cap."""
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class WrongBranchError(OrchestrationError):
|
|
61
|
+
"""Raised to signal incorrect routing in a conditional edge."""
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class StateCorruptionError(OrchestrationError):
|
|
65
|
+
"""Raised when agent state has been silently corrupted."""
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class MissingHandoffError(OrchestrationError):
|
|
69
|
+
"""Raised when a multi-agent handoff message is dropped."""
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
# ---------------------------------------------------------------------------
|
|
73
|
+
# Enums and config
|
|
74
|
+
# ---------------------------------------------------------------------------
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class OrchestrationFailureType(StrEnum):
|
|
78
|
+
NONE = "none"
|
|
79
|
+
INFINITE_LOOP = "infinite_loop"
|
|
80
|
+
WRONG_BRANCH = "wrong_branch"
|
|
81
|
+
STATE_CORRUPTION = "state_corruption"
|
|
82
|
+
MISSING_HANDOFF = "missing_handoff"
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
@dataclass
|
|
86
|
+
class OrchestrationFailureConfig:
|
|
87
|
+
"""Configuration for orchestration failure injection.
|
|
88
|
+
|
|
89
|
+
Attributes:
|
|
90
|
+
failure_type: Which orchestration failure mode to inject.
|
|
91
|
+
failure_probability: Probability [0, 1] of injecting on each call.
|
|
92
|
+
max_loop_iterations: How many loop iterations before raising (INFINITE_LOOP).
|
|
93
|
+
corrupt_keys: State keys to corrupt (STATE_CORRUPTION).
|
|
94
|
+
If empty, a random key is selected.
|
|
95
|
+
wrong_branch_target: The incorrect branch label to return (WRONG_BRANCH).
|
|
96
|
+
If None, returns "__end__" prematurely.
|
|
97
|
+
seed: RNG seed for reproducible experiments.
|
|
98
|
+
"""
|
|
99
|
+
|
|
100
|
+
failure_type: OrchestrationFailureType = OrchestrationFailureType.NONE
|
|
101
|
+
failure_probability: float = 1.0
|
|
102
|
+
max_loop_iterations: int = 5
|
|
103
|
+
corrupt_keys: list[str] | None = None
|
|
104
|
+
wrong_branch_target: str | None = None
|
|
105
|
+
seed: int | None = None
|
|
106
|
+
|
|
107
|
+
def __post_init__(self) -> None:
|
|
108
|
+
if not 0.0 <= self.failure_probability <= 1.0:
|
|
109
|
+
raise ValueError("failure_probability must be in [0, 1]")
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
# ---------------------------------------------------------------------------
|
|
113
|
+
# Injection result
|
|
114
|
+
# ---------------------------------------------------------------------------
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
@dataclass
|
|
118
|
+
class OrchestrationInjectionResult:
|
|
119
|
+
node_name: str
|
|
120
|
+
failure_type: OrchestrationFailureType
|
|
121
|
+
injected: bool
|
|
122
|
+
exception: Exception | None = None
|
|
123
|
+
latency_ms: float = 0.0
|
|
124
|
+
loop_iterations: int = 0
|
|
125
|
+
corrupted_keys: list[str] | None = None
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
# ---------------------------------------------------------------------------
|
|
129
|
+
# Injector
|
|
130
|
+
# ---------------------------------------------------------------------------
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
class OrchestrationFailureInjector:
|
|
134
|
+
"""Injects orchestration-level failures into LangGraph agent graphs."""
|
|
135
|
+
|
|
136
|
+
def __init__(self, config: OrchestrationFailureConfig | None = None) -> None:
|
|
137
|
+
self.config = config or OrchestrationFailureConfig()
|
|
138
|
+
self._rng = random.Random(self.config.seed)
|
|
139
|
+
self._log: list[OrchestrationInjectionResult] = []
|
|
140
|
+
self._loop_counters: dict[str, int] = {}
|
|
141
|
+
|
|
142
|
+
# ------------------------------------------------------------------
|
|
143
|
+
# Node wrapping
|
|
144
|
+
# ------------------------------------------------------------------
|
|
145
|
+
|
|
146
|
+
def wrap_node(self, fn: Callable, node_name: str = "") -> Callable:
|
|
147
|
+
"""Wrap a LangGraph node function with orchestration failure injection.
|
|
148
|
+
|
|
149
|
+
The wrapped function accepts the LangGraph state dict and returns
|
|
150
|
+
a (possibly corrupted) state update dict.
|
|
151
|
+
"""
|
|
152
|
+
name = node_name or fn.__name__
|
|
153
|
+
|
|
154
|
+
@functools.wraps(fn)
|
|
155
|
+
def wrapper(state: dict[str, Any]) -> dict[str, Any]:
|
|
156
|
+
return self._execute_node(fn, name, state)
|
|
157
|
+
|
|
158
|
+
return wrapper
|
|
159
|
+
|
|
160
|
+
def wrap_router(self, fn: Callable, node_name: str = "") -> Callable:
|
|
161
|
+
"""Wrap a LangGraph conditional edge router with wrong-branch injection."""
|
|
162
|
+
name = node_name or fn.__name__
|
|
163
|
+
|
|
164
|
+
@functools.wraps(fn)
|
|
165
|
+
def wrapper(state: dict[str, Any]) -> str:
|
|
166
|
+
return self._execute_router(fn, name, state)
|
|
167
|
+
|
|
168
|
+
return wrapper
|
|
169
|
+
|
|
170
|
+
# ------------------------------------------------------------------
|
|
171
|
+
# Observation
|
|
172
|
+
# ------------------------------------------------------------------
|
|
173
|
+
|
|
174
|
+
@property
|
|
175
|
+
def injection_log(self) -> list[OrchestrationInjectionResult]:
|
|
176
|
+
return list(self._log)
|
|
177
|
+
|
|
178
|
+
def reset(self) -> None:
|
|
179
|
+
self._log.clear()
|
|
180
|
+
self._loop_counters.clear()
|
|
181
|
+
self._rng = random.Random(self.config.seed)
|
|
182
|
+
|
|
183
|
+
def summary(self) -> dict[str, Any]:
|
|
184
|
+
total = len(self._log)
|
|
185
|
+
injected = sum(1 for r in self._log if r.injected)
|
|
186
|
+
return {
|
|
187
|
+
"total_node_calls": total,
|
|
188
|
+
"injected_failures": injected,
|
|
189
|
+
"injection_rate": injected / total if total > 0 else 0.0,
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
# ------------------------------------------------------------------
|
|
193
|
+
# Internal
|
|
194
|
+
# ------------------------------------------------------------------
|
|
195
|
+
|
|
196
|
+
def _should_inject(self) -> bool:
|
|
197
|
+
if self.config.failure_type == OrchestrationFailureType.NONE:
|
|
198
|
+
return False
|
|
199
|
+
return self._rng.random() < self.config.failure_probability
|
|
200
|
+
|
|
201
|
+
def _execute_node(self, fn: Callable, name: str, state: dict[str, Any]) -> dict[str, Any]:
|
|
202
|
+
inject = self._should_inject()
|
|
203
|
+
start = time.perf_counter()
|
|
204
|
+
|
|
205
|
+
with _tracer.start_as_current_span(
|
|
206
|
+
f"orchestration.node.{name}",
|
|
207
|
+
attributes={
|
|
208
|
+
"node.name": name,
|
|
209
|
+
"failure.injected": inject,
|
|
210
|
+
"failure.type": self.config.failure_type.value if inject else "none",
|
|
211
|
+
},
|
|
212
|
+
) as span:
|
|
213
|
+
rec = OrchestrationInjectionResult(
|
|
214
|
+
node_name=name,
|
|
215
|
+
failure_type=self.config.failure_type if inject else OrchestrationFailureType.NONE,
|
|
216
|
+
injected=inject,
|
|
217
|
+
)
|
|
218
|
+
try:
|
|
219
|
+
if inject:
|
|
220
|
+
match self.config.failure_type:
|
|
221
|
+
case OrchestrationFailureType.INFINITE_LOOP:
|
|
222
|
+
return self._inject_loop(fn, name, state, span, rec)
|
|
223
|
+
case OrchestrationFailureType.STATE_CORRUPTION:
|
|
224
|
+
return self._inject_state_corruption(fn, name, state, span, rec)
|
|
225
|
+
case OrchestrationFailureType.MISSING_HANDOFF:
|
|
226
|
+
exc = MissingHandoffError(f"Handoff message dropped at node '{name}'")
|
|
227
|
+
span.set_status(Status(StatusCode.ERROR, str(exc)))
|
|
228
|
+
rec.exception = exc
|
|
229
|
+
raise exc
|
|
230
|
+
case _:
|
|
231
|
+
return fn(state)
|
|
232
|
+
return fn(state)
|
|
233
|
+
finally:
|
|
234
|
+
rec.latency_ms = (time.perf_counter() - start) * 1000
|
|
235
|
+
self._log.append(rec)
|
|
236
|
+
|
|
237
|
+
def _execute_router(self, fn: Callable, name: str, state: dict[str, Any]) -> str:
|
|
238
|
+
inject = self._should_inject()
|
|
239
|
+
start = time.perf_counter()
|
|
240
|
+
|
|
241
|
+
with _tracer.start_as_current_span(
|
|
242
|
+
f"orchestration.router.{name}",
|
|
243
|
+
attributes={
|
|
244
|
+
"router.name": name,
|
|
245
|
+
"failure.injected": inject,
|
|
246
|
+
"failure.type": self.config.failure_type.value if inject else "none",
|
|
247
|
+
},
|
|
248
|
+
) as span:
|
|
249
|
+
rec = OrchestrationInjectionResult(
|
|
250
|
+
node_name=name,
|
|
251
|
+
failure_type=self.config.failure_type if inject else OrchestrationFailureType.NONE,
|
|
252
|
+
injected=inject,
|
|
253
|
+
)
|
|
254
|
+
try:
|
|
255
|
+
if inject and self.config.failure_type == OrchestrationFailureType.WRONG_BRANCH:
|
|
256
|
+
wrong = self.config.wrong_branch_target or "__end__"
|
|
257
|
+
span.set_attribute("router.injected_branch", wrong)
|
|
258
|
+
span.set_attribute("failure.injected", True)
|
|
259
|
+
return wrong
|
|
260
|
+
return fn(state)
|
|
261
|
+
finally:
|
|
262
|
+
rec.latency_ms = (time.perf_counter() - start) * 1000
|
|
263
|
+
self._log.append(rec)
|
|
264
|
+
|
|
265
|
+
def _inject_loop(
|
|
266
|
+
self,
|
|
267
|
+
fn: Callable,
|
|
268
|
+
name: str,
|
|
269
|
+
state: dict[str, Any],
|
|
270
|
+
span: Any,
|
|
271
|
+
rec: OrchestrationInjectionResult,
|
|
272
|
+
) -> dict[str, Any]:
|
|
273
|
+
"""Run the node repeatedly until max_loop_iterations, then raise."""
|
|
274
|
+
counter = self._loop_counters.get(name, 0) + 1
|
|
275
|
+
self._loop_counters[name] = counter
|
|
276
|
+
rec.loop_iterations = counter
|
|
277
|
+
|
|
278
|
+
if counter >= self.config.max_loop_iterations:
|
|
279
|
+
exc = InfiniteLoopError(
|
|
280
|
+
f"Node '{name}' detected as infinite loop after {counter} iterations"
|
|
281
|
+
)
|
|
282
|
+
span.set_status(Status(StatusCode.ERROR, str(exc)))
|
|
283
|
+
rec.exception = exc
|
|
284
|
+
self._loop_counters[name] = 0
|
|
285
|
+
raise exc
|
|
286
|
+
|
|
287
|
+
return fn(state)
|
|
288
|
+
|
|
289
|
+
def _inject_state_corruption(
|
|
290
|
+
self,
|
|
291
|
+
fn: Callable,
|
|
292
|
+
name: str,
|
|
293
|
+
state: dict[str, Any],
|
|
294
|
+
span: Any,
|
|
295
|
+
rec: OrchestrationInjectionResult,
|
|
296
|
+
) -> dict[str, Any]:
|
|
297
|
+
"""Run the node, then silently corrupt keys in the returned state."""
|
|
298
|
+
result = fn(state)
|
|
299
|
+
keys_to_corrupt = self.config.corrupt_keys or (list(result.keys()) or list(state.keys()))
|
|
300
|
+
if not keys_to_corrupt:
|
|
301
|
+
return result
|
|
302
|
+
|
|
303
|
+
# Pick one key to corrupt
|
|
304
|
+
target_key = self._rng.choice(keys_to_corrupt)
|
|
305
|
+
original = result.get(target_key, state.get(target_key))
|
|
306
|
+
|
|
307
|
+
corrupted_result = dict(result)
|
|
308
|
+
corrupted_result[target_key] = _corrupt_value(original)
|
|
309
|
+
|
|
310
|
+
rec.corrupted_keys = [target_key]
|
|
311
|
+
span.set_attribute("corruption.key", target_key)
|
|
312
|
+
return corrupted_result
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
# ---------------------------------------------------------------------------
|
|
316
|
+
# Helpers
|
|
317
|
+
# ---------------------------------------------------------------------------
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def _corrupt_value(value: Any) -> Any:
|
|
321
|
+
"""Return a type-preserving but semantically wrong value."""
|
|
322
|
+
if isinstance(value, str):
|
|
323
|
+
return value[::-1] if value else "CORRUPTED"
|
|
324
|
+
if isinstance(value, int):
|
|
325
|
+
return -value - 1
|
|
326
|
+
if isinstance(value, float):
|
|
327
|
+
return float("nan")
|
|
328
|
+
if isinstance(value, list):
|
|
329
|
+
return list(reversed(value))
|
|
330
|
+
if isinstance(value, dict):
|
|
331
|
+
return {k: None for k in value}
|
|
332
|
+
return None
|