agentprobe-toolkit 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentprobe/__init__.py +42 -0
- agentprobe/agents/__init__.py +5 -0
- agentprobe/agents/autogen_agent.py +308 -0
- agentprobe/agents/base.py +97 -0
- agentprobe/agents/langchain_agent.py +140 -0
- agentprobe/agents/langgraph_agent.py +203 -0
- agentprobe/benchmark/__init__.py +67 -0
- agentprobe/benchmark/grader.py +162 -0
- agentprobe/benchmark/metadata.py +214 -0
- agentprobe/benchmark/metrics.py +247 -0
- agentprobe/benchmark/runner.py +434 -0
- agentprobe/benchmark/statistics.py +459 -0
- agentprobe/benchmark/tasks/__init__.py +75 -0
- agentprobe/benchmark/tasks/multi_agent.py +191 -0
- agentprobe/benchmark/tasks/rag.py +309 -0
- agentprobe/benchmark/tasks/tool_use.py +228 -0
- agentprobe/config.py +67 -0
- agentprobe/injector/__init__.py +66 -0
- agentprobe/injector/context_failure.py +328 -0
- agentprobe/injector/latency_failure.py +275 -0
- agentprobe/injector/orchestration_failure.py +332 -0
- agentprobe/injector/tool_failure.py +404 -0
- agentprobe/observer/__init__.py +5 -0
- agentprobe/observer/classifier.py +320 -0
- agentprobe/observer/dashboard.py +216 -0
- agentprobe/observer/tracer.py +137 -0
- agentprobe_toolkit-1.2.0.dist-info/METADATA +405 -0
- agentprobe_toolkit-1.2.0.dist-info/RECORD +30 -0
- agentprobe_toolkit-1.2.0.dist-info/WHEEL +4 -0
- agentprobe_toolkit-1.2.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,405 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: agentprobe-toolkit
|
|
3
|
+
Version: 1.2.0
|
|
4
|
+
Summary: Fault injection and OpenTelemetry observability toolkit for evaluating the reliability of LLM agents (LangGraph, LangChain, AutoGen)
|
|
5
|
+
Project-URL: Homepage, https://github.com/chhayankjain/AgentProbe
|
|
6
|
+
Project-URL: Repository, https://github.com/chhayankjain/AgentProbe
|
|
7
|
+
Project-URL: Issues, https://github.com/chhayankjain/AgentProbe/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/chhayankjain/AgentProbe/blob/main/CHANGELOG.md
|
|
9
|
+
Project-URL: Roadmap, https://github.com/chhayankjain/AgentProbe/issues/38
|
|
10
|
+
Author: Chhayank Jain, Sarthak Pattnaik, Eugene Pinsky
|
|
11
|
+
Maintainer: Chhayank Jain
|
|
12
|
+
License-Expression: MIT
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Keywords: agents,autogen,benchmarking,chaos-engineering,fault-injection,langchain,langgraph,llm,observability,opentelemetry,reliability
|
|
15
|
+
Classifier: Development Status :: 4 - Beta
|
|
16
|
+
Classifier: Intended Audience :: Science/Research
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
22
|
+
Classifier: Topic :: Software Development :: Testing
|
|
23
|
+
Classifier: Topic :: System :: Monitoring
|
|
24
|
+
Classifier: Typing :: Typed
|
|
25
|
+
Requires-Python: >=3.11
|
|
26
|
+
Requires-Dist: langchain-core>=1.0.0
|
|
27
|
+
Requires-Dist: langchain>=1.0.0
|
|
28
|
+
Requires-Dist: langgraph>=1.0.0
|
|
29
|
+
Requires-Dist: numpy>=1.26.0
|
|
30
|
+
Requires-Dist: opentelemetry-api>=1.27.0
|
|
31
|
+
Requires-Dist: opentelemetry-exporter-otlp-proto-grpc>=1.27.0
|
|
32
|
+
Requires-Dist: opentelemetry-exporter-prometheus>=0.48b0
|
|
33
|
+
Requires-Dist: opentelemetry-sdk>=1.27.0
|
|
34
|
+
Requires-Dist: prometheus-client>=0.21.0
|
|
35
|
+
Requires-Dist: pydantic>=2.0.0
|
|
36
|
+
Requires-Dist: rich>=13.0.0
|
|
37
|
+
Requires-Dist: scipy>=1.12.0
|
|
38
|
+
Provides-Extra: anthropic
|
|
39
|
+
Requires-Dist: langchain-anthropic>=0.2.0; extra == 'anthropic'
|
|
40
|
+
Provides-Extra: autogen
|
|
41
|
+
Requires-Dist: openai<1.0; extra == 'autogen'
|
|
42
|
+
Requires-Dist: pyautogen==0.1.14; extra == 'autogen'
|
|
43
|
+
Provides-Extra: dev
|
|
44
|
+
Requires-Dist: ipykernel>=6.29.0; extra == 'dev'
|
|
45
|
+
Requires-Dist: mypy>=1.11.0; extra == 'dev'
|
|
46
|
+
Requires-Dist: nbformat>=5.10.0; extra == 'dev'
|
|
47
|
+
Requires-Dist: pytest-asyncio>=0.24.0; extra == 'dev'
|
|
48
|
+
Requires-Dist: pytest>=8.0.0; extra == 'dev'
|
|
49
|
+
Requires-Dist: ruff>=0.7.0; extra == 'dev'
|
|
50
|
+
Provides-Extra: ollama
|
|
51
|
+
Requires-Dist: langchain-ollama>=0.2.0; extra == 'ollama'
|
|
52
|
+
Provides-Extra: openai
|
|
53
|
+
Requires-Dist: langchain-openai>=0.2.0; extra == 'openai'
|
|
54
|
+
Provides-Extra: plotting
|
|
55
|
+
Requires-Dist: matplotlib>=3.8.0; extra == 'plotting'
|
|
56
|
+
Requires-Dist: pandas>=2.0.0; extra == 'plotting'
|
|
57
|
+
Description-Content-Type: text/markdown
|
|
58
|
+
|
|
59
|
+
# AgentProbe
|
|
60
|
+
|
|
61
|
+
[](https://github.com/chhayankjain/AgentProbe/actions/workflows/ci.yml)
|
|
62
|
+
[](https://github.com/chhayankjain/AgentProbe/actions/workflows/codeql.yml)
|
|
63
|
+
[](https://www.python.org/downloads/)
|
|
64
|
+
[](https://github.com/chhayankjain/AgentProbe/blob/main/LICENSE)
|
|
65
|
+
[](https://pypi.org/project/agentprobe-toolkit/)
|
|
66
|
+
[](https://github.com/chhayankjain/AgentProbe/releases)
|
|
67
|
+
|
|
68
|
+
**Fault injection and observability toolkit for LLM agents**
|
|
69
|
+
|
|
70
|
+
AgentProbe wraps an agent's tools with seeded, logged fault injectors (timeouts, malformed output, rate limits, API errors), traces every invocation and tool call with OpenTelemetry, and reports failure rate, recovery rate, MTTR, and latency percentiles across frameworks. It ships adapters for LangGraph, LangChain, and AutoGen, plus the scripts and data behind the companion paper.
|
|
71
|
+
|
|
72
|
+
Companion to the paper: *"AgentProbe: Systematic Fault Injection and Observability for Evaluating the Reliability of LLM-Based Intelligent Agents"* (submitted to MAKE, MDPI, 2026).
|
|
73
|
+
|
|
74
|
+
---
|
|
75
|
+
|
|
76
|
+
## Failure Taxonomy
|
|
77
|
+
|
|
78
|
+
AgentProbe organises agent failures into five categories. Only the first has been exercised in a published experiment; the status column says exactly what exists for each.
|
|
79
|
+
|
|
80
|
+
| # | Category | Examples | Status in this repo |
|
|
81
|
+
|---|----------|----------|---------------------|
|
|
82
|
+
| 1 | **Tool Call** | API timeouts, malformed output, rate limits (HTTP 429), server errors (5xx) | Injector implemented, **used in all paper experiments** (`injector/tool_failure.py`) |
|
|
83
|
+
| 2 | **Orchestration** | Infinite loops, wrong branching, state corruption, dropped handoffs | Injector implemented and unit-tested, not yet used in an experiment (`injector/orchestration_failure.py`) |
|
|
84
|
+
| 3 | **Context** | Context window overflow, lost history, hallucinated history | Injector implemented and unit-tested, not yet used in an experiment (`injector/context_failure.py`) |
|
|
85
|
+
| 4 | **Latency** | P99 spikes, cascading slowdowns, jitter | Injector implemented; latency in the paper is a by-product of tool faults, not independently injected (`injector/latency_failure.py`) |
|
|
86
|
+
| 5 | **Consistency** | Non-deterministic outputs, divergent multi-agent state | Detected only, via a Jaccard-similarity classifier; nothing is injected (`observer/classifier.py`) |
|
|
87
|
+
|
|
88
|
+
Note that the current "malformed output" injector raises an exception at the tool boundary rather than returning corrupted data to the model, so it tests the agent's exception path, not its parsing. See issue [#28](https://github.com/chhayankjain/AgentProbe/issues/28).
|
|
89
|
+
|
|
90
|
+
---
|
|
91
|
+
|
|
92
|
+
## Installation
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
# From PyPI (LangGraph + LangChain adapters, local Ollama backend)
|
|
96
|
+
pip install "agentprobe-toolkit[ollama]"
|
|
97
|
+
|
|
98
|
+
# From a clone, for development or to run the paper's experiments
|
|
99
|
+
git clone https://github.com/chhayankjain/AgentProbe.git && cd AgentProbe
|
|
100
|
+
pip install -e ".[dev,plotting,ollama]"
|
|
101
|
+
|
|
102
|
+
# The AutoGen adapter targets pyautogen 0.1.14 and needs openai<1.0, which
|
|
103
|
+
# conflicts with langchain-openai. Install it in its own environment:
|
|
104
|
+
# pip install "agentprobe-toolkit[ollama,autogen]"
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
The distribution is named `agentprobe-toolkit` because `agentprobe` was already taken on PyPI by an unrelated project. The import name is `agentprobe`.
|
|
108
|
+
|
|
109
|
+
## Quick Start
|
|
110
|
+
|
|
111
|
+
```bash
|
|
112
|
+
|
|
113
|
+
# Start Ollama with Llama 3.1 (used in all paper experiments)
|
|
114
|
+
ollama pull llama3.1
|
|
115
|
+
|
|
116
|
+
# For cross-model validation (optional)
|
|
117
|
+
# ollama pull mistral:7b
|
|
118
|
+
# ollama pull qwen2.5:7b
|
|
119
|
+
|
|
120
|
+
# Run baseline benchmark (no failures)
|
|
121
|
+
python experiments/run_benchmarks.py --frameworks langgraph --tasks tool_use --baseline-only
|
|
122
|
+
|
|
123
|
+
# Run with failure injection
|
|
124
|
+
python experiments/run_benchmarks.py \
|
|
125
|
+
--frameworks langgraph langchain \
|
|
126
|
+
--tasks tool_use rag \
|
|
127
|
+
--failure-types none timeout rate_limit \
|
|
128
|
+
--n-runs 50 \
|
|
129
|
+
--save-results
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
## Programmatic Usage
|
|
133
|
+
|
|
134
|
+
```python
|
|
135
|
+
from langchain_ollama import ChatOllama
|
|
136
|
+
|
|
137
|
+
from agentprobe import (
|
|
138
|
+
AgentProbeTracer,
|
|
139
|
+
BenchmarkConfig,
|
|
140
|
+
BenchmarkRunner,
|
|
141
|
+
MetricsCalculator,
|
|
142
|
+
ToolFailureConfig,
|
|
143
|
+
ToolFailureType,
|
|
144
|
+
)
|
|
145
|
+
from agentprobe.agents.langgraph_agent import LangGraphToolAgent
|
|
146
|
+
|
|
147
|
+
# 1. Set up tracing (console by default; OTLP endpoint optional)
|
|
148
|
+
tracer = AgentProbeTracer(service_name="my-experiment", otlp_endpoint="http://localhost:4317")
|
|
149
|
+
|
|
150
|
+
# 2. Configure failure injection
|
|
151
|
+
failure_config = ToolFailureConfig(
|
|
152
|
+
failure_type=ToolFailureType.TIMEOUT,
|
|
153
|
+
failure_probability=0.2, # 20% of tool calls will time out
|
|
154
|
+
timeout_seconds=5.0,
|
|
155
|
+
seed=42, # reproducible injection sequence
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
# 3. Build an agent; it wraps its tools with a ToolFailureInjector internally
|
|
159
|
+
llm = ChatOllama(model="llama3.1", temperature=0.0)
|
|
160
|
+
agent = LangGraphToolAgent(llm=llm, failure_config=failure_config)
|
|
161
|
+
|
|
162
|
+
# 4. Run the benchmark (defaults to the paper's five-prompt set)
|
|
163
|
+
config = BenchmarkConfig(
|
|
164
|
+
framework="langgraph",
|
|
165
|
+
task="tool_use",
|
|
166
|
+
n_runs=50,
|
|
167
|
+
failure_config=failure_config,
|
|
168
|
+
recovery_attempts=1,
|
|
169
|
+
save_results_path="experiments/results/langgraph_tool_use_timeout",
|
|
170
|
+
)
|
|
171
|
+
results = BenchmarkRunner(agent=agent, config=config).run()
|
|
172
|
+
|
|
173
|
+
# 5. Compute metrics
|
|
174
|
+
metrics = MetricsCalculator().compute(results)
|
|
175
|
+
print(f"Failure rate: {metrics.failure_rate:.1%}")
|
|
176
|
+
print(f"Recovery rate: {metrics.recovery_rate}")
|
|
177
|
+
print(f"MTTR: {metrics.mttr_ms:.0f}ms")
|
|
178
|
+
print(f"P99 latency: {metrics.p99_latency_ms:.0f}ms")
|
|
179
|
+
print(agent.get_injector_summary())
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
### Injecting faults into your own tools
|
|
183
|
+
|
|
184
|
+
The injector is framework-agnostic. Wrap any callable, or any LangChain `BaseTool`:
|
|
185
|
+
|
|
186
|
+
```python
|
|
187
|
+
from agentprobe import ToolFailureConfig, ToolFailureInjector, ToolFailureType
|
|
188
|
+
|
|
189
|
+
injector = ToolFailureInjector(
|
|
190
|
+
ToolFailureConfig(failure_type=ToolFailureType.RATE_LIMIT, failure_probability=0.3, seed=7)
|
|
191
|
+
)
|
|
192
|
+
|
|
193
|
+
@injector.wrap
|
|
194
|
+
def lookup_price(sku: str) -> float:
|
|
195
|
+
...
|
|
196
|
+
|
|
197
|
+
faulty_tool = injector.inject_langchain_tool(my_langchain_tool) # in-place, keeps schema
|
|
198
|
+
print(injector.summary()) # total_calls, injected_failures, failures_by_type, ...
|
|
199
|
+
print(injector.injection_log) # one record per call with args, return/exception, span_id
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
### Two prompt sets
|
|
203
|
+
|
|
204
|
+
- **Paper prompts (default).** `BenchmarkRunner` cycles the five prompts per task in `agentprobe.benchmark.runner.PAPER_PROMPTS`. Every batch in the paper used these.
|
|
205
|
+
- **Graded corpora.** `agentprobe.benchmark.tasks` holds 25 tool-use, 15 RAG, and 15 multi-agent queries with keyword graders. They have not been used in a published experiment. Run them with `task_fn=corpus_task_fn("tool_use")` or `run_benchmarks.py --query-set corpus`.
|
|
206
|
+
|
|
207
|
+
### Metric definitions
|
|
208
|
+
|
|
209
|
+
- **Failure**: the agent invocation raised an exception that reached the harness. A wrong-but-exception-free answer is *not* a failure here; use the grader for that.
|
|
210
|
+
- **Recovery**: the harness re-invoked the agent once (injector still active) and that call succeeded. This is a harness-level retry, not agent-internal self-repair.
|
|
211
|
+
- **MTTR**: mean latency of successful re-invocations, in ms.
|
|
212
|
+
- **P50/P90/P99**: ceiling nearest-rank percentiles of per-run latency. For n < 100, P99 equals the sample maximum. Injected timeouts sleep for real, so latency under the timeout condition includes the injected delay.
|
|
213
|
+
- **Recovery rate** is `None` when a condition had no failures.
|
|
214
|
+
|
|
215
|
+
## Architecture
|
|
216
|
+
|
|
217
|
+
```
|
|
218
|
+
agentprobe/
|
|
219
|
+
├── injector/
|
|
220
|
+
│ ├── tool_failure.py # Category 1: tool call failures
|
|
221
|
+
│ ├── orchestration_failure.py # Category 2: orchestration failures
|
|
222
|
+
│ ├── context_failure.py # Category 3: context failures
|
|
223
|
+
│ └── latency_failure.py # Category 4: latency failures
|
|
224
|
+
├── observer/
|
|
225
|
+
│ ├── tracer.py # OpenTelemetry setup + span utilities
|
|
226
|
+
│ ├── classifier.py # Rule-based failure classification
|
|
227
|
+
│ └── dashboard.py # Prometheus metrics exporter
|
|
228
|
+
├── benchmark/
|
|
229
|
+
│ ├── runner.py # BenchmarkRunner + PAPER_PROMPTS (the paper's prompt set)
|
|
230
|
+
│ ├── tasks/ # Larger graded corpora (not used in the paper)
|
|
231
|
+
│ ├── metrics.py # Failure rate, recovery rate, MTTR, percentiles
|
|
232
|
+
│ ├── grader.py # Keyword / answer-presence grading
|
|
233
|
+
│ ├── statistics.py # Wilson CIs, Cohen's d/h, comparisons
|
|
234
|
+
│ └── metadata.py # Reproducibility metadata capture
|
|
235
|
+
└── agents/
|
|
236
|
+
├── base.py # BaseAgent protocol: invoke(dict) -> dict, reset()
|
|
237
|
+
├── langgraph_agent.py # LangGraph ReAct agent
|
|
238
|
+
├── langchain_agent.py # LangChain create_agent (compiles to LangGraph internally)
|
|
239
|
+
└── autogen_agent.py # AutoGen two-agent pipeline (pyautogen 0.1.14 + compat shim)
|
|
240
|
+
```
|
|
241
|
+
|
|
242
|
+
See [ARCHITECTURE.md](https://github.com/chhayankjain/AgentProbe/blob/main/ARCHITECTURE.md) for the data flow and design decisions.
|
|
243
|
+
|
|
244
|
+
## Observability Stack
|
|
245
|
+
|
|
246
|
+
```bash
|
|
247
|
+
# Start Prometheus + Grafana + Jaeger
|
|
248
|
+
docker compose up -d
|
|
249
|
+
|
|
250
|
+
# Grafana: http://localhost:3000 (admin/admin)
|
|
251
|
+
# Jaeger: http://localhost:16686
|
|
252
|
+
# Prometheus: http://localhost:9090
|
|
253
|
+
```
|
|
254
|
+
|
|
255
|
+
```python
|
|
256
|
+
from agentprobe.observer.dashboard import PrometheusExporter
|
|
257
|
+
|
|
258
|
+
exporter = PrometheusExporter(port=8000)
|
|
259
|
+
exporter.start() # /metrics on :8000
|
|
260
|
+
|
|
261
|
+
# Auto-record from benchmark run
|
|
262
|
+
exporter.record_benchmark_run(results)
|
|
263
|
+
```
|
|
264
|
+
|
|
265
|
+
## Supported Frameworks & LLMs
|
|
266
|
+
|
|
267
|
+
| Framework | Adapter | Validated in paper | Notes |
|
|
268
|
+
|-----------|---------|--------------------|-------|
|
|
269
|
+
| LangGraph | `LangGraphToolAgent` | Yes (Llama 3.1, Mistral 7B, Qwen2.5 7B) | Hand-built ReAct graph |
|
|
270
|
+
| LangChain | `LangChainToolAgent` | Yes (same models) | Uses `create_agent`, which compiles to LangGraph; shares an execution path with the LangGraph adapter, so results converge |
|
|
271
|
+
| AutoGen | `AutoGenToolAgent` | Yes, with caveats | Targets **pyautogen 0.1.14** through a `functions`→`tools` compatibility shim. Phase 1 found it silently bypassed tools; Phase 2 patched that. Modern `ag2` / `autogen-agentchat` 0.4+ is not yet supported ([roadmap](https://github.com/chhayankjain/AgentProbe/issues/38)) |
|
|
272
|
+
|
|
273
|
+
| LLM Backend | Install extra | Validated in paper | Notes |
|
|
274
|
+
|-------------|---------------|--------------------|-------|
|
|
275
|
+
| Ollama (Llama 3.1 8B, Mistral 7B, Qwen2.5 7B, Q4_K_M) | `.[ollama]` | Yes | Local, zero API cost. The Phase 2, qualification, and diagnostic scripts are Ollama-only |
|
|
276
|
+
| OpenAI (GPT-4o etc.) | `.[openai]` | **No** | Wired into `run_benchmarks.py --llm openai` for LangGraph/LangChain only; never run for the paper. See [#26](https://github.com/chhayankjain/AgentProbe/issues/26) |
|
|
277
|
+
| Anthropic (Claude) | `.[anthropic]` | **No** | Same as OpenAI |
|
|
278
|
+
|
|
279
|
+
## Running Tests
|
|
280
|
+
|
|
281
|
+
```bash
|
|
282
|
+
pip install -e ".[dev,plotting]" # from a clone
|
|
283
|
+
pytest tests/ -v
|
|
284
|
+
|
|
285
|
+
# Skip slow/integration tests
|
|
286
|
+
pytest tests/ -v -m "not slow and not integration"
|
|
287
|
+
|
|
288
|
+
# Lint
|
|
289
|
+
ruff check agentprobe/ tests/
|
|
290
|
+
```
|
|
291
|
+
|
|
292
|
+
## Reproducing Paper Results
|
|
293
|
+
|
|
294
|
+
The paper reports a two-phase study using Llama 3.1 8B (Q4_K_M) via Ollama, with injection seed 42 throughout.
|
|
295
|
+
|
|
296
|
+
### Phase 1 — Discovery (1,050 runs)
|
|
297
|
+
|
|
298
|
+
Phase 1 uses `run_benchmarks.py` with 30 runs for LangGraph/LangChain and 10 for AutoGen across 3 task types and 5 fault conditions. This phase discovered that AutoGen never invokes tools due to a legacy API-parameter mismatch.
|
|
299
|
+
|
|
300
|
+
> **Historical reproduction note:** The current codebase includes the patched AutoGen adapter (`_patch_openai_compat()`). To reproduce the original Phase 1 bypass behavior, use the pre-patch source at tag `v1.0.0`. Running Phase 1 commands with the current code will produce patched AutoGen behavior (tools are called), not the historical bypass.
|
|
301
|
+
|
|
302
|
+
```bash
|
|
303
|
+
# LangGraph / LangChain (30 runs per condition)
|
|
304
|
+
python experiments/run_benchmarks.py \
|
|
305
|
+
--frameworks langgraph langchain \
|
|
306
|
+
--tasks tool_use rag multi_agent \
|
|
307
|
+
--failure-types none timeout malformed_output rate_limit api_error \
|
|
308
|
+
--n-runs 30 --llm ollama --model llama3.1 --save-results
|
|
309
|
+
|
|
310
|
+
# AutoGen (10 runs per condition)
|
|
311
|
+
python experiments/run_benchmarks.py \
|
|
312
|
+
--frameworks autogen \
|
|
313
|
+
--tasks tool_use rag multi_agent \
|
|
314
|
+
--failure-types none timeout malformed_output rate_limit api_error \
|
|
315
|
+
--n-runs 10 --llm ollama --model llama3.1 --save-results
|
|
316
|
+
```
|
|
317
|
+
|
|
318
|
+
### Phase 2 — Patched Validation (450 runs)
|
|
319
|
+
|
|
320
|
+
Phase 2 uses `run_experiment_v2.py` with the patched AutoGen adapter (proper `tool_name=` recording, `_patch_openai_compat()` for the functions/tools protocol gap). Runs 30 trials per condition across 3 frameworks × 5 fault conditions on tool-use tasks only, with matched p=0.20 injection probability.
|
|
321
|
+
|
|
322
|
+
```bash
|
|
323
|
+
python experiments/run_experiment_v2.py \
|
|
324
|
+
--frameworks langgraph langchain autogen \
|
|
325
|
+
--model llama3.1 --seed 42
|
|
326
|
+
```
|
|
327
|
+
|
|
328
|
+
### Qualification Experiment (12 trials)
|
|
329
|
+
|
|
330
|
+
A small, targeted validation with deterministic calculator tasks and guaranteed fault injection (p=1.0). Produces a self-contained JSON artifact with inputs, outputs, tool events, injection log entries (with args/returns), and linked OTel trace span IDs.
|
|
331
|
+
|
|
332
|
+
```bash
|
|
333
|
+
python experiments/run_qualification.py \
|
|
334
|
+
--frameworks langgraph langchain autogen \
|
|
335
|
+
--model llama3.1
|
|
336
|
+
```
|
|
337
|
+
|
|
338
|
+
### Cross-Model Validation
|
|
339
|
+
|
|
340
|
+
Repeats the qualification and Phase 2 protocol on additional models (Mistral 7B, Qwen2.5 7B):
|
|
341
|
+
|
|
342
|
+
```bash
|
|
343
|
+
# Smoke test: verify tool calling works
|
|
344
|
+
python experiments/smoke_test_model.py --model mistral:7b
|
|
345
|
+
python experiments/smoke_test_model.py --model qwen2.5:7b
|
|
346
|
+
|
|
347
|
+
# Qualification (12 trials per model)
|
|
348
|
+
python experiments/run_qualification.py --model mistral:7b
|
|
349
|
+
python experiments/run_qualification.py --model qwen2.5:7b
|
|
350
|
+
|
|
351
|
+
# Full Phase 2 (450 runs per model)
|
|
352
|
+
python experiments/run_experiment_v2.py --model mistral:7b --seed 42
|
|
353
|
+
python experiments/run_experiment_v2.py --model qwen2.5:7b --seed 42
|
|
354
|
+
|
|
355
|
+
# Cross-model analysis
|
|
356
|
+
python experiments/analyze_cross_model.py \
|
|
357
|
+
--batches experiments/results_v2/20261111_113514 experiments/results_v2/20261111_233435 experiments/results_v2/20261112_040156 \
|
|
358
|
+
--models llama3.1 mistral:7b qwen2.5:7b
|
|
359
|
+
```
|
|
360
|
+
|
|
361
|
+
Results are saved to `experiments/results_v2/`.
|
|
362
|
+
|
|
363
|
+
### AutoGen Diagnostic (Direct Client vs Shim)
|
|
364
|
+
|
|
365
|
+
A diagnostic comparing the AutoGen adapter pipeline against a direct OpenAI-compatible client on the same three deterministic tasks. The compared paths differ in message format, stopping policy, and temperature handling; results are consistent with the adapter layer being a major contributor to low answer-string presence but do not isolate a single cause:
|
|
366
|
+
|
|
367
|
+
```bash
|
|
368
|
+
# Llama 3.1
|
|
369
|
+
python experiments/autogen_diagnostic.py --model llama3.1
|
|
370
|
+
|
|
371
|
+
# Qwen2.5 7B
|
|
372
|
+
python experiments/autogen_diagnostic.py --model qwen2.5:7b
|
|
373
|
+
```
|
|
374
|
+
|
|
375
|
+
Diagnostic artifacts are saved to `experiments/results_v2/diagnostic_*/`.
|
|
376
|
+
|
|
377
|
+
### Analysis
|
|
378
|
+
|
|
379
|
+
Phase 2 uses the corrected summaries under `experiments/results_v2/`. The original Phase 1 output (now at `experiments/archive_v1/results/combined_summary.json`) contains known counting bugs and is kept only for the historical record. The paper's tables are derived from:
|
|
380
|
+
- **Phase 2 batch (Llama 3.1):** `experiments/results_v2/20261111_113514/`
|
|
381
|
+
- **Phase 2 batch (Mistral 7B):** `experiments/results_v2/20261111_233435/`
|
|
382
|
+
- **Phase 2 batch (Qwen2.5 7B LG/LC):** `experiments/results_v2/20261112_040156/`
|
|
383
|
+
- **Phase 2 batch (Qwen2.5 7B AutoGen rerun):** `experiments/results_v2/20261112_063451/` (corrected extractor)
|
|
384
|
+
- **Qualification (Llama 3.1):** `experiments/results_v2/qualification_20261111_211913/` (12 trials, 10 passed)
|
|
385
|
+
- **Qualification (Mistral 7B):** `experiments/results_v2/qualification_20261111_233116/` (12 trials, 8 passed)
|
|
386
|
+
- **Qualification (Qwen2.5 7B):** `experiments/results_v2/qualification_20261111_233320/` (12 trials, 10 passed)
|
|
387
|
+
|
|
388
|
+
> **Source map:** The authoritative mapping from each paper table cell to its batch directory is in [`experiments/results_v2/paper_source_map.json`](https://github.com/chhayankjain/AgentProbe/blob/main/experiments/results_v2/paper_source_map.json). The Qwen2.5 column combines two batches: LangGraph/LangChain from `20261112_040156` and AutoGen from `20261112_063451` (corrected output extractor). Llama and Mistral use their single original batches. The `analyze_cross_model.py` script does not automatically merge these; use the source map to select the correct batch per framework.
|
|
389
|
+
|
|
390
|
+
> **Model digests:** All three models use Q4_K_M quantization. Ollama digest prefixes: Llama `46e0c10c039e`, Mistral `6577803aa9a0`, Qwen `845dbda0ea48`. The `experiment_metadata.json` files in historical batches record `model_digest: unknown` because digest capture was added after the experiments ran; the digests listed here were identified post-hoc from the same Ollama installation.
|
|
391
|
+
|
|
392
|
+
## Citation
|
|
393
|
+
|
|
394
|
+
```bibtex
|
|
395
|
+
@misc{jain2026agentprobe,
|
|
396
|
+
title={AgentProbe: Systematic Fault Injection and Observability for Evaluating the Reliability of LLM-Based Intelligent Agents},
|
|
397
|
+
author={Jain, Chhayank and Pattnaik, Sarthak and Pinsky, Eugene},
|
|
398
|
+
year={2026},
|
|
399
|
+
note={Manuscript under review}
|
|
400
|
+
}
|
|
401
|
+
```
|
|
402
|
+
|
|
403
|
+
## License
|
|
404
|
+
|
|
405
|
+
MIT — see [LICENSE](https://github.com/chhayankjain/AgentProbe/blob/main/LICENSE).
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
agentprobe/__init__.py,sha256=V2P_wqbUYuf7cjuOs8d4xHruE27K_6zu9siIeRHhMdQ,1306
|
|
2
|
+
agentprobe/config.py,sha256=EothKhYpl4efnptkqZQ0qVFy0I2z6qu3f_KJ4sGL860,2254
|
|
3
|
+
agentprobe/agents/__init__.py,sha256=6XxPOs9UfsZbyLODeUsy8LstMZ-myyBiYmGSN4pA_pI,156
|
|
4
|
+
agentprobe/agents/autogen_agent.py,sha256=hPLnjFiwSxK6PpPVBxlh4s1gAiLS9eU2142pKEAmlPs,11397
|
|
5
|
+
agentprobe/agents/base.py,sha256=f1zRl6BXqLiHCnliDcr57S5pfPDqB0K9WF2OHuZCq7Q,3559
|
|
6
|
+
agentprobe/agents/langchain_agent.py,sha256=UHSeUGRXM_HIuqF5z7Lej6KiSGlAH0nybax67dIqh-4,4767
|
|
7
|
+
agentprobe/agents/langgraph_agent.py,sha256=LoRHtB3XuaapN5D2Bf-L2YT1VhHNHEM_n5tKTIrLNdw,7344
|
|
8
|
+
agentprobe/benchmark/__init__.py,sha256=WcXWFnmktf1c_MHwoG0OOQFOLe6CDdGbL_GFxZySOJQ,1594
|
|
9
|
+
agentprobe/benchmark/grader.py,sha256=JFfKdElVSpKTkv8vMb8et5ngREL8fRmWVNVfU0ossWk,5254
|
|
10
|
+
agentprobe/benchmark/metadata.py,sha256=NhBydzoatkmPW9JixLUETgUC9lY24b6Pc-PaTx2beGA,6240
|
|
11
|
+
agentprobe/benchmark/metrics.py,sha256=x_CugVAYC4pVHWz3V1Bag-_osP2RslH6a72FUQ1TquM,9308
|
|
12
|
+
agentprobe/benchmark/runner.py,sha256=yhXyWDeDIKiCpp8ZkwsEC4a4lBI5Zix2c8-o2RN7V6E,16855
|
|
13
|
+
agentprobe/benchmark/statistics.py,sha256=N9Jap6jvcMc1kEcWrvrnibSmCy4UyZhID9qFZkgzdc4,13079
|
|
14
|
+
agentprobe/benchmark/tasks/__init__.py,sha256=59VaPamxdiEFW3lVlhMNnTN5KkSjdOPlCqoCgx2sQFI,2792
|
|
15
|
+
agentprobe/benchmark/tasks/multi_agent.py,sha256=ANslRflqWIP3Q0PA1yRV1skuhZUYkALWESso9nWIsmk,7220
|
|
16
|
+
agentprobe/benchmark/tasks/rag.py,sha256=FroSK_6L2SQb9sVSL_2m3aMFtkJ7Wo1N9ZiHQNPoUnc,13248
|
|
17
|
+
agentprobe/benchmark/tasks/tool_use.py,sha256=j180cUx_V-BJJuNUjApg4FtCI8Yy858ZP9ko-_mDAb0,8289
|
|
18
|
+
agentprobe/injector/__init__.py,sha256=RxmUyWjtWBT-lpy9pbgJ_ATQWv0bMRF7Ch4x9ONiPL4,1986
|
|
19
|
+
agentprobe/injector/context_failure.py,sha256=GEjhyJlXR66ECF6TF0YZII29dKZNVIrotoKS1UGCPyg,11648
|
|
20
|
+
agentprobe/injector/latency_failure.py,sha256=Yt_HRRYa0L7T8T3y3jSOYpvxfyGGetRIh5zYMcd7Ro4,9998
|
|
21
|
+
agentprobe/injector/orchestration_failure.py,sha256=SuRcaWICUdyAWxxTxPxwP2NzYV30WWwIR8J_tBrV1Mw,11861
|
|
22
|
+
agentprobe/injector/tool_failure.py,sha256=5iZRe3n8AH_1cjmFjrArE6bMAOmNceUhRn1B8gnsOlc,14720
|
|
23
|
+
agentprobe/observer/__init__.py,sha256=b4QuKGj_PuVA8nrGvWt-9nXrRH7iFBJxwKcryMI0EUo,159
|
|
24
|
+
agentprobe/observer/classifier.py,sha256=3t269AdievPpEh3GlcWiYdPWDJMN70gINF88_0WwT4o,12214
|
|
25
|
+
agentprobe/observer/dashboard.py,sha256=eWVLg4fmt5e3YkKWZrfP306l_JoBkpIt7jmBaXCXBmo,7616
|
|
26
|
+
agentprobe/observer/tracer.py,sha256=s273NRTuo2gn19cZbLkd-cvJJaGVFa1581QqvRLZ92Y,4492
|
|
27
|
+
agentprobe_toolkit-1.2.0.dist-info/METADATA,sha256=fG3Mry-GoMK843HcLM4EpDykU_s3hS1ujBJJTOxujvk,20082
|
|
28
|
+
agentprobe_toolkit-1.2.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
29
|
+
agentprobe_toolkit-1.2.0.dist-info/licenses/LICENSE,sha256=2Z68-AdVu4cK46MWvYIl4qvcvcSoTY6c2rCcG2y5GyM,1103
|
|
30
|
+
agentprobe_toolkit-1.2.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Chhayank Jain, Sarthak Pattnaik, Eugene Pinsky
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|