agentprobe-toolkit 1.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentprobe_toolkit-1.2.0/.gitignore +73 -0
- agentprobe_toolkit-1.2.0/CHANGELOG.md +76 -0
- agentprobe_toolkit-1.2.0/CITATION.cff +39 -0
- agentprobe_toolkit-1.2.0/LICENSE +21 -0
- agentprobe_toolkit-1.2.0/PKG-INFO +405 -0
- agentprobe_toolkit-1.2.0/README.md +347 -0
- agentprobe_toolkit-1.2.0/agentprobe/__init__.py +42 -0
- agentprobe_toolkit-1.2.0/agentprobe/agents/__init__.py +5 -0
- agentprobe_toolkit-1.2.0/agentprobe/agents/autogen_agent.py +308 -0
- agentprobe_toolkit-1.2.0/agentprobe/agents/base.py +97 -0
- agentprobe_toolkit-1.2.0/agentprobe/agents/langchain_agent.py +140 -0
- agentprobe_toolkit-1.2.0/agentprobe/agents/langgraph_agent.py +203 -0
- agentprobe_toolkit-1.2.0/agentprobe/benchmark/__init__.py +67 -0
- agentprobe_toolkit-1.2.0/agentprobe/benchmark/grader.py +162 -0
- agentprobe_toolkit-1.2.0/agentprobe/benchmark/metadata.py +214 -0
- agentprobe_toolkit-1.2.0/agentprobe/benchmark/metrics.py +247 -0
- agentprobe_toolkit-1.2.0/agentprobe/benchmark/runner.py +434 -0
- agentprobe_toolkit-1.2.0/agentprobe/benchmark/statistics.py +459 -0
- agentprobe_toolkit-1.2.0/agentprobe/benchmark/tasks/__init__.py +75 -0
- agentprobe_toolkit-1.2.0/agentprobe/benchmark/tasks/multi_agent.py +191 -0
- agentprobe_toolkit-1.2.0/agentprobe/benchmark/tasks/rag.py +309 -0
- agentprobe_toolkit-1.2.0/agentprobe/benchmark/tasks/tool_use.py +228 -0
- agentprobe_toolkit-1.2.0/agentprobe/config.py +67 -0
- agentprobe_toolkit-1.2.0/agentprobe/injector/__init__.py +66 -0
- agentprobe_toolkit-1.2.0/agentprobe/injector/context_failure.py +328 -0
- agentprobe_toolkit-1.2.0/agentprobe/injector/latency_failure.py +275 -0
- agentprobe_toolkit-1.2.0/agentprobe/injector/orchestration_failure.py +332 -0
- agentprobe_toolkit-1.2.0/agentprobe/injector/tool_failure.py +404 -0
- agentprobe_toolkit-1.2.0/agentprobe/observer/__init__.py +5 -0
- agentprobe_toolkit-1.2.0/agentprobe/observer/classifier.py +320 -0
- agentprobe_toolkit-1.2.0/agentprobe/observer/dashboard.py +216 -0
- agentprobe_toolkit-1.2.0/agentprobe/observer/tracer.py +137 -0
- agentprobe_toolkit-1.2.0/pyproject.toml +131 -0
- agentprobe_toolkit-1.2.0/tests/__init__.py +0 -0
- agentprobe_toolkit-1.2.0/tests/test_benchmark_runner.py +552 -0
- agentprobe_toolkit-1.2.0/tests/test_benchmark_tasks.py +213 -0
- agentprobe_toolkit-1.2.0/tests/test_classifier.py +220 -0
- agentprobe_toolkit-1.2.0/tests/test_context_failure.py +125 -0
- agentprobe_toolkit-1.2.0/tests/test_experiment_state.py +150 -0
- agentprobe_toolkit-1.2.0/tests/test_grader.py +253 -0
- agentprobe_toolkit-1.2.0/tests/test_latency_failure.py +188 -0
- agentprobe_toolkit-1.2.0/tests/test_metadata.py +140 -0
- agentprobe_toolkit-1.2.0/tests/test_metrics.py +137 -0
- agentprobe_toolkit-1.2.0/tests/test_orchestration_failure.py +180 -0
- agentprobe_toolkit-1.2.0/tests/test_pilot_hung.py +189 -0
- agentprobe_toolkit-1.2.0/tests/test_public_api.py +284 -0
- agentprobe_toolkit-1.2.0/tests/test_qualification_checker.py +205 -0
- agentprobe_toolkit-1.2.0/tests/test_statistics.py +330 -0
- agentprobe_toolkit-1.2.0/tests/test_tool_failure.py +215 -0
- agentprobe_toolkit-1.2.0/tests/test_tracer.py +47 -0
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
*.so
|
|
6
|
+
.Python
|
|
7
|
+
build/
|
|
8
|
+
dist/
|
|
9
|
+
*.egg-info/
|
|
10
|
+
.eggs/
|
|
11
|
+
*.egg
|
|
12
|
+
MANIFEST
|
|
13
|
+
|
|
14
|
+
# Virtual environments
|
|
15
|
+
.venv/
|
|
16
|
+
venv/
|
|
17
|
+
env/
|
|
18
|
+
ENV/
|
|
19
|
+
|
|
20
|
+
# Testing
|
|
21
|
+
.pytest_cache/
|
|
22
|
+
.coverage
|
|
23
|
+
htmlcov/
|
|
24
|
+
.tox/
|
|
25
|
+
.mypy_cache/
|
|
26
|
+
.ruff_cache/
|
|
27
|
+
|
|
28
|
+
# Jupyter
|
|
29
|
+
.ipynb_checkpoints/
|
|
30
|
+
*.ipynb_metadata/
|
|
31
|
+
|
|
32
|
+
# Experiments & results
|
|
33
|
+
experiments/results/*.parquet
|
|
34
|
+
!experiments/results/.gitkeep
|
|
35
|
+
|
|
36
|
+
# Secrets / API keys
|
|
37
|
+
.env
|
|
38
|
+
.env.*
|
|
39
|
+
!.env.example
|
|
40
|
+
*.pem
|
|
41
|
+
*.key
|
|
42
|
+
|
|
43
|
+
# OS
|
|
44
|
+
.DS_Store
|
|
45
|
+
Thumbs.db
|
|
46
|
+
|
|
47
|
+
# IDE
|
|
48
|
+
.vscode/
|
|
49
|
+
.idea/
|
|
50
|
+
*.swp
|
|
51
|
+
*.swo
|
|
52
|
+
|
|
53
|
+
# Local caches
|
|
54
|
+
.claude/
|
|
55
|
+
.cache/
|
|
56
|
+
|
|
57
|
+
# Prometheus / Grafana local data
|
|
58
|
+
prometheus_data/
|
|
59
|
+
grafana_data/
|
|
60
|
+
|
|
61
|
+
# OpenTelemetry collector local data
|
|
62
|
+
otel-collector-data/
|
|
63
|
+
|
|
64
|
+
# LaTeX build artifacts
|
|
65
|
+
paper/*.aux
|
|
66
|
+
paper/*.log
|
|
67
|
+
paper/*.out
|
|
68
|
+
paper/*.pdf
|
|
69
|
+
paper/*.bbl
|
|
70
|
+
paper/*.blg
|
|
71
|
+
paper/*.fls
|
|
72
|
+
paper/*.fdb_latexmk
|
|
73
|
+
paper/*.synctex.gz
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## [1.2.0] - 2026-11-14
|
|
4
|
+
|
|
5
|
+
First release published to PyPI, as **`agentprobe-toolkit`** (the name `agentprobe` was already
|
|
6
|
+
taken by an unrelated project). The import name is unchanged: `import agentprobe`.
|
|
7
|
+
|
|
8
|
+
### Added
|
|
9
|
+
- PyPI distribution `agentprobe-toolkit` with Trusted Publishing release workflow
|
|
10
|
+
(`.github/workflows/release.yml`): tag `vX.Y.Z` publishes to PyPI; manual dispatch publishes to TestPyPI
|
|
11
|
+
- Packaging check in CI (`python -m build`, `twine check`, wheel import smoke test)
|
|
12
|
+
- Python 3.14 in the CI matrix and classifiers
|
|
13
|
+
|
|
14
|
+
### Fixed
|
|
15
|
+
- `from agentprobe import ToolFailureInjector, AgentProbeTracer` (the README example) now works: the top-level package re-exports the public API
|
|
16
|
+
- Removed the duplicate Pydantic `ToolFailureConfig`/`ToolFailureType` in `agentprobe/injector/types.py`; `agentprobe.injector` now exports the dataclass versions the injector actually accepts (the old export crashed `ToolFailureInjector` with `AttributeError`)
|
|
17
|
+
- `BaseAgent` protocol now matches the real adapter surface (`invoke(dict) -> dict`, `reset()`, `get_injector_summary()`); all three adapters gained `reset()` and an `injector` property
|
|
18
|
+
- Dependency floors corrected: `langchain>=1.0.0`, `langchain-core>=1.0.0`, `langgraph>=1.0.0` (the LangChain adapter needs `create_agent`); unused `langchain-community` dropped
|
|
19
|
+
- Tracer no longer suggests a non-existent `agentprobe[otlp]` extra
|
|
20
|
+
- Runner progress summary uses the same ceiling nearest-rank P99 as `MetricsCalculator`
|
|
21
|
+
|
|
22
|
+
### Added (API)
|
|
23
|
+
- `agentprobe[autogen]` extra pinning `pyautogen==0.1.14` and `openai<1.0`
|
|
24
|
+
- `agentprobe.benchmark.tasks.get_task()` and `corpus_task_fn()` to run the 55-query graded corpora through `BenchmarkRunner`; `run_benchmarks.py --query-set corpus`
|
|
25
|
+
- `runner.PAPER_PROMPTS` / `paper_task_fn` naming the five-prompt set every published batch used
|
|
26
|
+
|
|
27
|
+
### Removed
|
|
28
|
+
- Dead Pydantic schema layer `agentprobe/benchmark/types.py` (`TaskType`, `AgentFramework`, Pydantic `RunResult`/`BenchmarkConfig`/`BenchmarkResults`/`BenchmarkMetrics`) that no runner code used
|
|
29
|
+
|
|
30
|
+
### Changed
|
|
31
|
+
- README rewritten to state which taxonomy categories, frameworks, and LLM backends were actually validated in the paper, with metric definitions
|
|
32
|
+
- `ROADMAP.md` replaced by a pointer to the GitHub roadmap issue (the file described a pre-v1.0 build plan)
|
|
33
|
+
|
|
34
|
+
## [1.1.0] - 2026-11-14
|
|
35
|
+
|
|
36
|
+
### Added
|
|
37
|
+
- Phase 2 experiment runner (`run_experiment_v2.py`) with matched injection probability and answer-presence grading
|
|
38
|
+
- Qualification runner (`run_qualification.py`) for deterministic pipeline validation at p=1.0
|
|
39
|
+
- Model smoke test (`smoke_test_model.py`) for quick tool-calling verification
|
|
40
|
+
- AutoGen diagnostic (`autogen_diagnostic.py`) comparing direct OpenAI client vs AutoGen shim
|
|
41
|
+
- Cross-model analysis tooling (`analyze_cross_model.py`) and data-driven figure generation
|
|
42
|
+
- Checkpoint validation with exact record count, run ID range, and condition identity matching
|
|
43
|
+
- Resume safety: reject metadata mismatches (model, seed, n_runs) on experiment resume
|
|
44
|
+
- Full experiment data for Llama 3.1, Mistral 7B, and Qwen2.5 7B (Phase 2, qualification, diagnostic)
|
|
45
|
+
- Phase 1 data archived to `experiments/archive_v1/`
|
|
46
|
+
- Paper source map (`paper_source_map.json`) for batch-to-table provenance
|
|
47
|
+
- ORCID identifiers in CITATION.cff
|
|
48
|
+
|
|
49
|
+
### Fixed
|
|
50
|
+
- AutoGen adapter: patch legacy `functions`/`function_call` format for Ollama compatibility
|
|
51
|
+
- AutoGen output extraction: select correct assistant message instead of empty tool-call messages
|
|
52
|
+
- Numeric answer validation: word-boundary regex to prevent substring false positives (e.g., "12000" matching "120000")
|
|
53
|
+
- `recovery_rate` returns `None` (not `0.0`) when no failures occur in a condition
|
|
54
|
+
- P99 percentile: switched from floor-based to ceiling nearest-rank
|
|
55
|
+
- `None` recovery rate handling in reporting scripts, plot generation, and analysis
|
|
56
|
+
- Figure output directory aligned with LaTeX `\graphicspath`
|
|
57
|
+
- Pilot hung-invocation control flow and empty-output parsing
|
|
58
|
+
|
|
59
|
+
### Changed
|
|
60
|
+
- Manuscript updated with two-phase study, cross-model validation, and MDPI/MAKE formatting
|
|
61
|
+
- Phase 1 latency ratio corrected from 7x to 11x
|
|
62
|
+
- Scope claims narrowed to experimentally validated fault categories
|
|
63
|
+
- Removed uninformative chi-square test from identical-implementation comparison
|
|
64
|
+
- LaTeX commands use `\providecommand` for Overleaf/mdpi.cls compatibility
|
|
65
|
+
|
|
66
|
+
## [1.0.0] - 2026-09-13
|
|
67
|
+
|
|
68
|
+
### Added
|
|
69
|
+
- Initial public release
|
|
70
|
+
- Five-category failure taxonomy (Tool Call, Orchestration, Context, Latency, Consistency)
|
|
71
|
+
- Tool call fault injectors: timeout, malformed output, rate limit, API error
|
|
72
|
+
- OpenTelemetry-based tracing with Jaeger/Prometheus integration
|
|
73
|
+
- Benchmark runner with MTTR, failure rate, and P99 latency metrics
|
|
74
|
+
- Agent adapters for LangGraph, LangChain, and AutoGen
|
|
75
|
+
- 15 benchmark queries across tool use, RAG, and multi-agent tasks
|
|
76
|
+
- Phase 1 experiment data (1,050 runs)
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
cff-version: 1.2.0
|
|
2
|
+
message: "If you use this software, please cite it as below."
|
|
3
|
+
type: software
|
|
4
|
+
title: "AgentProbe: Systematic Fault Injection and Observability for Evaluating the Reliability of LLM-Based Intelligent Agents"
|
|
5
|
+
authors:
|
|
6
|
+
- family-names: Jain
|
|
7
|
+
given-names: Chhayank
|
|
8
|
+
affiliation: Arizona State University
|
|
9
|
+
orcid: "https://orcid.org/0009-0006-6831-426X"
|
|
10
|
+
- family-names: Pattnaik
|
|
11
|
+
given-names: Sarthak
|
|
12
|
+
affiliation: Boston University
|
|
13
|
+
orcid: "https://orcid.org/0009-0004-4994-4800"
|
|
14
|
+
- family-names: Pinsky
|
|
15
|
+
given-names: Eugene
|
|
16
|
+
affiliation: Boston University
|
|
17
|
+
orcid: "https://orcid.org/0000-0002-3836-1851"
|
|
18
|
+
version: "1.2.0"
|
|
19
|
+
date-released: "2026-11-14"
|
|
20
|
+
license: MIT
|
|
21
|
+
repository-code: "https://github.com/chhayankjain/AgentProbe"
|
|
22
|
+
url: "https://pypi.org/project/agentprobe-toolkit/"
|
|
23
|
+
keywords:
|
|
24
|
+
- llm-agents
|
|
25
|
+
- fault-injection
|
|
26
|
+
- benchmarking
|
|
27
|
+
- observability
|
|
28
|
+
- reliability
|
|
29
|
+
- chaos-engineering
|
|
30
|
+
- langchain
|
|
31
|
+
- langgraph
|
|
32
|
+
- autogen
|
|
33
|
+
- opentelemetry
|
|
34
|
+
abstract: >-
|
|
35
|
+
AgentProbe is an open-source fault-injection and observability toolkit
|
|
36
|
+
for systematically evaluating the reliability of LLM-based intelligent
|
|
37
|
+
agents. It implements a five-category failure taxonomy with configurable
|
|
38
|
+
injectors, OpenTelemetry-based tracing, and benchmark queries
|
|
39
|
+
spanning tool use, retrieval-style, and multi-agent-style prompts.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Chhayank Jain, Sarthak Pattnaik, Eugene Pinsky
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,405 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: agentprobe-toolkit
|
|
3
|
+
Version: 1.2.0
|
|
4
|
+
Summary: Fault injection and OpenTelemetry observability toolkit for evaluating the reliability of LLM agents (LangGraph, LangChain, AutoGen)
|
|
5
|
+
Project-URL: Homepage, https://github.com/chhayankjain/AgentProbe
|
|
6
|
+
Project-URL: Repository, https://github.com/chhayankjain/AgentProbe
|
|
7
|
+
Project-URL: Issues, https://github.com/chhayankjain/AgentProbe/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/chhayankjain/AgentProbe/blob/main/CHANGELOG.md
|
|
9
|
+
Project-URL: Roadmap, https://github.com/chhayankjain/AgentProbe/issues/38
|
|
10
|
+
Author: Chhayank Jain, Sarthak Pattnaik, Eugene Pinsky
|
|
11
|
+
Maintainer: Chhayank Jain
|
|
12
|
+
License-Expression: MIT
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Keywords: agents,autogen,benchmarking,chaos-engineering,fault-injection,langchain,langgraph,llm,observability,opentelemetry,reliability
|
|
15
|
+
Classifier: Development Status :: 4 - Beta
|
|
16
|
+
Classifier: Intended Audience :: Science/Research
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
22
|
+
Classifier: Topic :: Software Development :: Testing
|
|
23
|
+
Classifier: Topic :: System :: Monitoring
|
|
24
|
+
Classifier: Typing :: Typed
|
|
25
|
+
Requires-Python: >=3.11
|
|
26
|
+
Requires-Dist: langchain-core>=1.0.0
|
|
27
|
+
Requires-Dist: langchain>=1.0.0
|
|
28
|
+
Requires-Dist: langgraph>=1.0.0
|
|
29
|
+
Requires-Dist: numpy>=1.26.0
|
|
30
|
+
Requires-Dist: opentelemetry-api>=1.27.0
|
|
31
|
+
Requires-Dist: opentelemetry-exporter-otlp-proto-grpc>=1.27.0
|
|
32
|
+
Requires-Dist: opentelemetry-exporter-prometheus>=0.48b0
|
|
33
|
+
Requires-Dist: opentelemetry-sdk>=1.27.0
|
|
34
|
+
Requires-Dist: prometheus-client>=0.21.0
|
|
35
|
+
Requires-Dist: pydantic>=2.0.0
|
|
36
|
+
Requires-Dist: rich>=13.0.0
|
|
37
|
+
Requires-Dist: scipy>=1.12.0
|
|
38
|
+
Provides-Extra: anthropic
|
|
39
|
+
Requires-Dist: langchain-anthropic>=0.2.0; extra == 'anthropic'
|
|
40
|
+
Provides-Extra: autogen
|
|
41
|
+
Requires-Dist: openai<1.0; extra == 'autogen'
|
|
42
|
+
Requires-Dist: pyautogen==0.1.14; extra == 'autogen'
|
|
43
|
+
Provides-Extra: dev
|
|
44
|
+
Requires-Dist: ipykernel>=6.29.0; extra == 'dev'
|
|
45
|
+
Requires-Dist: mypy>=1.11.0; extra == 'dev'
|
|
46
|
+
Requires-Dist: nbformat>=5.10.0; extra == 'dev'
|
|
47
|
+
Requires-Dist: pytest-asyncio>=0.24.0; extra == 'dev'
|
|
48
|
+
Requires-Dist: pytest>=8.0.0; extra == 'dev'
|
|
49
|
+
Requires-Dist: ruff>=0.7.0; extra == 'dev'
|
|
50
|
+
Provides-Extra: ollama
|
|
51
|
+
Requires-Dist: langchain-ollama>=0.2.0; extra == 'ollama'
|
|
52
|
+
Provides-Extra: openai
|
|
53
|
+
Requires-Dist: langchain-openai>=0.2.0; extra == 'openai'
|
|
54
|
+
Provides-Extra: plotting
|
|
55
|
+
Requires-Dist: matplotlib>=3.8.0; extra == 'plotting'
|
|
56
|
+
Requires-Dist: pandas>=2.0.0; extra == 'plotting'
|
|
57
|
+
Description-Content-Type: text/markdown
|
|
58
|
+
|
|
59
|
+
# AgentProbe
|
|
60
|
+
|
|
61
|
+
[](https://github.com/chhayankjain/AgentProbe/actions/workflows/ci.yml)
|
|
62
|
+
[](https://github.com/chhayankjain/AgentProbe/actions/workflows/codeql.yml)
|
|
63
|
+
[](https://www.python.org/downloads/)
|
|
64
|
+
[](https://github.com/chhayankjain/AgentProbe/blob/main/LICENSE)
|
|
65
|
+
[](https://pypi.org/project/agentprobe-toolkit/)
|
|
66
|
+
[](https://github.com/chhayankjain/AgentProbe/releases)
|
|
67
|
+
|
|
68
|
+
**Fault injection and observability toolkit for LLM agents**
|
|
69
|
+
|
|
70
|
+
AgentProbe wraps an agent's tools with seeded, logged fault injectors (timeouts, malformed output, rate limits, API errors), traces every invocation and tool call with OpenTelemetry, and reports failure rate, recovery rate, MTTR, and latency percentiles across frameworks. It ships adapters for LangGraph, LangChain, and AutoGen, plus the scripts and data behind the companion paper.
|
|
71
|
+
|
|
72
|
+
Companion to the paper: *"AgentProbe: Systematic Fault Injection and Observability for Evaluating the Reliability of LLM-Based Intelligent Agents"* (submitted to MAKE, MDPI, 2026).
|
|
73
|
+
|
|
74
|
+
---
|
|
75
|
+
|
|
76
|
+
## Failure Taxonomy
|
|
77
|
+
|
|
78
|
+
AgentProbe organises agent failures into five categories. Only the first has been exercised in a published experiment; the status column says exactly what exists for each.
|
|
79
|
+
|
|
80
|
+
| # | Category | Examples | Status in this repo |
|
|
81
|
+
|---|----------|----------|---------------------|
|
|
82
|
+
| 1 | **Tool Call** | API timeouts, malformed output, rate limits (HTTP 429), server errors (5xx) | Injector implemented, **used in all paper experiments** (`injector/tool_failure.py`) |
|
|
83
|
+
| 2 | **Orchestration** | Infinite loops, wrong branching, state corruption, dropped handoffs | Injector implemented and unit-tested, not yet used in an experiment (`injector/orchestration_failure.py`) |
|
|
84
|
+
| 3 | **Context** | Context window overflow, lost history, hallucinated history | Injector implemented and unit-tested, not yet used in an experiment (`injector/context_failure.py`) |
|
|
85
|
+
| 4 | **Latency** | P99 spikes, cascading slowdowns, jitter | Injector implemented; latency in the paper is a by-product of tool faults, not independently injected (`injector/latency_failure.py`) |
|
|
86
|
+
| 5 | **Consistency** | Non-deterministic outputs, divergent multi-agent state | Detected only, via a Jaccard-similarity classifier; nothing is injected (`observer/classifier.py`) |
|
|
87
|
+
|
|
88
|
+
Note that the current "malformed output" injector raises an exception at the tool boundary rather than returning corrupted data to the model, so it tests the agent's exception path, not its parsing. See issue [#28](https://github.com/chhayankjain/AgentProbe/issues/28).
|
|
89
|
+
|
|
90
|
+
---
|
|
91
|
+
|
|
92
|
+
## Installation
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
# From PyPI (LangGraph + LangChain adapters, local Ollama backend)
|
|
96
|
+
pip install "agentprobe-toolkit[ollama]"
|
|
97
|
+
|
|
98
|
+
# From a clone, for development or to run the paper's experiments
|
|
99
|
+
git clone https://github.com/chhayankjain/AgentProbe.git && cd AgentProbe
|
|
100
|
+
pip install -e ".[dev,plotting,ollama]"
|
|
101
|
+
|
|
102
|
+
# The AutoGen adapter targets pyautogen 0.1.14 and needs openai<1.0, which
|
|
103
|
+
# conflicts with langchain-openai. Install it in its own environment:
|
|
104
|
+
# pip install "agentprobe-toolkit[ollama,autogen]"
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
The distribution is named `agentprobe-toolkit` because `agentprobe` was already taken on PyPI by an unrelated project. The import name is `agentprobe`.
|
|
108
|
+
|
|
109
|
+
## Quick Start
|
|
110
|
+
|
|
111
|
+
```bash
|
|
112
|
+
|
|
113
|
+
# Start Ollama with Llama 3.1 (used in all paper experiments)
|
|
114
|
+
ollama pull llama3.1
|
|
115
|
+
|
|
116
|
+
# For cross-model validation (optional)
|
|
117
|
+
# ollama pull mistral:7b
|
|
118
|
+
# ollama pull qwen2.5:7b
|
|
119
|
+
|
|
120
|
+
# Run baseline benchmark (no failures)
|
|
121
|
+
python experiments/run_benchmarks.py --frameworks langgraph --tasks tool_use --baseline-only
|
|
122
|
+
|
|
123
|
+
# Run with failure injection
|
|
124
|
+
python experiments/run_benchmarks.py \
|
|
125
|
+
--frameworks langgraph langchain \
|
|
126
|
+
--tasks tool_use rag \
|
|
127
|
+
--failure-types none timeout rate_limit \
|
|
128
|
+
--n-runs 50 \
|
|
129
|
+
--save-results
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
## Programmatic Usage
|
|
133
|
+
|
|
134
|
+
```python
|
|
135
|
+
from langchain_ollama import ChatOllama
|
|
136
|
+
|
|
137
|
+
from agentprobe import (
|
|
138
|
+
AgentProbeTracer,
|
|
139
|
+
BenchmarkConfig,
|
|
140
|
+
BenchmarkRunner,
|
|
141
|
+
MetricsCalculator,
|
|
142
|
+
ToolFailureConfig,
|
|
143
|
+
ToolFailureType,
|
|
144
|
+
)
|
|
145
|
+
from agentprobe.agents.langgraph_agent import LangGraphToolAgent
|
|
146
|
+
|
|
147
|
+
# 1. Set up tracing (console by default; OTLP endpoint optional)
|
|
148
|
+
tracer = AgentProbeTracer(service_name="my-experiment", otlp_endpoint="http://localhost:4317")
|
|
149
|
+
|
|
150
|
+
# 2. Configure failure injection
|
|
151
|
+
failure_config = ToolFailureConfig(
|
|
152
|
+
failure_type=ToolFailureType.TIMEOUT,
|
|
153
|
+
failure_probability=0.2, # 20% of tool calls will time out
|
|
154
|
+
timeout_seconds=5.0,
|
|
155
|
+
seed=42, # reproducible injection sequence
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
# 3. Build an agent; it wraps its tools with a ToolFailureInjector internally
|
|
159
|
+
llm = ChatOllama(model="llama3.1", temperature=0.0)
|
|
160
|
+
agent = LangGraphToolAgent(llm=llm, failure_config=failure_config)
|
|
161
|
+
|
|
162
|
+
# 4. Run the benchmark (defaults to the paper's five-prompt set)
|
|
163
|
+
config = BenchmarkConfig(
|
|
164
|
+
framework="langgraph",
|
|
165
|
+
task="tool_use",
|
|
166
|
+
n_runs=50,
|
|
167
|
+
failure_config=failure_config,
|
|
168
|
+
recovery_attempts=1,
|
|
169
|
+
save_results_path="experiments/results/langgraph_tool_use_timeout",
|
|
170
|
+
)
|
|
171
|
+
results = BenchmarkRunner(agent=agent, config=config).run()
|
|
172
|
+
|
|
173
|
+
# 5. Compute metrics
|
|
174
|
+
metrics = MetricsCalculator().compute(results)
|
|
175
|
+
print(f"Failure rate: {metrics.failure_rate:.1%}")
|
|
176
|
+
print(f"Recovery rate: {metrics.recovery_rate}")
|
|
177
|
+
print(f"MTTR: {metrics.mttr_ms:.0f}ms")
|
|
178
|
+
print(f"P99 latency: {metrics.p99_latency_ms:.0f}ms")
|
|
179
|
+
print(agent.get_injector_summary())
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
### Injecting faults into your own tools
|
|
183
|
+
|
|
184
|
+
The injector is framework-agnostic. Wrap any callable, or any LangChain `BaseTool`:
|
|
185
|
+
|
|
186
|
+
```python
|
|
187
|
+
from agentprobe import ToolFailureConfig, ToolFailureInjector, ToolFailureType
|
|
188
|
+
|
|
189
|
+
injector = ToolFailureInjector(
|
|
190
|
+
ToolFailureConfig(failure_type=ToolFailureType.RATE_LIMIT, failure_probability=0.3, seed=7)
|
|
191
|
+
)
|
|
192
|
+
|
|
193
|
+
@injector.wrap
|
|
194
|
+
def lookup_price(sku: str) -> float:
|
|
195
|
+
...
|
|
196
|
+
|
|
197
|
+
faulty_tool = injector.inject_langchain_tool(my_langchain_tool) # in-place, keeps schema
|
|
198
|
+
print(injector.summary()) # total_calls, injected_failures, failures_by_type, ...
|
|
199
|
+
print(injector.injection_log) # one record per call with args, return/exception, span_id
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
### Two prompt sets
|
|
203
|
+
|
|
204
|
+
- **Paper prompts (default).** `BenchmarkRunner` cycles the five prompts per task in `agentprobe.benchmark.runner.PAPER_PROMPTS`. Every batch in the paper used these.
|
|
205
|
+
- **Graded corpora.** `agentprobe.benchmark.tasks` holds 25 tool-use, 15 RAG, and 15 multi-agent queries with keyword graders. They have not been used in a published experiment. Run them with `task_fn=corpus_task_fn("tool_use")` or `run_benchmarks.py --query-set corpus`.
|
|
206
|
+
|
|
207
|
+
### Metric definitions
|
|
208
|
+
|
|
209
|
+
- **Failure**: the agent invocation raised an exception that reached the harness. A wrong-but-exception-free answer is *not* a failure here; use the grader for that.
|
|
210
|
+
- **Recovery**: the harness re-invoked the agent once (injector still active) and that call succeeded. This is a harness-level retry, not agent-internal self-repair.
|
|
211
|
+
- **MTTR**: mean latency of successful re-invocations, in ms.
|
|
212
|
+
- **P50/P90/P99**: ceiling nearest-rank percentiles of per-run latency. For n < 100, P99 equals the sample maximum. Injected timeouts sleep for real, so latency under the timeout condition includes the injected delay.
|
|
213
|
+
- **Recovery rate** is `None` when a condition had no failures.
|
|
214
|
+
|
|
215
|
+
## Architecture
|
|
216
|
+
|
|
217
|
+
```
|
|
218
|
+
agentprobe/
|
|
219
|
+
├── injector/
|
|
220
|
+
│ ├── tool_failure.py # Category 1: tool call failures
|
|
221
|
+
│ ├── orchestration_failure.py # Category 2: orchestration failures
|
|
222
|
+
│ ├── context_failure.py # Category 3: context failures
|
|
223
|
+
│ └── latency_failure.py # Category 4: latency failures
|
|
224
|
+
├── observer/
|
|
225
|
+
│ ├── tracer.py # OpenTelemetry setup + span utilities
|
|
226
|
+
│ ├── classifier.py # Rule-based failure classification
|
|
227
|
+
│ └── dashboard.py # Prometheus metrics exporter
|
|
228
|
+
├── benchmark/
|
|
229
|
+
│ ├── runner.py # BenchmarkRunner + PAPER_PROMPTS (the paper's prompt set)
|
|
230
|
+
│ ├── tasks/ # Larger graded corpora (not used in the paper)
|
|
231
|
+
│ ├── metrics.py # Failure rate, recovery rate, MTTR, percentiles
|
|
232
|
+
│ ├── grader.py # Keyword / answer-presence grading
|
|
233
|
+
│ ├── statistics.py # Wilson CIs, Cohen's d/h, comparisons
|
|
234
|
+
│ └── metadata.py # Reproducibility metadata capture
|
|
235
|
+
└── agents/
|
|
236
|
+
├── base.py # BaseAgent protocol: invoke(dict) -> dict, reset()
|
|
237
|
+
├── langgraph_agent.py # LangGraph ReAct agent
|
|
238
|
+
├── langchain_agent.py # LangChain create_agent (compiles to LangGraph internally)
|
|
239
|
+
└── autogen_agent.py # AutoGen two-agent pipeline (pyautogen 0.1.14 + compat shim)
|
|
240
|
+
```
|
|
241
|
+
|
|
242
|
+
See [ARCHITECTURE.md](https://github.com/chhayankjain/AgentProbe/blob/main/ARCHITECTURE.md) for the data flow and design decisions.
|
|
243
|
+
|
|
244
|
+
## Observability Stack
|
|
245
|
+
|
|
246
|
+
```bash
|
|
247
|
+
# Start Prometheus + Grafana + Jaeger
|
|
248
|
+
docker compose up -d
|
|
249
|
+
|
|
250
|
+
# Grafana: http://localhost:3000 (admin/admin)
|
|
251
|
+
# Jaeger: http://localhost:16686
|
|
252
|
+
# Prometheus: http://localhost:9090
|
|
253
|
+
```
|
|
254
|
+
|
|
255
|
+
```python
|
|
256
|
+
from agentprobe.observer.dashboard import PrometheusExporter
|
|
257
|
+
|
|
258
|
+
exporter = PrometheusExporter(port=8000)
|
|
259
|
+
exporter.start() # /metrics on :8000
|
|
260
|
+
|
|
261
|
+
# Auto-record from benchmark run
|
|
262
|
+
exporter.record_benchmark_run(results)
|
|
263
|
+
```
|
|
264
|
+
|
|
265
|
+
## Supported Frameworks & LLMs
|
|
266
|
+
|
|
267
|
+
| Framework | Adapter | Validated in paper | Notes |
|
|
268
|
+
|-----------|---------|--------------------|-------|
|
|
269
|
+
| LangGraph | `LangGraphToolAgent` | Yes (Llama 3.1, Mistral 7B, Qwen2.5 7B) | Hand-built ReAct graph |
|
|
270
|
+
| LangChain | `LangChainToolAgent` | Yes (same models) | Uses `create_agent`, which compiles to LangGraph; shares an execution path with the LangGraph adapter, so results converge |
|
|
271
|
+
| AutoGen | `AutoGenToolAgent` | Yes, with caveats | Targets **pyautogen 0.1.14** through a `functions`→`tools` compatibility shim. Phase 1 found it silently bypassed tools; Phase 2 patched that. Modern `ag2` / `autogen-agentchat` 0.4+ is not yet supported ([roadmap](https://github.com/chhayankjain/AgentProbe/issues/38)) |
|
|
272
|
+
|
|
273
|
+
| LLM Backend | Install extra | Validated in paper | Notes |
|
|
274
|
+
|-------------|---------------|--------------------|-------|
|
|
275
|
+
| Ollama (Llama 3.1 8B, Mistral 7B, Qwen2.5 7B, Q4_K_M) | `.[ollama]` | Yes | Local, zero API cost. The Phase 2, qualification, and diagnostic scripts are Ollama-only |
|
|
276
|
+
| OpenAI (GPT-4o etc.) | `.[openai]` | **No** | Wired into `run_benchmarks.py --llm openai` for LangGraph/LangChain only; never run for the paper. See [#26](https://github.com/chhayankjain/AgentProbe/issues/26) |
|
|
277
|
+
| Anthropic (Claude) | `.[anthropic]` | **No** | Same as OpenAI |
|
|
278
|
+
|
|
279
|
+
## Running Tests
|
|
280
|
+
|
|
281
|
+
```bash
|
|
282
|
+
pip install -e ".[dev,plotting]" # from a clone
|
|
283
|
+
pytest tests/ -v
|
|
284
|
+
|
|
285
|
+
# Skip slow/integration tests
|
|
286
|
+
pytest tests/ -v -m "not slow and not integration"
|
|
287
|
+
|
|
288
|
+
# Lint
|
|
289
|
+
ruff check agentprobe/ tests/
|
|
290
|
+
```
|
|
291
|
+
|
|
292
|
+
## Reproducing Paper Results
|
|
293
|
+
|
|
294
|
+
The paper reports a two-phase study using Llama 3.1 8B (Q4_K_M) via Ollama, with injection seed 42 throughout.
|
|
295
|
+
|
|
296
|
+
### Phase 1 — Discovery (1,050 runs)
|
|
297
|
+
|
|
298
|
+
Phase 1 uses `run_benchmarks.py` with 30 runs for LangGraph/LangChain and 10 for AutoGen across 3 task types and 5 fault conditions. This phase discovered that AutoGen never invokes tools due to a legacy API-parameter mismatch.
|
|
299
|
+
|
|
300
|
+
> **Historical reproduction note:** The current codebase includes the patched AutoGen adapter (`_patch_openai_compat()`). To reproduce the original Phase 1 bypass behavior, use the pre-patch source at tag `v1.0.0`. Running Phase 1 commands with the current code will produce patched AutoGen behavior (tools are called), not the historical bypass.
|
|
301
|
+
|
|
302
|
+
```bash
|
|
303
|
+
# LangGraph / LangChain (30 runs per condition)
|
|
304
|
+
python experiments/run_benchmarks.py \
|
|
305
|
+
--frameworks langgraph langchain \
|
|
306
|
+
--tasks tool_use rag multi_agent \
|
|
307
|
+
--failure-types none timeout malformed_output rate_limit api_error \
|
|
308
|
+
--n-runs 30 --llm ollama --model llama3.1 --save-results
|
|
309
|
+
|
|
310
|
+
# AutoGen (10 runs per condition)
|
|
311
|
+
python experiments/run_benchmarks.py \
|
|
312
|
+
--frameworks autogen \
|
|
313
|
+
--tasks tool_use rag multi_agent \
|
|
314
|
+
--failure-types none timeout malformed_output rate_limit api_error \
|
|
315
|
+
--n-runs 10 --llm ollama --model llama3.1 --save-results
|
|
316
|
+
```
|
|
317
|
+
|
|
318
|
+
### Phase 2 — Patched Validation (450 runs)
|
|
319
|
+
|
|
320
|
+
Phase 2 uses `run_experiment_v2.py` with the patched AutoGen adapter (proper `tool_name=` recording, `_patch_openai_compat()` for the functions/tools protocol gap). Runs 30 trials per condition across 3 frameworks × 5 fault conditions on tool-use tasks only, with matched p=0.20 injection probability.
|
|
321
|
+
|
|
322
|
+
```bash
|
|
323
|
+
python experiments/run_experiment_v2.py \
|
|
324
|
+
--frameworks langgraph langchain autogen \
|
|
325
|
+
--model llama3.1 --seed 42
|
|
326
|
+
```
|
|
327
|
+
|
|
328
|
+
### Qualification Experiment (12 trials)
|
|
329
|
+
|
|
330
|
+
A small, targeted validation with deterministic calculator tasks and guaranteed fault injection (p=1.0). Produces a self-contained JSON artifact with inputs, outputs, tool events, injection log entries (with args/returns), and linked OTel trace span IDs.
|
|
331
|
+
|
|
332
|
+
```bash
|
|
333
|
+
python experiments/run_qualification.py \
|
|
334
|
+
--frameworks langgraph langchain autogen \
|
|
335
|
+
--model llama3.1
|
|
336
|
+
```
|
|
337
|
+
|
|
338
|
+
### Cross-Model Validation
|
|
339
|
+
|
|
340
|
+
Repeats the qualification and Phase 2 protocol on additional models (Mistral 7B, Qwen2.5 7B):
|
|
341
|
+
|
|
342
|
+
```bash
|
|
343
|
+
# Smoke test: verify tool calling works
|
|
344
|
+
python experiments/smoke_test_model.py --model mistral:7b
|
|
345
|
+
python experiments/smoke_test_model.py --model qwen2.5:7b
|
|
346
|
+
|
|
347
|
+
# Qualification (12 trials per model)
|
|
348
|
+
python experiments/run_qualification.py --model mistral:7b
|
|
349
|
+
python experiments/run_qualification.py --model qwen2.5:7b
|
|
350
|
+
|
|
351
|
+
# Full Phase 2 (450 runs per model)
|
|
352
|
+
python experiments/run_experiment_v2.py --model mistral:7b --seed 42
|
|
353
|
+
python experiments/run_experiment_v2.py --model qwen2.5:7b --seed 42
|
|
354
|
+
|
|
355
|
+
# Cross-model analysis
|
|
356
|
+
python experiments/analyze_cross_model.py \
|
|
357
|
+
--batches experiments/results_v2/20261111_113514 experiments/results_v2/20261111_233435 experiments/results_v2/20261112_040156 \
|
|
358
|
+
--models llama3.1 mistral:7b qwen2.5:7b
|
|
359
|
+
```
|
|
360
|
+
|
|
361
|
+
Results are saved to `experiments/results_v2/`.
|
|
362
|
+
|
|
363
|
+
### AutoGen Diagnostic (Direct Client vs Shim)
|
|
364
|
+
|
|
365
|
+
A diagnostic comparing the AutoGen adapter pipeline against a direct OpenAI-compatible client on the same three deterministic tasks. The compared paths differ in message format, stopping policy, and temperature handling; results are consistent with the adapter layer being a major contributor to low answer-string presence but do not isolate a single cause:
|
|
366
|
+
|
|
367
|
+
```bash
|
|
368
|
+
# Llama 3.1
|
|
369
|
+
python experiments/autogen_diagnostic.py --model llama3.1
|
|
370
|
+
|
|
371
|
+
# Qwen2.5 7B
|
|
372
|
+
python experiments/autogen_diagnostic.py --model qwen2.5:7b
|
|
373
|
+
```
|
|
374
|
+
|
|
375
|
+
Diagnostic artifacts are saved to `experiments/results_v2/diagnostic_*/`.
|
|
376
|
+
|
|
377
|
+
### Analysis
|
|
378
|
+
|
|
379
|
+
Phase 2 uses the corrected summaries under `experiments/results_v2/`. The original Phase 1 output (now at `experiments/archive_v1/results/combined_summary.json`) contains known counting bugs and is kept only for the historical record. The paper's tables are derived from:
|
|
380
|
+
- **Phase 2 batch (Llama 3.1):** `experiments/results_v2/20261111_113514/`
|
|
381
|
+
- **Phase 2 batch (Mistral 7B):** `experiments/results_v2/20261111_233435/`
|
|
382
|
+
- **Phase 2 batch (Qwen2.5 7B LG/LC):** `experiments/results_v2/20261112_040156/`
|
|
383
|
+
- **Phase 2 batch (Qwen2.5 7B AutoGen rerun):** `experiments/results_v2/20261112_063451/` (corrected extractor)
|
|
384
|
+
- **Qualification (Llama 3.1):** `experiments/results_v2/qualification_20261111_211913/` (12 trials, 10 passed)
|
|
385
|
+
- **Qualification (Mistral 7B):** `experiments/results_v2/qualification_20261111_233116/` (12 trials, 8 passed)
|
|
386
|
+
- **Qualification (Qwen2.5 7B):** `experiments/results_v2/qualification_20261111_233320/` (12 trials, 10 passed)
|
|
387
|
+
|
|
388
|
+
> **Source map:** The authoritative mapping from each paper table cell to its batch directory is in [`experiments/results_v2/paper_source_map.json`](https://github.com/chhayankjain/AgentProbe/blob/main/experiments/results_v2/paper_source_map.json). The Qwen2.5 column combines two batches: LangGraph/LangChain from `20261112_040156` and AutoGen from `20261112_063451` (corrected output extractor). Llama and Mistral use their single original batches. The `analyze_cross_model.py` script does not automatically merge these; use the source map to select the correct batch per framework.
|
|
389
|
+
|
|
390
|
+
> **Model digests:** All three models use Q4_K_M quantization. Ollama digest prefixes: Llama `46e0c10c039e`, Mistral `6577803aa9a0`, Qwen `845dbda0ea48`. The `experiment_metadata.json` files in historical batches record `model_digest: unknown` because digest capture was added after the experiments ran; the digests listed here were identified post-hoc from the same Ollama installation.
|
|
391
|
+
|
|
392
|
+
## Citation
|
|
393
|
+
|
|
394
|
+
```bibtex
|
|
395
|
+
@misc{jain2026agentprobe,
|
|
396
|
+
title={AgentProbe: Systematic Fault Injection and Observability for Evaluating the Reliability of LLM-Based Intelligent Agents},
|
|
397
|
+
author={Jain, Chhayank and Pattnaik, Sarthak and Pinsky, Eugene},
|
|
398
|
+
year={2026},
|
|
399
|
+
note={Manuscript under review}
|
|
400
|
+
}
|
|
401
|
+
```
|
|
402
|
+
|
|
403
|
+
## License
|
|
404
|
+
|
|
405
|
+
MIT — see [LICENSE](https://github.com/chhayankjain/AgentProbe/blob/main/LICENSE).
|