agentprobe-toolkit 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentprobe/__init__.py +42 -0
- agentprobe/agents/__init__.py +5 -0
- agentprobe/agents/autogen_agent.py +308 -0
- agentprobe/agents/base.py +97 -0
- agentprobe/agents/langchain_agent.py +140 -0
- agentprobe/agents/langgraph_agent.py +203 -0
- agentprobe/benchmark/__init__.py +67 -0
- agentprobe/benchmark/grader.py +162 -0
- agentprobe/benchmark/metadata.py +214 -0
- agentprobe/benchmark/metrics.py +247 -0
- agentprobe/benchmark/runner.py +434 -0
- agentprobe/benchmark/statistics.py +459 -0
- agentprobe/benchmark/tasks/__init__.py +75 -0
- agentprobe/benchmark/tasks/multi_agent.py +191 -0
- agentprobe/benchmark/tasks/rag.py +309 -0
- agentprobe/benchmark/tasks/tool_use.py +228 -0
- agentprobe/config.py +67 -0
- agentprobe/injector/__init__.py +66 -0
- agentprobe/injector/context_failure.py +328 -0
- agentprobe/injector/latency_failure.py +275 -0
- agentprobe/injector/orchestration_failure.py +332 -0
- agentprobe/injector/tool_failure.py +404 -0
- agentprobe/observer/__init__.py +5 -0
- agentprobe/observer/classifier.py +320 -0
- agentprobe/observer/dashboard.py +216 -0
- agentprobe/observer/tracer.py +137 -0
- agentprobe_toolkit-1.2.0.dist-info/METADATA +405 -0
- agentprobe_toolkit-1.2.0.dist-info/RECORD +30 -0
- agentprobe_toolkit-1.2.0.dist-info/WHEEL +4 -0
- agentprobe_toolkit-1.2.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
"""Multi-agent benchmark task.
|
|
2
|
+
|
|
3
|
+
The MultiAgentTask presents a researcher + writer two-agent pipeline.
|
|
4
|
+
The researcher calls tools to gather information; the writer produces
|
|
5
|
+
a structured output. Used to benchmark:
|
|
6
|
+
- Handoff failures between agents
|
|
7
|
+
- State consistency across agent boundaries
|
|
8
|
+
- Cascading failures when the researcher agent fails
|
|
9
|
+
|
|
10
|
+
The task inputs are static and self-contained — no external data required.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass
|
|
20
|
+
class MultiAgentQuery:
|
|
21
|
+
query_id: str
|
|
22
|
+
input: str
|
|
23
|
+
researcher_tools: list[str]
|
|
24
|
+
expected_output_keywords: list[str]
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
MULTI_AGENT_QUERIES: list[MultiAgentQuery] = [
|
|
28
|
+
MultiAgentQuery(
|
|
29
|
+
query_id="ma_001",
|
|
30
|
+
input="Research the current state of LLM reliability in production and summarize findings.",
|
|
31
|
+
researcher_tools=["web_search"],
|
|
32
|
+
expected_output_keywords=["reliability", "production", "llm"],
|
|
33
|
+
),
|
|
34
|
+
MultiAgentQuery(
|
|
35
|
+
query_id="ma_002",
|
|
36
|
+
input="Find the top 3 open-source agent frameworks and write a comparison.",
|
|
37
|
+
researcher_tools=["web_search"],
|
|
38
|
+
expected_output_keywords=["langgraph", "autogen", "comparison"],
|
|
39
|
+
),
|
|
40
|
+
MultiAgentQuery(
|
|
41
|
+
query_id="ma_003",
|
|
42
|
+
input="Research observability tools for LLM systems and draft a feature comparison table.",
|
|
43
|
+
researcher_tools=["web_search"],
|
|
44
|
+
expected_output_keywords=["observability", "langsmith", "opentelemetry"],
|
|
45
|
+
),
|
|
46
|
+
MultiAgentQuery(
|
|
47
|
+
query_id="ma_004",
|
|
48
|
+
input="Find recent benchmarks comparing GPT-4o and Claude Sonnet on coding tasks.",
|
|
49
|
+
researcher_tools=["web_search"],
|
|
50
|
+
expected_output_keywords=["gpt", "claude", "coding", "benchmark"],
|
|
51
|
+
),
|
|
52
|
+
MultiAgentQuery(
|
|
53
|
+
query_id="ma_005",
|
|
54
|
+
input="Research MLSys 2025 workshop topics and draft a submission-ready abstract outline.",
|
|
55
|
+
researcher_tools=["web_search"],
|
|
56
|
+
expected_output_keywords=["mlsys", "workshop", "abstract"],
|
|
57
|
+
),
|
|
58
|
+
# --- Expanded queries (ma_006 – ma_015) ---
|
|
59
|
+
MultiAgentQuery(
|
|
60
|
+
query_id="ma_006",
|
|
61
|
+
input=(
|
|
62
|
+
"Research the latest advances in protein structure prediction "
|
|
63
|
+
"and write a technical blog post suitable for a machine learning audience."
|
|
64
|
+
),
|
|
65
|
+
researcher_tools=["web_search"],
|
|
66
|
+
expected_output_keywords=["protein", "alphafold", "structure", "prediction"],
|
|
67
|
+
),
|
|
68
|
+
MultiAgentQuery(
|
|
69
|
+
query_id="ma_007",
|
|
70
|
+
input=(
|
|
71
|
+
"Analyze the S&P 500 performance over the last quarter "
|
|
72
|
+
"and produce a risk assessment report with key takeaways."
|
|
73
|
+
),
|
|
74
|
+
researcher_tools=["web_search", "calculator"],
|
|
75
|
+
expected_output_keywords=["s&p", "return", "risk", "quarter"],
|
|
76
|
+
),
|
|
77
|
+
MultiAgentQuery(
|
|
78
|
+
query_id="ma_008",
|
|
79
|
+
input=(
|
|
80
|
+
"Research climate change mitigation strategies in urban areas "
|
|
81
|
+
"and draft a policy brief with three actionable recommendations."
|
|
82
|
+
),
|
|
83
|
+
researcher_tools=["web_search"],
|
|
84
|
+
expected_output_keywords=["climate", "urban", "mitigation", "recommendation"],
|
|
85
|
+
),
|
|
86
|
+
MultiAgentQuery(
|
|
87
|
+
query_id="ma_009",
|
|
88
|
+
input=(
|
|
89
|
+
"Find the top 5 most cited papers on prompt engineering in 2024 "
|
|
90
|
+
"and write a literature review section summarizing their contributions."
|
|
91
|
+
),
|
|
92
|
+
researcher_tools=["web_search"],
|
|
93
|
+
expected_output_keywords=["prompt", "engineering", "cited", "review"],
|
|
94
|
+
),
|
|
95
|
+
MultiAgentQuery(
|
|
96
|
+
query_id="ma_010",
|
|
97
|
+
input=(
|
|
98
|
+
"Research the trade-offs between fine-tuning and retrieval-augmented "
|
|
99
|
+
"generation for domain-specific tasks, then produce a decision matrix "
|
|
100
|
+
"that a practitioner could use to choose between them."
|
|
101
|
+
),
|
|
102
|
+
researcher_tools=["web_search"],
|
|
103
|
+
expected_output_keywords=["fine-tuning", "rag", "trade-off", "decision"],
|
|
104
|
+
),
|
|
105
|
+
MultiAgentQuery(
|
|
106
|
+
query_id="ma_011",
|
|
107
|
+
input=(
|
|
108
|
+
"One agent should propose a software architecture for a real-time "
|
|
109
|
+
"fraud detection system, and another agent should critique the design "
|
|
110
|
+
"and suggest improvements."
|
|
111
|
+
),
|
|
112
|
+
researcher_tools=["web_search"],
|
|
113
|
+
expected_output_keywords=["fraud", "architecture", "critique", "improvement"],
|
|
114
|
+
),
|
|
115
|
+
MultiAgentQuery(
|
|
116
|
+
query_id="ma_012",
|
|
117
|
+
input=(
|
|
118
|
+
"Plan a three-phase rollout strategy for deploying an LLM-powered "
|
|
119
|
+
"customer support chatbot, then write a risk register for each phase."
|
|
120
|
+
),
|
|
121
|
+
researcher_tools=["web_search"],
|
|
122
|
+
expected_output_keywords=["rollout", "phase", "risk", "chatbot"],
|
|
123
|
+
),
|
|
124
|
+
MultiAgentQuery(
|
|
125
|
+
query_id="ma_013",
|
|
126
|
+
input=(
|
|
127
|
+
"Research the energy consumption of training large language models "
|
|
128
|
+
"and draft an executive summary with both environmental costs "
|
|
129
|
+
"and proposed efficiency improvements."
|
|
130
|
+
),
|
|
131
|
+
researcher_tools=["web_search", "calculator"],
|
|
132
|
+
expected_output_keywords=["energy", "training", "efficiency", "carbon"],
|
|
133
|
+
),
|
|
134
|
+
MultiAgentQuery(
|
|
135
|
+
query_id="ma_014",
|
|
136
|
+
input=(
|
|
137
|
+
"A researcher agent must gather data on global semiconductor supply "
|
|
138
|
+
"chain disruptions in 2024, while a writer agent must produce a "
|
|
139
|
+
"concise briefing document with sourced claims. The writer should "
|
|
140
|
+
"flag any claims that lack sufficient evidence."
|
|
141
|
+
),
|
|
142
|
+
researcher_tools=["web_search"],
|
|
143
|
+
expected_output_keywords=["semiconductor", "supply chain", "disruption", "source"],
|
|
144
|
+
),
|
|
145
|
+
MultiAgentQuery(
|
|
146
|
+
query_id="ma_015",
|
|
147
|
+
input=(
|
|
148
|
+
"Research both the benefits and risks of autonomous AI agents "
|
|
149
|
+
"in healthcare diagnostics, then produce a balanced position paper "
|
|
150
|
+
"with arguments for and against deployment."
|
|
151
|
+
),
|
|
152
|
+
researcher_tools=["web_search"],
|
|
153
|
+
expected_output_keywords=["healthcare", "autonomous", "benefit", "risk"],
|
|
154
|
+
),
|
|
155
|
+
]
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
class MultiAgentTask:
|
|
159
|
+
"""Benchmark task: two-agent researcher + writer pipeline.
|
|
160
|
+
|
|
161
|
+
The task tests handoff reliability and state propagation
|
|
162
|
+
between agents in a sequential multi-agent workflow.
|
|
163
|
+
"""
|
|
164
|
+
|
|
165
|
+
name = "multi_agent"
|
|
166
|
+
description = "Two-agent researcher+writer pipeline benchmark"
|
|
167
|
+
|
|
168
|
+
def __init__(self) -> None:
|
|
169
|
+
self._queries = list(MULTI_AGENT_QUERIES)
|
|
170
|
+
|
|
171
|
+
def get_inputs(self) -> list[dict[str, Any]]:
|
|
172
|
+
return [
|
|
173
|
+
{
|
|
174
|
+
"input": q.input,
|
|
175
|
+
"task": self.name,
|
|
176
|
+
"query_id": q.query_id,
|
|
177
|
+
"pipeline": "researcher->writer",
|
|
178
|
+
}
|
|
179
|
+
for q in self._queries
|
|
180
|
+
]
|
|
181
|
+
|
|
182
|
+
def grade(self, query_id: str, output: str) -> bool:
|
|
183
|
+
query = next((q for q in self._queries if q.query_id == query_id), None)
|
|
184
|
+
if query is None:
|
|
185
|
+
return False
|
|
186
|
+
output_lower = output.lower()
|
|
187
|
+
expected = query.expected_output_keywords
|
|
188
|
+
return any(kw.lower() in output_lower for kw in expected)
|
|
189
|
+
|
|
190
|
+
def __len__(self) -> int:
|
|
191
|
+
return len(self._queries)
|
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
"""RAG (Retrieval-Augmented Generation) benchmark task.
|
|
2
|
+
|
|
3
|
+
The RAGTask presents the agent with questions that require retrieving
|
|
4
|
+
context from a document corpus before generating an answer. Used to
|
|
5
|
+
benchmark context-layer failures, including:
|
|
6
|
+
- Context window overflow under long document injection
|
|
7
|
+
- Lost state when retrieval history is dropped
|
|
8
|
+
- Hallucinated history injection
|
|
9
|
+
|
|
10
|
+
The corpus is a static set of AgentProbe paper excerpts — fully
|
|
11
|
+
self-contained, no external API or database required.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from dataclasses import dataclass
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
# ---------------------------------------------------------------------------
|
|
20
|
+
# Static in-memory corpus (paper excerpts — reproducible, no external deps)
|
|
21
|
+
# ---------------------------------------------------------------------------
|
|
22
|
+
|
|
23
|
+
CORPUS: list[dict[str, str]] = [
|
|
24
|
+
{
|
|
25
|
+
"doc_id": "agentprobe_s1",
|
|
26
|
+
"title": "Introduction",
|
|
27
|
+
"content": (
|
|
28
|
+
"Agentic LLM systems — systems where a language model iteratively "
|
|
29
|
+
"calls tools, plans, and executes multi-step tasks — are rapidly "
|
|
30
|
+
"moving from research demonstrations to production deployments. "
|
|
31
|
+
"Unlike single-call LLM APIs, agentic systems introduce new failure "
|
|
32
|
+
"modes at the orchestration, tool-use, and context layers that are "
|
|
33
|
+
"poorly understood and under-instrumented."
|
|
34
|
+
),
|
|
35
|
+
},
|
|
36
|
+
{
|
|
37
|
+
"doc_id": "agentprobe_s2",
|
|
38
|
+
"title": "Failure Taxonomy",
|
|
39
|
+
"content": (
|
|
40
|
+
"We classify agentic failures into five categories: "
|
|
41
|
+
"(1) Tool call failures — API timeouts, malformed outputs, rate limits; "
|
|
42
|
+
"(2) Orchestration failures — infinite loops, wrong branching, state corruption; "
|
|
43
|
+
"(3) Context failures — context window overflow, lost state, hallucinated history; "
|
|
44
|
+
"(4) Latency failures — P99 spikes, cascading slowdowns, cold start delays; "
|
|
45
|
+
"(5) Consistency failures — non-deterministic outputs, divergent multi-agent state."
|
|
46
|
+
),
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
"doc_id": "agentprobe_s3",
|
|
50
|
+
"title": "AgentProbe Design",
|
|
51
|
+
"content": (
|
|
52
|
+
"AgentProbe provides four core modules: a failure injector, an observability "
|
|
53
|
+
"layer based on OpenTelemetry, a benchmark suite with standard tasks, and "
|
|
54
|
+
"a metrics calculator. Reliability is measured using MTTR (mean time to "
|
|
55
|
+
"recovery), failure rate, recovery rate, and P99 latency distributions."
|
|
56
|
+
),
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
"doc_id": "agentprobe_s4",
|
|
60
|
+
"title": "Experimental Setup",
|
|
61
|
+
"content": (
|
|
62
|
+
"Experiments compare LangGraph, LangChain, and AutoGen across three query groups: "
|
|
63
|
+
"tool-use, retrieval-style, and multi-agent-style prompts. The benchmark uses "
|
|
64
|
+
"Llama-3 via Ollama (local, zero API cost) as the default LLM backend. "
|
|
65
|
+
"All experiments are reproducible from the public repository."
|
|
66
|
+
),
|
|
67
|
+
},
|
|
68
|
+
{
|
|
69
|
+
"doc_id": "agentprobe_s5",
|
|
70
|
+
"title": "Results",
|
|
71
|
+
"content": (
|
|
72
|
+
"Under tool-call fault injection, error rates varied by framework and "
|
|
73
|
+
"configuration. Recovery rates depended on injection probability and fault type. "
|
|
74
|
+
"Trace analysis revealed that some framework-model pairings silently bypass "
|
|
75
|
+
"tool execution, inflating apparent resilience. "
|
|
76
|
+
"P99 latency increased substantially under fault injection for all frameworks."
|
|
77
|
+
),
|
|
78
|
+
},
|
|
79
|
+
# --- Expanded corpus documents (s6 – s10) ---
|
|
80
|
+
{
|
|
81
|
+
"doc_id": "agentprobe_s6",
|
|
82
|
+
"title": "Fault Injection Methodology",
|
|
83
|
+
"content": (
|
|
84
|
+
"Fault injection in AgentProbe operates at the tool-call boundary. "
|
|
85
|
+
"Injectors wrap tool functions with configurable failure modes: timeout "
|
|
86
|
+
"delays, malformed JSON responses, HTTP 429 rate-limit errors, and "
|
|
87
|
+
"generic API errors. Each injector is stateless and composable, allowing "
|
|
88
|
+
"multiple failure modes to be layered in a single experiment run. "
|
|
89
|
+
"Injection probability is configurable from 0% to 100%."
|
|
90
|
+
),
|
|
91
|
+
},
|
|
92
|
+
{
|
|
93
|
+
"doc_id": "agentprobe_s7",
|
|
94
|
+
"title": "Observability Patterns",
|
|
95
|
+
"content": (
|
|
96
|
+
"The observability layer exports OpenTelemetry spans for every agent step, "
|
|
97
|
+
"tool invocation, and LLM call. Spans carry structured attributes including "
|
|
98
|
+
"failure type, retry count, token usage, and latency. A Prometheus exporter "
|
|
99
|
+
"aggregates step-level metrics into counters, histograms, and gauges. "
|
|
100
|
+
"Grafana dashboards visualize failure rates, MTTR trends, and P99 latency "
|
|
101
|
+
"distributions in real time."
|
|
102
|
+
),
|
|
103
|
+
},
|
|
104
|
+
{
|
|
105
|
+
"doc_id": "agentprobe_s8",
|
|
106
|
+
"title": "LLM Agent Architectures",
|
|
107
|
+
"content": (
|
|
108
|
+
"Three agent architectures are evaluated: LangGraph uses a finite state "
|
|
109
|
+
"machine with explicit edges and conditional branching. LangChain uses a "
|
|
110
|
+
"ReAct-style agent loop with an AgentExecutor that iterates until a final "
|
|
111
|
+
"answer is produced. AutoGen uses a conversational multi-agent protocol "
|
|
112
|
+
"where agents exchange messages in a group chat. Each architecture handles "
|
|
113
|
+
"tool failures differently: LangGraph retries via graph edges, LangChain "
|
|
114
|
+
"relies on the LLM to self-correct, and AutoGen delegates to peer agents."
|
|
115
|
+
),
|
|
116
|
+
},
|
|
117
|
+
{
|
|
118
|
+
"doc_id": "agentprobe_s9",
|
|
119
|
+
"title": "Chaos Engineering for AI",
|
|
120
|
+
"content": (
|
|
121
|
+
"Chaos engineering principles — originally developed for distributed systems "
|
|
122
|
+
"by Netflix — are adapted for agentic AI. The steady-state hypothesis is "
|
|
123
|
+
"defined as the agent completing its task within a latency budget and "
|
|
124
|
+
"producing a correct answer. Perturbations include tool failures, context "
|
|
125
|
+
"window exhaustion, and inter-agent message loss. Blast radius is limited "
|
|
126
|
+
"by running experiments in isolated sandbox environments with mock tools."
|
|
127
|
+
),
|
|
128
|
+
},
|
|
129
|
+
{
|
|
130
|
+
"doc_id": "agentprobe_s10",
|
|
131
|
+
"title": "Benchmark Design Principles",
|
|
132
|
+
"content": (
|
|
133
|
+
"Benchmark tasks are designed for reproducibility, coverage, and minimal "
|
|
134
|
+
"external dependencies. Each task provides static inputs, deterministic "
|
|
135
|
+
"grading via keyword matching, and pre-defined tool schemas. The three "
|
|
136
|
+
"task categories — tool-use, RAG, and multi-agent — cover single-agent "
|
|
137
|
+
"tool invocation, retrieval-augmented generation with a static corpus, "
|
|
138
|
+
"and multi-agent coordination pipelines respectively. Task difficulty is "
|
|
139
|
+
"controlled by the number of required tool calls and reasoning steps."
|
|
140
|
+
),
|
|
141
|
+
},
|
|
142
|
+
]
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
@dataclass
|
|
146
|
+
class RAGQuery:
|
|
147
|
+
query_id: str
|
|
148
|
+
input: str
|
|
149
|
+
relevant_doc_ids: list[str]
|
|
150
|
+
expected_keywords: list[str]
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
RAG_QUERIES: list[RAGQuery] = [
|
|
154
|
+
RAGQuery(
|
|
155
|
+
query_id="rag_001",
|
|
156
|
+
input="What failure categories does AgentProbe define?",
|
|
157
|
+
relevant_doc_ids=["agentprobe_s2"],
|
|
158
|
+
expected_keywords=["tool", "orchestration", "context", "latency", "consistency"],
|
|
159
|
+
),
|
|
160
|
+
RAGQuery(
|
|
161
|
+
query_id="rag_002",
|
|
162
|
+
input="How does AgentProbe measure agent reliability?",
|
|
163
|
+
relevant_doc_ids=["agentprobe_s3"],
|
|
164
|
+
expected_keywords=["mttr", "failure rate", "p99", "recovery"],
|
|
165
|
+
),
|
|
166
|
+
RAGQuery(
|
|
167
|
+
query_id="rag_003",
|
|
168
|
+
input="Which agent framework had the lowest failure rate under timeout injection?",
|
|
169
|
+
relevant_doc_ids=["agentprobe_s5"],
|
|
170
|
+
expected_keywords=["error", "framework", "fault"],
|
|
171
|
+
),
|
|
172
|
+
RAGQuery(
|
|
173
|
+
query_id="rag_004",
|
|
174
|
+
input="What LLMs are used in the AgentProbe experiments?",
|
|
175
|
+
relevant_doc_ids=["agentprobe_s4"],
|
|
176
|
+
expected_keywords=["llama", "gpt", "claude", "ollama"],
|
|
177
|
+
),
|
|
178
|
+
RAGQuery(
|
|
179
|
+
query_id="rag_005",
|
|
180
|
+
input="Why are agentic systems harder to observe than single-call LLM APIs?",
|
|
181
|
+
relevant_doc_ids=["agentprobe_s1"],
|
|
182
|
+
expected_keywords=["orchestration", "tool-use", "context", "failure"],
|
|
183
|
+
),
|
|
184
|
+
# --- Expanded queries (rag_006 – rag_015) ---
|
|
185
|
+
RAGQuery(
|
|
186
|
+
query_id="rag_006",
|
|
187
|
+
input="How does AgentProbe inject faults at the tool-call boundary?",
|
|
188
|
+
relevant_doc_ids=["agentprobe_s6"],
|
|
189
|
+
expected_keywords=["timeout", "malformed", "rate-limit", "injection"],
|
|
190
|
+
),
|
|
191
|
+
RAGQuery(
|
|
192
|
+
query_id="rag_007",
|
|
193
|
+
input="What observability signals does AgentProbe export via OpenTelemetry?",
|
|
194
|
+
relevant_doc_ids=["agentprobe_s7"],
|
|
195
|
+
expected_keywords=["spans", "opentelemetry", "prometheus", "latency"],
|
|
196
|
+
),
|
|
197
|
+
RAGQuery(
|
|
198
|
+
query_id="rag_008",
|
|
199
|
+
input="Compare how LangGraph, LangChain, and AutoGen handle tool failures.",
|
|
200
|
+
relevant_doc_ids=["agentprobe_s8", "agentprobe_s5"],
|
|
201
|
+
expected_keywords=["retry", "langgraph", "langchain", "autogen"],
|
|
202
|
+
),
|
|
203
|
+
RAGQuery(
|
|
204
|
+
query_id="rag_009",
|
|
205
|
+
input="How are chaos engineering principles adapted for agentic AI systems?",
|
|
206
|
+
relevant_doc_ids=["agentprobe_s9"],
|
|
207
|
+
expected_keywords=["chaos", "steady-state", "perturbation", "blast radius"],
|
|
208
|
+
),
|
|
209
|
+
RAGQuery(
|
|
210
|
+
query_id="rag_010",
|
|
211
|
+
input="What design principles guide the AgentProbe benchmark tasks?",
|
|
212
|
+
relevant_doc_ids=["agentprobe_s10"],
|
|
213
|
+
expected_keywords=["reproducibility", "coverage", "keyword", "grading"],
|
|
214
|
+
),
|
|
215
|
+
RAGQuery(
|
|
216
|
+
query_id="rag_011",
|
|
217
|
+
input=(
|
|
218
|
+
"Synthesize the fault injection methodology and observability patterns: "
|
|
219
|
+
"how do injected faults become visible in the monitoring stack?"
|
|
220
|
+
),
|
|
221
|
+
relevant_doc_ids=["agentprobe_s6", "agentprobe_s7"],
|
|
222
|
+
expected_keywords=["injection", "spans", "failure type", "prometheus"],
|
|
223
|
+
),
|
|
224
|
+
RAGQuery(
|
|
225
|
+
query_id="rag_012",
|
|
226
|
+
input="What is the injection probability range supported by AgentProbe?",
|
|
227
|
+
relevant_doc_ids=["agentprobe_s6"],
|
|
228
|
+
expected_keywords=["0%", "100%", "configurable"],
|
|
229
|
+
),
|
|
230
|
+
RAGQuery(
|
|
231
|
+
query_id="rag_013",
|
|
232
|
+
input=(
|
|
233
|
+
"Which agent architecture uses a finite state machine, and what advantage "
|
|
234
|
+
"does this provide for retry handling?"
|
|
235
|
+
),
|
|
236
|
+
relevant_doc_ids=["agentprobe_s8"],
|
|
237
|
+
expected_keywords=["langgraph", "state machine", "edges", "retry"],
|
|
238
|
+
),
|
|
239
|
+
RAGQuery(
|
|
240
|
+
query_id="rag_014",
|
|
241
|
+
input=(
|
|
242
|
+
"How does AgentProbe limit the blast radius of chaos experiments, "
|
|
243
|
+
"and why is this important?"
|
|
244
|
+
),
|
|
245
|
+
relevant_doc_ids=["agentprobe_s9"],
|
|
246
|
+
expected_keywords=["sandbox", "mock", "isolated", "blast radius"],
|
|
247
|
+
),
|
|
248
|
+
RAGQuery(
|
|
249
|
+
query_id="rag_015",
|
|
250
|
+
input=(
|
|
251
|
+
"Across all sections, what metrics does AgentProbe use to quantify "
|
|
252
|
+
"agent reliability and performance?"
|
|
253
|
+
),
|
|
254
|
+
relevant_doc_ids=["agentprobe_s3", "agentprobe_s7"],
|
|
255
|
+
expected_keywords=["mttr", "failure rate", "p99", "histogram"],
|
|
256
|
+
),
|
|
257
|
+
]
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
class RAGTask:
|
|
261
|
+
"""Benchmark task: RAG agent with static in-memory corpus.
|
|
262
|
+
|
|
263
|
+
No external vector database required — retrieval is keyword-based over
|
|
264
|
+
the static CORPUS for fully reproducible benchmarking.
|
|
265
|
+
"""
|
|
266
|
+
|
|
267
|
+
name = "rag"
|
|
268
|
+
description = "RAG agent benchmark with static document corpus"
|
|
269
|
+
|
|
270
|
+
def __init__(self) -> None:
|
|
271
|
+
self._queries = list(RAG_QUERIES)
|
|
272
|
+
self._corpus = {doc["doc_id"]: doc for doc in CORPUS}
|
|
273
|
+
|
|
274
|
+
def retrieve(self, query: str, top_k: int = 2) -> list[dict[str, str]]:
|
|
275
|
+
"""Simple keyword-overlap retrieval (no embeddings required)."""
|
|
276
|
+
query_tokens = set(query.lower().split())
|
|
277
|
+
scored: list[tuple[float, dict[str, str]]] = []
|
|
278
|
+
for doc in self._corpus.values():
|
|
279
|
+
doc_tokens = set(doc["content"].lower().split())
|
|
280
|
+
overlap = len(query_tokens & doc_tokens) / max(len(query_tokens), 1)
|
|
281
|
+
scored.append((overlap, doc))
|
|
282
|
+
scored.sort(key=lambda x: x[0], reverse=True)
|
|
283
|
+
return [doc for _, doc in scored[:top_k]]
|
|
284
|
+
|
|
285
|
+
def get_inputs(self) -> list[dict[str, Any]]:
|
|
286
|
+
"""Return task inputs with pre-retrieved context."""
|
|
287
|
+
inputs = []
|
|
288
|
+
for q in self._queries:
|
|
289
|
+
context_docs = self.retrieve(q.input)
|
|
290
|
+
context = "\n\n".join(f"[{d['title']}]: {d['content']}" for d in context_docs)
|
|
291
|
+
inputs.append(
|
|
292
|
+
{
|
|
293
|
+
"input": q.input,
|
|
294
|
+
"context": context,
|
|
295
|
+
"task": self.name,
|
|
296
|
+
"query_id": q.query_id,
|
|
297
|
+
}
|
|
298
|
+
)
|
|
299
|
+
return inputs
|
|
300
|
+
|
|
301
|
+
def grade(self, query_id: str, output: str) -> bool:
|
|
302
|
+
query = next((q for q in self._queries if q.query_id == query_id), None)
|
|
303
|
+
if query is None:
|
|
304
|
+
return False
|
|
305
|
+
output_lower = output.lower()
|
|
306
|
+
return any(kw.lower() in output_lower for kw in query.expected_keywords)
|
|
307
|
+
|
|
308
|
+
def __len__(self) -> int:
|
|
309
|
+
return len(self._queries)
|