agentprobe-toolkit 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentprobe/__init__.py +42 -0
- agentprobe/agents/__init__.py +5 -0
- agentprobe/agents/autogen_agent.py +308 -0
- agentprobe/agents/base.py +97 -0
- agentprobe/agents/langchain_agent.py +140 -0
- agentprobe/agents/langgraph_agent.py +203 -0
- agentprobe/benchmark/__init__.py +67 -0
- agentprobe/benchmark/grader.py +162 -0
- agentprobe/benchmark/metadata.py +214 -0
- agentprobe/benchmark/metrics.py +247 -0
- agentprobe/benchmark/runner.py +434 -0
- agentprobe/benchmark/statistics.py +459 -0
- agentprobe/benchmark/tasks/__init__.py +75 -0
- agentprobe/benchmark/tasks/multi_agent.py +191 -0
- agentprobe/benchmark/tasks/rag.py +309 -0
- agentprobe/benchmark/tasks/tool_use.py +228 -0
- agentprobe/config.py +67 -0
- agentprobe/injector/__init__.py +66 -0
- agentprobe/injector/context_failure.py +328 -0
- agentprobe/injector/latency_failure.py +275 -0
- agentprobe/injector/orchestration_failure.py +332 -0
- agentprobe/injector/tool_failure.py +404 -0
- agentprobe/observer/__init__.py +5 -0
- agentprobe/observer/classifier.py +320 -0
- agentprobe/observer/dashboard.py +216 -0
- agentprobe/observer/tracer.py +137 -0
- agentprobe_toolkit-1.2.0.dist-info/METADATA +405 -0
- agentprobe_toolkit-1.2.0.dist-info/RECORD +30 -0
- agentprobe_toolkit-1.2.0.dist-info/WHEEL +4 -0
- agentprobe_toolkit-1.2.0.dist-info/licenses/LICENSE +21 -0
agentprobe/__init__.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""AgentProbe — fault injection and observability toolkit for LLM agents.
|
|
2
|
+
|
|
3
|
+
Usage::
|
|
4
|
+
|
|
5
|
+
from agentprobe import ToolFailureConfig, ToolFailureInjector, ToolFailureType
|
|
6
|
+
from agentprobe import AgentProbeTracer, BenchmarkConfig, BenchmarkRunner, MetricsCalculator
|
|
7
|
+
|
|
8
|
+
Framework adapters live in :mod:`agentprobe.agents` and are imported explicitly
|
|
9
|
+
(``from agentprobe.agents.langgraph_agent import LangGraphToolAgent``) so that
|
|
10
|
+
importing the package does not require every framework to be installed.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from agentprobe.benchmark.metrics import AgentMetrics, MetricsCalculator
|
|
14
|
+
from agentprobe.benchmark.runner import BenchmarkConfig, BenchmarkRunner, RunResult
|
|
15
|
+
from agentprobe.injector.tool_failure import (
|
|
16
|
+
ToolFailureConfig,
|
|
17
|
+
ToolFailureInjector,
|
|
18
|
+
ToolFailureType,
|
|
19
|
+
)
|
|
20
|
+
from agentprobe.observer.classifier import FailureCategory, FailureClassifier
|
|
21
|
+
from agentprobe.observer.tracer import AgentProbeTracer, get_tracer
|
|
22
|
+
|
|
23
|
+
__version__ = "1.2.0"
|
|
24
|
+
|
|
25
|
+
__all__ = [
|
|
26
|
+
"__version__",
|
|
27
|
+
# Injection
|
|
28
|
+
"ToolFailureConfig",
|
|
29
|
+
"ToolFailureInjector",
|
|
30
|
+
"ToolFailureType",
|
|
31
|
+
# Observability
|
|
32
|
+
"AgentProbeTracer",
|
|
33
|
+
"get_tracer",
|
|
34
|
+
"FailureClassifier",
|
|
35
|
+
"FailureCategory",
|
|
36
|
+
# Benchmark
|
|
37
|
+
"BenchmarkConfig",
|
|
38
|
+
"BenchmarkRunner",
|
|
39
|
+
"RunResult",
|
|
40
|
+
"MetricsCalculator",
|
|
41
|
+
"AgentMetrics",
|
|
42
|
+
]
|
|
@@ -0,0 +1,308 @@
|
|
|
1
|
+
"""AutoGen agent for AgentProbe benchmarking.
|
|
2
|
+
|
|
3
|
+
Implements a two-agent AutoGen pipeline (UserProxy + AssistantAgent)
|
|
4
|
+
using the same tool set as LangGraph and LangChain agents for
|
|
5
|
+
apples-to-apples comparison.
|
|
6
|
+
|
|
7
|
+
The AutoGen agent uses the OpenAI-compatible API endpoint, which can
|
|
8
|
+
point to an Ollama local server:
|
|
9
|
+
|
|
10
|
+
AUTOGEN_LLM_CONFIG = {
|
|
11
|
+
"config_list": [
|
|
12
|
+
{
|
|
13
|
+
"model": "llama3",
|
|
14
|
+
"base_url": "http://localhost:11434/v1",
|
|
15
|
+
"api_key": "ollama",
|
|
16
|
+
}
|
|
17
|
+
]
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
**Version pin:** this adapter targets ``pyautogen==0.1.14`` with ``openai<1.0``
|
|
21
|
+
(install with ``pip install "agentprobe[autogen]"``). Modern AutoGen releases
|
|
22
|
+
(``ag2`` / ``autogen-agentchat`` 0.4+) have a different API and are not yet
|
|
23
|
+
supported; see the project roadmap.
|
|
24
|
+
|
|
25
|
+
**Protocol fix (Nov 2026):** pyautogen 0.1.14 uses the legacy OpenAI
|
|
26
|
+
``functions`` / ``function_call`` format, but Ollama 0.31+ only handles
|
|
27
|
+
the newer ``tools`` / ``tool_calls`` format. :func:`_patch_openai_compat`
|
|
28
|
+
installs a one-time monkey-patch on ``openai.ChatCompletion.create`` that
|
|
29
|
+
converts between the two formats transparently.
|
|
30
|
+
|
|
31
|
+
Usage::
|
|
32
|
+
|
|
33
|
+
from agentprobe.agents.autogen_agent import AutoGenToolAgent
|
|
34
|
+
|
|
35
|
+
agent = AutoGenToolAgent(
|
|
36
|
+
llm_config=AUTOGEN_LLM_CONFIG,
|
|
37
|
+
failure_config=ToolFailureConfig(
|
|
38
|
+
failure_type=ToolFailureType.MALFORMED_OUTPUT,
|
|
39
|
+
failure_probability=0.1,
|
|
40
|
+
),
|
|
41
|
+
)
|
|
42
|
+
result = agent.invoke({"input": "Find the top 3 LLM papers from 2024."})
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
from __future__ import annotations
|
|
46
|
+
|
|
47
|
+
import time
|
|
48
|
+
from typing import Any
|
|
49
|
+
|
|
50
|
+
from agentprobe.injector.tool_failure import ToolFailureConfig, ToolFailureInjector
|
|
51
|
+
from agentprobe.observer.tracer import get_tracer
|
|
52
|
+
|
|
53
|
+
_tracer = get_tracer(__name__)
|
|
54
|
+
|
|
55
|
+
_COMPAT_PATCHED = False
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _patch_openai_compat() -> None:
|
|
59
|
+
"""Monkey-patch ``openai.ChatCompletion.create`` to bridge the
|
|
60
|
+
``functions`` ↔ ``tools`` protocol gap between pyautogen 0.1.14
|
|
61
|
+
and Ollama's OpenAI-compatible endpoint.
|
|
62
|
+
|
|
63
|
+
Request: ``functions`` list → ``tools`` list (wrapped in type/function)
|
|
64
|
+
Response: ``tool_calls`` list → single ``function_call`` dict
|
|
65
|
+
|
|
66
|
+
The patch is idempotent and applied at most once per process.
|
|
67
|
+
"""
|
|
68
|
+
global _COMPAT_PATCHED
|
|
69
|
+
if _COMPAT_PATCHED:
|
|
70
|
+
return
|
|
71
|
+
|
|
72
|
+
import openai
|
|
73
|
+
|
|
74
|
+
_original_create = openai.ChatCompletion.create
|
|
75
|
+
|
|
76
|
+
@staticmethod # type: ignore[misc]
|
|
77
|
+
def _patched_create(*args: Any, **kwargs: Any) -> Any:
|
|
78
|
+
# --- request: functions → tools ---
|
|
79
|
+
if "functions" in kwargs and "tools" not in kwargs:
|
|
80
|
+
functions = kwargs.pop("functions")
|
|
81
|
+
kwargs["tools"] = [{"type": "function", "function": f} for f in functions]
|
|
82
|
+
# Also remove function_call if present (incompatible with tools)
|
|
83
|
+
kwargs.pop("function_call", None)
|
|
84
|
+
|
|
85
|
+
resp = _original_create(*args, **kwargs)
|
|
86
|
+
|
|
87
|
+
# --- response: tool_calls → function_call ---
|
|
88
|
+
for choice in resp.get("choices", []):
|
|
89
|
+
msg = choice.get("message", {})
|
|
90
|
+
tool_calls = msg.get("tool_calls")
|
|
91
|
+
if tool_calls and not msg.get("function_call"):
|
|
92
|
+
first = tool_calls[0]
|
|
93
|
+
fn = first.get("function", {})
|
|
94
|
+
msg["function_call"] = {
|
|
95
|
+
"name": fn.get("name", ""),
|
|
96
|
+
"arguments": fn.get("arguments", "{}"),
|
|
97
|
+
}
|
|
98
|
+
return resp
|
|
99
|
+
|
|
100
|
+
openai.ChatCompletion.create = _patched_create
|
|
101
|
+
_COMPAT_PATCHED = True
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class AutoGenToolAgent:
|
|
105
|
+
"""AutoGen AssistantAgent + UserProxy with AgentProbe instrumentation.
|
|
106
|
+
|
|
107
|
+
Falls back gracefully if pyautogen is not installed, logging a clear
|
|
108
|
+
ImportError with install instructions.
|
|
109
|
+
|
|
110
|
+
Args:
|
|
111
|
+
llm_config: AutoGen llm_config dict (see AutoGen docs).
|
|
112
|
+
failure_config: Optional tool failure injection configuration.
|
|
113
|
+
max_turns: Maximum conversation turns between agents.
|
|
114
|
+
"""
|
|
115
|
+
|
|
116
|
+
framework = "autogen"
|
|
117
|
+
|
|
118
|
+
def __init__(
|
|
119
|
+
self,
|
|
120
|
+
llm_config: dict[str, Any],
|
|
121
|
+
failure_config: ToolFailureConfig | None = None,
|
|
122
|
+
max_turns: int = 5,
|
|
123
|
+
) -> None:
|
|
124
|
+
self._llm_config = llm_config
|
|
125
|
+
self._max_turns = max_turns
|
|
126
|
+
self._injector = ToolFailureInjector(failure_config) if failure_config else None
|
|
127
|
+
_patch_openai_compat()
|
|
128
|
+
self._agents = self._build_agents()
|
|
129
|
+
|
|
130
|
+
def invoke(self, input: dict[str, Any]) -> dict[str, Any]:
|
|
131
|
+
try:
|
|
132
|
+
import autogen # noqa: F401
|
|
133
|
+
except ImportError as e:
|
|
134
|
+
raise ImportError(
|
|
135
|
+
"pyautogen is not installed. Install with: pip install pyautogen"
|
|
136
|
+
) from e
|
|
137
|
+
|
|
138
|
+
with _tracer.start_as_current_span(
|
|
139
|
+
"autogen.agent.invoke",
|
|
140
|
+
attributes={"framework": self.framework},
|
|
141
|
+
) as span:
|
|
142
|
+
start = time.perf_counter()
|
|
143
|
+
query = str(input.get("input", ""))
|
|
144
|
+
if context := input.get("context"):
|
|
145
|
+
query = f"Context:\n{context}\n\nQuestion: {query}"
|
|
146
|
+
|
|
147
|
+
user_proxy, assistant = self._agents
|
|
148
|
+
# Clear conversation history and response cache between runs
|
|
149
|
+
self.reset()
|
|
150
|
+
user_proxy.initiate_chat(
|
|
151
|
+
assistant,
|
|
152
|
+
message=query,
|
|
153
|
+
max_turns=self._max_turns,
|
|
154
|
+
)
|
|
155
|
+
latency_ms = (time.perf_counter() - start) * 1000
|
|
156
|
+
span.set_attribute("latency_ms", latency_ms)
|
|
157
|
+
|
|
158
|
+
# Extract last assistant message with nonempty content from
|
|
159
|
+
# conversation history. pyautogen 0.1.14 stores messages on the
|
|
160
|
+
# agent objects, not the return value. Tool-call messages have
|
|
161
|
+
# role="assistant" but empty/None content, so we skip those to
|
|
162
|
+
# find the actual natural-language response (if any).
|
|
163
|
+
messages = assistant.chat_messages.get(user_proxy, [])
|
|
164
|
+
output = ""
|
|
165
|
+
for m in reversed(messages):
|
|
166
|
+
if m.get("role") == "assistant":
|
|
167
|
+
content = (m.get("content") or "").strip()
|
|
168
|
+
if content:
|
|
169
|
+
output = content
|
|
170
|
+
break
|
|
171
|
+
|
|
172
|
+
# Count tool executions from conversation history
|
|
173
|
+
tool_calls = [m for m in messages if m.get("role") == "function"]
|
|
174
|
+
span.set_attribute("tool_call_count", len(tool_calls))
|
|
175
|
+
|
|
176
|
+
return {
|
|
177
|
+
"output": output,
|
|
178
|
+
"latency_ms": latency_ms,
|
|
179
|
+
"framework": self.framework,
|
|
180
|
+
"tool_calls": len(tool_calls),
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
def reset(self) -> None:
|
|
184
|
+
"""Clear both agents' conversation history and pyautogen's on-disk
|
|
185
|
+
completion cache. Called automatically at the start of every
|
|
186
|
+
``invoke()``. Injector state is intentionally preserved (condition-wide).
|
|
187
|
+
"""
|
|
188
|
+
user_proxy, assistant = self._agents
|
|
189
|
+
if user_proxy is None or assistant is None:
|
|
190
|
+
return # pyautogen not installed; invoke() will raise a clear ImportError
|
|
191
|
+
user_proxy.reset()
|
|
192
|
+
assistant.reset()
|
|
193
|
+
try:
|
|
194
|
+
from autogen.oai.completion import Completion
|
|
195
|
+
|
|
196
|
+
Completion.clear_cache()
|
|
197
|
+
except Exception:
|
|
198
|
+
pass
|
|
199
|
+
|
|
200
|
+
@property
|
|
201
|
+
def injector(self) -> ToolFailureInjector | None:
|
|
202
|
+
return self._injector
|
|
203
|
+
|
|
204
|
+
def get_injector_summary(self) -> dict[str, Any]:
|
|
205
|
+
if self._injector:
|
|
206
|
+
return self._injector.summary()
|
|
207
|
+
return {}
|
|
208
|
+
|
|
209
|
+
def _build_agents(self) -> tuple[Any, Any]:
|
|
210
|
+
try:
|
|
211
|
+
import autogen
|
|
212
|
+
|
|
213
|
+
def _web_search(query: str) -> str:
|
|
214
|
+
def _web_search_impl(q: str) -> str:
|
|
215
|
+
return f"[MOCK WEB SEARCH] Results for: {q}\nFound 3 relevant results."
|
|
216
|
+
|
|
217
|
+
if self._injector:
|
|
218
|
+
return self._injector.call(
|
|
219
|
+
_web_search_impl,
|
|
220
|
+
query,
|
|
221
|
+
tool_name="web_search",
|
|
222
|
+
)
|
|
223
|
+
return _web_search_impl(query)
|
|
224
|
+
|
|
225
|
+
def _calculator(expression: str) -> str:
|
|
226
|
+
def _calculator_impl(expr: str) -> str:
|
|
227
|
+
allowed = set("0123456789+-*/()., ")
|
|
228
|
+
if not all(c in allowed for c in expr):
|
|
229
|
+
return "Error: invalid characters"
|
|
230
|
+
try:
|
|
231
|
+
return str(eval(expr, {"__builtins__": {}})) # noqa: S307
|
|
232
|
+
except Exception as e:
|
|
233
|
+
return f"Error: {e}"
|
|
234
|
+
|
|
235
|
+
if self._injector:
|
|
236
|
+
return self._injector.call(
|
|
237
|
+
_calculator_impl,
|
|
238
|
+
expression,
|
|
239
|
+
tool_name="calculator",
|
|
240
|
+
)
|
|
241
|
+
return _calculator_impl(expression)
|
|
242
|
+
|
|
243
|
+
# Define function schemas for the LLM
|
|
244
|
+
# These are in the legacy 'functions' format; _patch_openai_compat()
|
|
245
|
+
# converts them to 'tools' format before sending to the API.
|
|
246
|
+
functions = [
|
|
247
|
+
{
|
|
248
|
+
"name": "web_search",
|
|
249
|
+
"description": "Search the web for current information.",
|
|
250
|
+
"parameters": {
|
|
251
|
+
"type": "object",
|
|
252
|
+
"properties": {
|
|
253
|
+
"query": {
|
|
254
|
+
"type": "string",
|
|
255
|
+
"description": "The search query",
|
|
256
|
+
}
|
|
257
|
+
},
|
|
258
|
+
"required": ["query"],
|
|
259
|
+
},
|
|
260
|
+
},
|
|
261
|
+
{
|
|
262
|
+
"name": "calculator",
|
|
263
|
+
"description": "Evaluate a mathematical expression.",
|
|
264
|
+
"parameters": {
|
|
265
|
+
"type": "object",
|
|
266
|
+
"properties": {
|
|
267
|
+
"expression": {
|
|
268
|
+
"type": "string",
|
|
269
|
+
"description": "The mathematical expression to evaluate",
|
|
270
|
+
}
|
|
271
|
+
},
|
|
272
|
+
"required": ["expression"],
|
|
273
|
+
},
|
|
274
|
+
},
|
|
275
|
+
]
|
|
276
|
+
|
|
277
|
+
# Add function definitions to llm_config
|
|
278
|
+
llm_config_with_functions = {
|
|
279
|
+
**self._llm_config,
|
|
280
|
+
"functions": functions,
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
assistant = autogen.AssistantAgent(
|
|
284
|
+
name="assistant",
|
|
285
|
+
llm_config=llm_config_with_functions,
|
|
286
|
+
system_message=(
|
|
287
|
+
"You are a reliable AI assistant. Use the provided functions to "
|
|
288
|
+
"answer questions. If a tool call fails, acknowledge and try another approach."
|
|
289
|
+
),
|
|
290
|
+
)
|
|
291
|
+
|
|
292
|
+
# Register functions on the user_proxy so it can execute them
|
|
293
|
+
user_proxy = autogen.UserProxyAgent(
|
|
294
|
+
name="user_proxy",
|
|
295
|
+
human_input_mode="NEVER",
|
|
296
|
+
max_consecutive_auto_reply=self._max_turns,
|
|
297
|
+
code_execution_config=False,
|
|
298
|
+
function_map={
|
|
299
|
+
"web_search": _web_search,
|
|
300
|
+
"calculator": _calculator,
|
|
301
|
+
},
|
|
302
|
+
)
|
|
303
|
+
|
|
304
|
+
return user_proxy, assistant
|
|
305
|
+
|
|
306
|
+
except ImportError:
|
|
307
|
+
# Deferred — ImportError raised at invoke() time instead
|
|
308
|
+
return None, None # type: ignore[return-value]
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"""Shared protocol and result helper for all AgentProbe agent adapters.
|
|
2
|
+
|
|
3
|
+
Every adapter exposes the same minimal surface so that
|
|
4
|
+
:class:`~agentprobe.benchmark.runner.BenchmarkRunner` can drive any framework
|
|
5
|
+
without knowing which one it is:
|
|
6
|
+
|
|
7
|
+
* ``framework`` — a short string identifier (``"langgraph"``, ``"langchain"``, ``"autogen"``).
|
|
8
|
+
* ``invoke(input)`` — run one task. ``input`` is a dict with at least an
|
|
9
|
+
``"input"`` key (the user query) and optionally ``"context"`` (RAG context).
|
|
10
|
+
Returns a dict with at least ``"output"``, ``"latency_ms"``, and ``"framework"``.
|
|
11
|
+
* ``reset()`` — clear per-conversation state so the next ``invoke()`` starts fresh.
|
|
12
|
+
This does **not** reset the fault injector: injection state is condition-wide
|
|
13
|
+
by design (see :meth:`ToolFailureInjector.reset`).
|
|
14
|
+
* ``get_injector_summary()`` — aggregate injection statistics for the condition.
|
|
15
|
+
|
|
16
|
+
Usage::
|
|
17
|
+
|
|
18
|
+
from agentprobe.agents import BaseAgent
|
|
19
|
+
|
|
20
|
+
class MyAgent:
|
|
21
|
+
framework = "my_framework"
|
|
22
|
+
|
|
23
|
+
def invoke(self, input: dict[str, Any]) -> dict[str, Any]:
|
|
24
|
+
...
|
|
25
|
+
|
|
26
|
+
def reset(self) -> None:
|
|
27
|
+
...
|
|
28
|
+
|
|
29
|
+
def get_injector_summary(self) -> dict[str, Any]:
|
|
30
|
+
return {}
|
|
31
|
+
|
|
32
|
+
assert isinstance(MyAgent(), BaseAgent)
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
from __future__ import annotations
|
|
36
|
+
|
|
37
|
+
from typing import Any, Protocol, runtime_checkable
|
|
38
|
+
|
|
39
|
+
from pydantic import BaseModel, Field
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class AgentResult(BaseModel):
|
|
43
|
+
"""Optional structured view of one ``invoke()`` outcome.
|
|
44
|
+
|
|
45
|
+
Adapters return plain dicts from ``invoke()`` so that framework-specific
|
|
46
|
+
extras (e.g. AutoGen's ``tool_calls`` count) pass through untouched.
|
|
47
|
+
``AgentResult`` is a typed convenience for code that wants validation,
|
|
48
|
+
for example external adapters that wrap a non-LLM pipeline.
|
|
49
|
+
|
|
50
|
+
Args:
|
|
51
|
+
success: Whether the agent completed the task without unrecovered failure.
|
|
52
|
+
output: Final text output from the agent. ``None`` on unrecovered failure.
|
|
53
|
+
duration_ms: Wall-clock time for the run in milliseconds.
|
|
54
|
+
retries: Number of retry attempts made during the run.
|
|
55
|
+
error_message: Human-readable error detail when ``success`` is ``False``.
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
success: bool
|
|
59
|
+
output: str | None = None
|
|
60
|
+
duration_ms: float = Field(default=0.0, ge=0.0)
|
|
61
|
+
retries: int = Field(default=0, ge=0)
|
|
62
|
+
error_message: str | None = None
|
|
63
|
+
|
|
64
|
+
@classmethod
|
|
65
|
+
def from_invoke(cls, result: dict[str, Any]) -> AgentResult:
|
|
66
|
+
"""Build an ``AgentResult`` from an adapter's ``invoke()`` return dict."""
|
|
67
|
+
output = result.get("output")
|
|
68
|
+
return cls(
|
|
69
|
+
success=bool(output),
|
|
70
|
+
output=str(output) if output else None,
|
|
71
|
+
duration_ms=float(result.get("latency_ms", 0.0)),
|
|
72
|
+
retries=int(result.get("retries", 0)),
|
|
73
|
+
error_message=result.get("error_message"),
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@runtime_checkable
|
|
78
|
+
class BaseAgent(Protocol):
|
|
79
|
+
"""Structural protocol satisfied by all AgentProbe agent adapters.
|
|
80
|
+
|
|
81
|
+
The protocol is ``runtime_checkable`` so ``isinstance`` checks work in the
|
|
82
|
+
benchmark runner without importing concrete adapter classes.
|
|
83
|
+
"""
|
|
84
|
+
|
|
85
|
+
framework: str
|
|
86
|
+
|
|
87
|
+
def invoke(self, input: dict[str, Any]) -> dict[str, Any]:
|
|
88
|
+
"""Run one task and return a dict with ``output``, ``latency_ms``, ``framework``."""
|
|
89
|
+
...
|
|
90
|
+
|
|
91
|
+
def reset(self) -> None:
|
|
92
|
+
"""Clear per-conversation state so the next ``invoke()`` starts fresh."""
|
|
93
|
+
...
|
|
94
|
+
|
|
95
|
+
def get_injector_summary(self) -> dict[str, Any]:
|
|
96
|
+
"""Return aggregate fault-injection statistics (empty if no injector)."""
|
|
97
|
+
...
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
"""LangChain agent for AgentProbe benchmarking.
|
|
2
|
+
|
|
3
|
+
Uses LangChain v1.3+ ``create_agent`` (which wraps LangGraph internally)
|
|
4
|
+
with the same tools and prompts as the LangGraph agent, enabling
|
|
5
|
+
apples-to-apples framework comparison.
|
|
6
|
+
|
|
7
|
+
Usage::
|
|
8
|
+
|
|
9
|
+
from langchain_ollama import ChatOllama
|
|
10
|
+
from agentprobe.agents.langchain_agent import LangChainToolAgent
|
|
11
|
+
from agentprobe.injector.tool_failure import ToolFailureConfig, ToolFailureType
|
|
12
|
+
|
|
13
|
+
llm = ChatOllama(model="llama3.1")
|
|
14
|
+
agent = LangChainToolAgent(
|
|
15
|
+
llm=llm,
|
|
16
|
+
failure_config=ToolFailureConfig(
|
|
17
|
+
failure_type=ToolFailureType.RATE_LIMIT,
|
|
18
|
+
failure_probability=0.15,
|
|
19
|
+
),
|
|
20
|
+
)
|
|
21
|
+
result = agent.invoke({"input": "What is 2^32?"})
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import time
|
|
27
|
+
from typing import Any
|
|
28
|
+
|
|
29
|
+
from langchain.agents import create_agent
|
|
30
|
+
from langchain_core.tools import tool
|
|
31
|
+
|
|
32
|
+
from agentprobe.injector.tool_failure import ToolFailureConfig, ToolFailureInjector
|
|
33
|
+
from agentprobe.observer.tracer import get_tracer
|
|
34
|
+
|
|
35
|
+
_tracer = get_tracer(__name__)
|
|
36
|
+
|
|
37
|
+
_SYSTEM_PROMPT = (
|
|
38
|
+
"You are a reliable AI assistant. Use the provided tools to answer questions "
|
|
39
|
+
"accurately. If a tool call fails, acknowledge the failure and try a different approach."
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _make_tools(injector: ToolFailureInjector | None = None) -> list[Any]:
|
|
44
|
+
@tool
|
|
45
|
+
def web_search(query: str) -> str:
|
|
46
|
+
"""Search the web for current information."""
|
|
47
|
+
return f"[MOCK WEB SEARCH] Results for: {query}\nFound 3 relevant results."
|
|
48
|
+
|
|
49
|
+
@tool
|
|
50
|
+
def calculator(expression: str) -> str:
|
|
51
|
+
"""Evaluate a mathematical expression."""
|
|
52
|
+
try:
|
|
53
|
+
allowed = set("0123456789+-*/()., ")
|
|
54
|
+
if not all(c in allowed for c in expression):
|
|
55
|
+
return "Error: invalid characters in expression"
|
|
56
|
+
result = eval(expression, {"__builtins__": {}}) # noqa: S307
|
|
57
|
+
return str(result)
|
|
58
|
+
except Exception as e:
|
|
59
|
+
return f"Calculation error: {e}"
|
|
60
|
+
|
|
61
|
+
tools = [web_search, calculator]
|
|
62
|
+
if injector is not None:
|
|
63
|
+
tools = [injector.inject_langchain_tool(t) for t in tools]
|
|
64
|
+
return tools
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class LangChainToolAgent:
|
|
68
|
+
"""LangChain create_agent with AgentProbe instrumentation.
|
|
69
|
+
|
|
70
|
+
Uses LangChain v1.3+ ``create_agent`` which returns a compiled
|
|
71
|
+
LangGraph under the hood but exposes the LangChain tool/prompt API.
|
|
72
|
+
|
|
73
|
+
Args:
|
|
74
|
+
llm: A LangChain chat model supporting tool calling.
|
|
75
|
+
failure_config: Optional tool failure injection configuration.
|
|
76
|
+
recursion_limit: Maximum agent loop iterations.
|
|
77
|
+
"""
|
|
78
|
+
|
|
79
|
+
framework = "langchain"
|
|
80
|
+
|
|
81
|
+
def __init__(
|
|
82
|
+
self,
|
|
83
|
+
llm: Any,
|
|
84
|
+
failure_config: ToolFailureConfig | None = None,
|
|
85
|
+
recursion_limit: int = 25,
|
|
86
|
+
) -> None:
|
|
87
|
+
self._llm = llm
|
|
88
|
+
self._injector = ToolFailureInjector(failure_config) if failure_config else None
|
|
89
|
+
self._tools = _make_tools(self._injector)
|
|
90
|
+
self._agent = create_agent(
|
|
91
|
+
self._llm,
|
|
92
|
+
self._tools,
|
|
93
|
+
system_prompt=_SYSTEM_PROMPT,
|
|
94
|
+
)
|
|
95
|
+
self._recursion_limit = recursion_limit
|
|
96
|
+
|
|
97
|
+
def invoke(self, input: dict[str, Any]) -> dict[str, Any]:
|
|
98
|
+
with _tracer.start_as_current_span(
|
|
99
|
+
"langchain.agent.invoke",
|
|
100
|
+
attributes={"framework": self.framework},
|
|
101
|
+
) as span:
|
|
102
|
+
start = time.perf_counter()
|
|
103
|
+
query = str(input.get("input", ""))
|
|
104
|
+
if context := input.get("context"):
|
|
105
|
+
query = f"Context:\n{context}\n\nQuestion: {query}"
|
|
106
|
+
|
|
107
|
+
result = self._agent.invoke(
|
|
108
|
+
{"messages": [{"role": "user", "content": query}]},
|
|
109
|
+
{"recursion_limit": self._recursion_limit},
|
|
110
|
+
)
|
|
111
|
+
latency_ms = (time.perf_counter() - start) * 1000
|
|
112
|
+
span.set_attribute("latency_ms", latency_ms)
|
|
113
|
+
|
|
114
|
+
# Extract last AI message content
|
|
115
|
+
messages = result.get("messages", [])
|
|
116
|
+
output = ""
|
|
117
|
+
for msg in reversed(messages):
|
|
118
|
+
if hasattr(msg, "content") and type(msg).__name__ == "AIMessage" and msg.content:
|
|
119
|
+
output = msg.content
|
|
120
|
+
break
|
|
121
|
+
|
|
122
|
+
return {
|
|
123
|
+
"output": output,
|
|
124
|
+
"latency_ms": latency_ms,
|
|
125
|
+
"framework": self.framework,
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
def reset(self) -> None:
|
|
129
|
+
"""No-op: each ``invoke()`` starts a new message list. Injector state is
|
|
130
|
+
intentionally preserved (condition-wide); use ``agent.injector.reset()``.
|
|
131
|
+
"""
|
|
132
|
+
|
|
133
|
+
@property
|
|
134
|
+
def injector(self) -> ToolFailureInjector | None:
|
|
135
|
+
return self._injector
|
|
136
|
+
|
|
137
|
+
def get_injector_summary(self) -> dict[str, Any]:
|
|
138
|
+
if self._injector:
|
|
139
|
+
return self._injector.summary()
|
|
140
|
+
return {}
|