agentprobe-toolkit 1.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
agentprobe/__init__.py ADDED
@@ -0,0 +1,42 @@
1
+ """AgentProbe — fault injection and observability toolkit for LLM agents.
2
+
3
+ Usage::
4
+
5
+ from agentprobe import ToolFailureConfig, ToolFailureInjector, ToolFailureType
6
+ from agentprobe import AgentProbeTracer, BenchmarkConfig, BenchmarkRunner, MetricsCalculator
7
+
8
+ Framework adapters live in :mod:`agentprobe.agents` and are imported explicitly
9
+ (``from agentprobe.agents.langgraph_agent import LangGraphToolAgent``) so that
10
+ importing the package does not require every framework to be installed.
11
+ """
12
+
13
+ from agentprobe.benchmark.metrics import AgentMetrics, MetricsCalculator
14
+ from agentprobe.benchmark.runner import BenchmarkConfig, BenchmarkRunner, RunResult
15
+ from agentprobe.injector.tool_failure import (
16
+ ToolFailureConfig,
17
+ ToolFailureInjector,
18
+ ToolFailureType,
19
+ )
20
+ from agentprobe.observer.classifier import FailureCategory, FailureClassifier
21
+ from agentprobe.observer.tracer import AgentProbeTracer, get_tracer
22
+
23
+ __version__ = "1.2.0"
24
+
25
+ __all__ = [
26
+ "__version__",
27
+ # Injection
28
+ "ToolFailureConfig",
29
+ "ToolFailureInjector",
30
+ "ToolFailureType",
31
+ # Observability
32
+ "AgentProbeTracer",
33
+ "get_tracer",
34
+ "FailureClassifier",
35
+ "FailureCategory",
36
+ # Benchmark
37
+ "BenchmarkConfig",
38
+ "BenchmarkRunner",
39
+ "RunResult",
40
+ "MetricsCalculator",
41
+ "AgentMetrics",
42
+ ]
@@ -0,0 +1,5 @@
1
+ """Agent implementations for AgentProbe benchmarking."""
2
+
3
+ from agentprobe.agents.base import AgentResult, BaseAgent
4
+
5
+ __all__ = ["AgentResult", "BaseAgent"]
@@ -0,0 +1,308 @@
1
+ """AutoGen agent for AgentProbe benchmarking.
2
+
3
+ Implements a two-agent AutoGen pipeline (UserProxy + AssistantAgent)
4
+ using the same tool set as LangGraph and LangChain agents for
5
+ apples-to-apples comparison.
6
+
7
+ The AutoGen agent uses the OpenAI-compatible API endpoint, which can
8
+ point to an Ollama local server:
9
+
10
+ AUTOGEN_LLM_CONFIG = {
11
+ "config_list": [
12
+ {
13
+ "model": "llama3",
14
+ "base_url": "http://localhost:11434/v1",
15
+ "api_key": "ollama",
16
+ }
17
+ ]
18
+ }
19
+
20
+ **Version pin:** this adapter targets ``pyautogen==0.1.14`` with ``openai<1.0``
21
+ (install with ``pip install "agentprobe[autogen]"``). Modern AutoGen releases
22
+ (``ag2`` / ``autogen-agentchat`` 0.4+) have a different API and are not yet
23
+ supported; see the project roadmap.
24
+
25
+ **Protocol fix (Nov 2026):** pyautogen 0.1.14 uses the legacy OpenAI
26
+ ``functions`` / ``function_call`` format, but Ollama 0.31+ only handles
27
+ the newer ``tools`` / ``tool_calls`` format. :func:`_patch_openai_compat`
28
+ installs a one-time monkey-patch on ``openai.ChatCompletion.create`` that
29
+ converts between the two formats transparently.
30
+
31
+ Usage::
32
+
33
+ from agentprobe.agents.autogen_agent import AutoGenToolAgent
34
+
35
+ agent = AutoGenToolAgent(
36
+ llm_config=AUTOGEN_LLM_CONFIG,
37
+ failure_config=ToolFailureConfig(
38
+ failure_type=ToolFailureType.MALFORMED_OUTPUT,
39
+ failure_probability=0.1,
40
+ ),
41
+ )
42
+ result = agent.invoke({"input": "Find the top 3 LLM papers from 2024."})
43
+ """
44
+
45
+ from __future__ import annotations
46
+
47
+ import time
48
+ from typing import Any
49
+
50
+ from agentprobe.injector.tool_failure import ToolFailureConfig, ToolFailureInjector
51
+ from agentprobe.observer.tracer import get_tracer
52
+
53
+ _tracer = get_tracer(__name__)
54
+
55
+ _COMPAT_PATCHED = False
56
+
57
+
58
+ def _patch_openai_compat() -> None:
59
+ """Monkey-patch ``openai.ChatCompletion.create`` to bridge the
60
+ ``functions`` ↔ ``tools`` protocol gap between pyautogen 0.1.14
61
+ and Ollama's OpenAI-compatible endpoint.
62
+
63
+ Request: ``functions`` list → ``tools`` list (wrapped in type/function)
64
+ Response: ``tool_calls`` list → single ``function_call`` dict
65
+
66
+ The patch is idempotent and applied at most once per process.
67
+ """
68
+ global _COMPAT_PATCHED
69
+ if _COMPAT_PATCHED:
70
+ return
71
+
72
+ import openai
73
+
74
+ _original_create = openai.ChatCompletion.create
75
+
76
+ @staticmethod # type: ignore[misc]
77
+ def _patched_create(*args: Any, **kwargs: Any) -> Any:
78
+ # --- request: functions → tools ---
79
+ if "functions" in kwargs and "tools" not in kwargs:
80
+ functions = kwargs.pop("functions")
81
+ kwargs["tools"] = [{"type": "function", "function": f} for f in functions]
82
+ # Also remove function_call if present (incompatible with tools)
83
+ kwargs.pop("function_call", None)
84
+
85
+ resp = _original_create(*args, **kwargs)
86
+
87
+ # --- response: tool_calls → function_call ---
88
+ for choice in resp.get("choices", []):
89
+ msg = choice.get("message", {})
90
+ tool_calls = msg.get("tool_calls")
91
+ if tool_calls and not msg.get("function_call"):
92
+ first = tool_calls[0]
93
+ fn = first.get("function", {})
94
+ msg["function_call"] = {
95
+ "name": fn.get("name", ""),
96
+ "arguments": fn.get("arguments", "{}"),
97
+ }
98
+ return resp
99
+
100
+ openai.ChatCompletion.create = _patched_create
101
+ _COMPAT_PATCHED = True
102
+
103
+
104
+ class AutoGenToolAgent:
105
+ """AutoGen AssistantAgent + UserProxy with AgentProbe instrumentation.
106
+
107
+ Falls back gracefully if pyautogen is not installed, logging a clear
108
+ ImportError with install instructions.
109
+
110
+ Args:
111
+ llm_config: AutoGen llm_config dict (see AutoGen docs).
112
+ failure_config: Optional tool failure injection configuration.
113
+ max_turns: Maximum conversation turns between agents.
114
+ """
115
+
116
+ framework = "autogen"
117
+
118
+ def __init__(
119
+ self,
120
+ llm_config: dict[str, Any],
121
+ failure_config: ToolFailureConfig | None = None,
122
+ max_turns: int = 5,
123
+ ) -> None:
124
+ self._llm_config = llm_config
125
+ self._max_turns = max_turns
126
+ self._injector = ToolFailureInjector(failure_config) if failure_config else None
127
+ _patch_openai_compat()
128
+ self._agents = self._build_agents()
129
+
130
+ def invoke(self, input: dict[str, Any]) -> dict[str, Any]:
131
+ try:
132
+ import autogen # noqa: F401
133
+ except ImportError as e:
134
+ raise ImportError(
135
+ "pyautogen is not installed. Install with: pip install pyautogen"
136
+ ) from e
137
+
138
+ with _tracer.start_as_current_span(
139
+ "autogen.agent.invoke",
140
+ attributes={"framework": self.framework},
141
+ ) as span:
142
+ start = time.perf_counter()
143
+ query = str(input.get("input", ""))
144
+ if context := input.get("context"):
145
+ query = f"Context:\n{context}\n\nQuestion: {query}"
146
+
147
+ user_proxy, assistant = self._agents
148
+ # Clear conversation history and response cache between runs
149
+ self.reset()
150
+ user_proxy.initiate_chat(
151
+ assistant,
152
+ message=query,
153
+ max_turns=self._max_turns,
154
+ )
155
+ latency_ms = (time.perf_counter() - start) * 1000
156
+ span.set_attribute("latency_ms", latency_ms)
157
+
158
+ # Extract last assistant message with nonempty content from
159
+ # conversation history. pyautogen 0.1.14 stores messages on the
160
+ # agent objects, not the return value. Tool-call messages have
161
+ # role="assistant" but empty/None content, so we skip those to
162
+ # find the actual natural-language response (if any).
163
+ messages = assistant.chat_messages.get(user_proxy, [])
164
+ output = ""
165
+ for m in reversed(messages):
166
+ if m.get("role") == "assistant":
167
+ content = (m.get("content") or "").strip()
168
+ if content:
169
+ output = content
170
+ break
171
+
172
+ # Count tool executions from conversation history
173
+ tool_calls = [m for m in messages if m.get("role") == "function"]
174
+ span.set_attribute("tool_call_count", len(tool_calls))
175
+
176
+ return {
177
+ "output": output,
178
+ "latency_ms": latency_ms,
179
+ "framework": self.framework,
180
+ "tool_calls": len(tool_calls),
181
+ }
182
+
183
+ def reset(self) -> None:
184
+ """Clear both agents' conversation history and pyautogen's on-disk
185
+ completion cache. Called automatically at the start of every
186
+ ``invoke()``. Injector state is intentionally preserved (condition-wide).
187
+ """
188
+ user_proxy, assistant = self._agents
189
+ if user_proxy is None or assistant is None:
190
+ return # pyautogen not installed; invoke() will raise a clear ImportError
191
+ user_proxy.reset()
192
+ assistant.reset()
193
+ try:
194
+ from autogen.oai.completion import Completion
195
+
196
+ Completion.clear_cache()
197
+ except Exception:
198
+ pass
199
+
200
+ @property
201
+ def injector(self) -> ToolFailureInjector | None:
202
+ return self._injector
203
+
204
+ def get_injector_summary(self) -> dict[str, Any]:
205
+ if self._injector:
206
+ return self._injector.summary()
207
+ return {}
208
+
209
+ def _build_agents(self) -> tuple[Any, Any]:
210
+ try:
211
+ import autogen
212
+
213
+ def _web_search(query: str) -> str:
214
+ def _web_search_impl(q: str) -> str:
215
+ return f"[MOCK WEB SEARCH] Results for: {q}\nFound 3 relevant results."
216
+
217
+ if self._injector:
218
+ return self._injector.call(
219
+ _web_search_impl,
220
+ query,
221
+ tool_name="web_search",
222
+ )
223
+ return _web_search_impl(query)
224
+
225
+ def _calculator(expression: str) -> str:
226
+ def _calculator_impl(expr: str) -> str:
227
+ allowed = set("0123456789+-*/()., ")
228
+ if not all(c in allowed for c in expr):
229
+ return "Error: invalid characters"
230
+ try:
231
+ return str(eval(expr, {"__builtins__": {}})) # noqa: S307
232
+ except Exception as e:
233
+ return f"Error: {e}"
234
+
235
+ if self._injector:
236
+ return self._injector.call(
237
+ _calculator_impl,
238
+ expression,
239
+ tool_name="calculator",
240
+ )
241
+ return _calculator_impl(expression)
242
+
243
+ # Define function schemas for the LLM
244
+ # These are in the legacy 'functions' format; _patch_openai_compat()
245
+ # converts them to 'tools' format before sending to the API.
246
+ functions = [
247
+ {
248
+ "name": "web_search",
249
+ "description": "Search the web for current information.",
250
+ "parameters": {
251
+ "type": "object",
252
+ "properties": {
253
+ "query": {
254
+ "type": "string",
255
+ "description": "The search query",
256
+ }
257
+ },
258
+ "required": ["query"],
259
+ },
260
+ },
261
+ {
262
+ "name": "calculator",
263
+ "description": "Evaluate a mathematical expression.",
264
+ "parameters": {
265
+ "type": "object",
266
+ "properties": {
267
+ "expression": {
268
+ "type": "string",
269
+ "description": "The mathematical expression to evaluate",
270
+ }
271
+ },
272
+ "required": ["expression"],
273
+ },
274
+ },
275
+ ]
276
+
277
+ # Add function definitions to llm_config
278
+ llm_config_with_functions = {
279
+ **self._llm_config,
280
+ "functions": functions,
281
+ }
282
+
283
+ assistant = autogen.AssistantAgent(
284
+ name="assistant",
285
+ llm_config=llm_config_with_functions,
286
+ system_message=(
287
+ "You are a reliable AI assistant. Use the provided functions to "
288
+ "answer questions. If a tool call fails, acknowledge and try another approach."
289
+ ),
290
+ )
291
+
292
+ # Register functions on the user_proxy so it can execute them
293
+ user_proxy = autogen.UserProxyAgent(
294
+ name="user_proxy",
295
+ human_input_mode="NEVER",
296
+ max_consecutive_auto_reply=self._max_turns,
297
+ code_execution_config=False,
298
+ function_map={
299
+ "web_search": _web_search,
300
+ "calculator": _calculator,
301
+ },
302
+ )
303
+
304
+ return user_proxy, assistant
305
+
306
+ except ImportError:
307
+ # Deferred — ImportError raised at invoke() time instead
308
+ return None, None # type: ignore[return-value]
@@ -0,0 +1,97 @@
1
+ """Shared protocol and result helper for all AgentProbe agent adapters.
2
+
3
+ Every adapter exposes the same minimal surface so that
4
+ :class:`~agentprobe.benchmark.runner.BenchmarkRunner` can drive any framework
5
+ without knowing which one it is:
6
+
7
+ * ``framework`` — a short string identifier (``"langgraph"``, ``"langchain"``, ``"autogen"``).
8
+ * ``invoke(input)`` — run one task. ``input`` is a dict with at least an
9
+ ``"input"`` key (the user query) and optionally ``"context"`` (RAG context).
10
+ Returns a dict with at least ``"output"``, ``"latency_ms"``, and ``"framework"``.
11
+ * ``reset()`` — clear per-conversation state so the next ``invoke()`` starts fresh.
12
+ This does **not** reset the fault injector: injection state is condition-wide
13
+ by design (see :meth:`ToolFailureInjector.reset`).
14
+ * ``get_injector_summary()`` — aggregate injection statistics for the condition.
15
+
16
+ Usage::
17
+
18
+ from agentprobe.agents import BaseAgent
19
+
20
+ class MyAgent:
21
+ framework = "my_framework"
22
+
23
+ def invoke(self, input: dict[str, Any]) -> dict[str, Any]:
24
+ ...
25
+
26
+ def reset(self) -> None:
27
+ ...
28
+
29
+ def get_injector_summary(self) -> dict[str, Any]:
30
+ return {}
31
+
32
+ assert isinstance(MyAgent(), BaseAgent)
33
+ """
34
+
35
+ from __future__ import annotations
36
+
37
+ from typing import Any, Protocol, runtime_checkable
38
+
39
+ from pydantic import BaseModel, Field
40
+
41
+
42
+ class AgentResult(BaseModel):
43
+ """Optional structured view of one ``invoke()`` outcome.
44
+
45
+ Adapters return plain dicts from ``invoke()`` so that framework-specific
46
+ extras (e.g. AutoGen's ``tool_calls`` count) pass through untouched.
47
+ ``AgentResult`` is a typed convenience for code that wants validation,
48
+ for example external adapters that wrap a non-LLM pipeline.
49
+
50
+ Args:
51
+ success: Whether the agent completed the task without unrecovered failure.
52
+ output: Final text output from the agent. ``None`` on unrecovered failure.
53
+ duration_ms: Wall-clock time for the run in milliseconds.
54
+ retries: Number of retry attempts made during the run.
55
+ error_message: Human-readable error detail when ``success`` is ``False``.
56
+ """
57
+
58
+ success: bool
59
+ output: str | None = None
60
+ duration_ms: float = Field(default=0.0, ge=0.0)
61
+ retries: int = Field(default=0, ge=0)
62
+ error_message: str | None = None
63
+
64
+ @classmethod
65
+ def from_invoke(cls, result: dict[str, Any]) -> AgentResult:
66
+ """Build an ``AgentResult`` from an adapter's ``invoke()`` return dict."""
67
+ output = result.get("output")
68
+ return cls(
69
+ success=bool(output),
70
+ output=str(output) if output else None,
71
+ duration_ms=float(result.get("latency_ms", 0.0)),
72
+ retries=int(result.get("retries", 0)),
73
+ error_message=result.get("error_message"),
74
+ )
75
+
76
+
77
+ @runtime_checkable
78
+ class BaseAgent(Protocol):
79
+ """Structural protocol satisfied by all AgentProbe agent adapters.
80
+
81
+ The protocol is ``runtime_checkable`` so ``isinstance`` checks work in the
82
+ benchmark runner without importing concrete adapter classes.
83
+ """
84
+
85
+ framework: str
86
+
87
+ def invoke(self, input: dict[str, Any]) -> dict[str, Any]:
88
+ """Run one task and return a dict with ``output``, ``latency_ms``, ``framework``."""
89
+ ...
90
+
91
+ def reset(self) -> None:
92
+ """Clear per-conversation state so the next ``invoke()`` starts fresh."""
93
+ ...
94
+
95
+ def get_injector_summary(self) -> dict[str, Any]:
96
+ """Return aggregate fault-injection statistics (empty if no injector)."""
97
+ ...
@@ -0,0 +1,140 @@
1
+ """LangChain agent for AgentProbe benchmarking.
2
+
3
+ Uses LangChain v1.3+ ``create_agent`` (which wraps LangGraph internally)
4
+ with the same tools and prompts as the LangGraph agent, enabling
5
+ apples-to-apples framework comparison.
6
+
7
+ Usage::
8
+
9
+ from langchain_ollama import ChatOllama
10
+ from agentprobe.agents.langchain_agent import LangChainToolAgent
11
+ from agentprobe.injector.tool_failure import ToolFailureConfig, ToolFailureType
12
+
13
+ llm = ChatOllama(model="llama3.1")
14
+ agent = LangChainToolAgent(
15
+ llm=llm,
16
+ failure_config=ToolFailureConfig(
17
+ failure_type=ToolFailureType.RATE_LIMIT,
18
+ failure_probability=0.15,
19
+ ),
20
+ )
21
+ result = agent.invoke({"input": "What is 2^32?"})
22
+ """
23
+
24
+ from __future__ import annotations
25
+
26
+ import time
27
+ from typing import Any
28
+
29
+ from langchain.agents import create_agent
30
+ from langchain_core.tools import tool
31
+
32
+ from agentprobe.injector.tool_failure import ToolFailureConfig, ToolFailureInjector
33
+ from agentprobe.observer.tracer import get_tracer
34
+
35
+ _tracer = get_tracer(__name__)
36
+
37
+ _SYSTEM_PROMPT = (
38
+ "You are a reliable AI assistant. Use the provided tools to answer questions "
39
+ "accurately. If a tool call fails, acknowledge the failure and try a different approach."
40
+ )
41
+
42
+
43
+ def _make_tools(injector: ToolFailureInjector | None = None) -> list[Any]:
44
+ @tool
45
+ def web_search(query: str) -> str:
46
+ """Search the web for current information."""
47
+ return f"[MOCK WEB SEARCH] Results for: {query}\nFound 3 relevant results."
48
+
49
+ @tool
50
+ def calculator(expression: str) -> str:
51
+ """Evaluate a mathematical expression."""
52
+ try:
53
+ allowed = set("0123456789+-*/()., ")
54
+ if not all(c in allowed for c in expression):
55
+ return "Error: invalid characters in expression"
56
+ result = eval(expression, {"__builtins__": {}}) # noqa: S307
57
+ return str(result)
58
+ except Exception as e:
59
+ return f"Calculation error: {e}"
60
+
61
+ tools = [web_search, calculator]
62
+ if injector is not None:
63
+ tools = [injector.inject_langchain_tool(t) for t in tools]
64
+ return tools
65
+
66
+
67
+ class LangChainToolAgent:
68
+ """LangChain create_agent with AgentProbe instrumentation.
69
+
70
+ Uses LangChain v1.3+ ``create_agent`` which returns a compiled
71
+ LangGraph under the hood but exposes the LangChain tool/prompt API.
72
+
73
+ Args:
74
+ llm: A LangChain chat model supporting tool calling.
75
+ failure_config: Optional tool failure injection configuration.
76
+ recursion_limit: Maximum agent loop iterations.
77
+ """
78
+
79
+ framework = "langchain"
80
+
81
+ def __init__(
82
+ self,
83
+ llm: Any,
84
+ failure_config: ToolFailureConfig | None = None,
85
+ recursion_limit: int = 25,
86
+ ) -> None:
87
+ self._llm = llm
88
+ self._injector = ToolFailureInjector(failure_config) if failure_config else None
89
+ self._tools = _make_tools(self._injector)
90
+ self._agent = create_agent(
91
+ self._llm,
92
+ self._tools,
93
+ system_prompt=_SYSTEM_PROMPT,
94
+ )
95
+ self._recursion_limit = recursion_limit
96
+
97
+ def invoke(self, input: dict[str, Any]) -> dict[str, Any]:
98
+ with _tracer.start_as_current_span(
99
+ "langchain.agent.invoke",
100
+ attributes={"framework": self.framework},
101
+ ) as span:
102
+ start = time.perf_counter()
103
+ query = str(input.get("input", ""))
104
+ if context := input.get("context"):
105
+ query = f"Context:\n{context}\n\nQuestion: {query}"
106
+
107
+ result = self._agent.invoke(
108
+ {"messages": [{"role": "user", "content": query}]},
109
+ {"recursion_limit": self._recursion_limit},
110
+ )
111
+ latency_ms = (time.perf_counter() - start) * 1000
112
+ span.set_attribute("latency_ms", latency_ms)
113
+
114
+ # Extract last AI message content
115
+ messages = result.get("messages", [])
116
+ output = ""
117
+ for msg in reversed(messages):
118
+ if hasattr(msg, "content") and type(msg).__name__ == "AIMessage" and msg.content:
119
+ output = msg.content
120
+ break
121
+
122
+ return {
123
+ "output": output,
124
+ "latency_ms": latency_ms,
125
+ "framework": self.framework,
126
+ }
127
+
128
+ def reset(self) -> None:
129
+ """No-op: each ``invoke()`` starts a new message list. Injector state is
130
+ intentionally preserved (condition-wide); use ``agent.injector.reset()``.
131
+ """
132
+
133
+ @property
134
+ def injector(self) -> ToolFailureInjector | None:
135
+ return self._injector
136
+
137
+ def get_injector_summary(self) -> dict[str, Any]:
138
+ if self._injector:
139
+ return self._injector.summary()
140
+ return {}