agentprobe-toolkit 1.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,203 @@
1
+ """LangGraph ReAct agent for AgentProbe benchmarking.
2
+
3
+ Implements a standard LangGraph ReAct agent with:
4
+ - Configurable LLM backend (Ollama/OpenAI/Anthropic)
5
+ - Two benchmark tools: web_search (mocked) and calculator
6
+ - Full OpenTelemetry instrumentation on every node and tool call
7
+ - Failure injector hooks on tool execution
8
+
9
+ The graph structure:
10
+ [START] -> agent_node -> tool_node -> agent_node -> ... -> [END]
11
+
12
+ Usage::
13
+
14
+ from langchain_ollama import ChatOllama
15
+ from agentprobe.agents.langgraph_agent import LangGraphToolAgent
16
+ from agentprobe.injector.tool_failure import ToolFailureConfig, ToolFailureType
17
+
18
+ llm = ChatOllama(model="llama3")
19
+ agent = LangGraphToolAgent(
20
+ llm=llm,
21
+ failure_config=ToolFailureConfig(
22
+ failure_type=ToolFailureType.TIMEOUT,
23
+ failure_probability=0.2,
24
+ ),
25
+ )
26
+ result = agent.invoke({"input": "What is the capital of France?"})
27
+ print(result["output"], result["latency_ms"])
28
+ """
29
+
30
+ from __future__ import annotations
31
+
32
+ import time
33
+ from collections.abc import Sequence
34
+ from typing import Annotated, Any, TypedDict
35
+
36
+ from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage
37
+ from langchain_core.tools import tool
38
+ from langgraph.graph import END, StateGraph
39
+ from langgraph.graph.message import add_messages
40
+ from langgraph.prebuilt import ToolNode
41
+
42
+ from agentprobe.injector.tool_failure import ToolFailureConfig, ToolFailureInjector
43
+ from agentprobe.observer.tracer import get_tracer
44
+
45
+ _tracer = get_tracer(__name__)
46
+
47
+ SYSTEM_PROMPT = (
48
+ "You are a reliable AI assistant. Use the provided tools to answer questions "
49
+ "accurately. If a tool call fails, acknowledge the failure and try a different approach."
50
+ )
51
+
52
+
53
+ # ---------------------------------------------------------------------------
54
+ # State
55
+ # ---------------------------------------------------------------------------
56
+
57
+
58
+ class AgentState(TypedDict):
59
+ messages: Annotated[Sequence[BaseMessage], add_messages]
60
+
61
+
62
+ # ---------------------------------------------------------------------------
63
+ # Tools (mockable for offline benchmarking)
64
+ # ---------------------------------------------------------------------------
65
+
66
+
67
+ def _make_tools(injector: ToolFailureInjector | None = None) -> list[Any]:
68
+ """Create benchmark tools, optionally wrapping with failure injector."""
69
+
70
+ @tool
71
+ def web_search(query: str) -> str:
72
+ """Search the web for current information."""
73
+ # Mock response — replace with real search in production experiments
74
+ return f"[MOCK WEB SEARCH] Results for: {query}\nFound 3 relevant results."
75
+
76
+ @tool
77
+ def calculator(expression: str) -> str:
78
+ """Evaluate a mathematical expression."""
79
+ try:
80
+ # Safe eval: only allow numeric expressions
81
+ allowed = set("0123456789+-*/()., ")
82
+ if not all(c in allowed for c in expression):
83
+ return "Error: invalid characters in expression"
84
+ result = eval(expression, {"__builtins__": {}}) # noqa: S307
85
+ return str(result)
86
+ except Exception as e:
87
+ return f"Calculation error: {e}"
88
+
89
+ tools = [web_search, calculator]
90
+
91
+ if injector is not None:
92
+ tools = [injector.inject_langchain_tool(t) for t in tools]
93
+
94
+ return tools
95
+
96
+
97
+ # ---------------------------------------------------------------------------
98
+ # Agent
99
+ # ---------------------------------------------------------------------------
100
+
101
+
102
+ class LangGraphToolAgent:
103
+ """LangGraph ReAct agent with AgentProbe instrumentation.
104
+
105
+ Args:
106
+ llm: A LangChain chat model (ChatOllama, ChatOpenAI, ChatAnthropic, etc.)
107
+ failure_config: Optional tool failure injection configuration.
108
+ max_iterations: Maximum ReAct loop iterations before forced termination.
109
+ """
110
+
111
+ framework = "langgraph"
112
+
113
+ def __init__(
114
+ self,
115
+ llm: Any,
116
+ failure_config: ToolFailureConfig | None = None,
117
+ max_iterations: int = 10,
118
+ ) -> None:
119
+ self._llm = llm
120
+ self._max_iterations = max_iterations
121
+ self._injector = ToolFailureInjector(failure_config) if failure_config else None
122
+ self._tools = _make_tools(self._injector)
123
+ self._graph = self._build_graph()
124
+
125
+ # ------------------------------------------------------------------
126
+ # Public interface
127
+ # ------------------------------------------------------------------
128
+
129
+ def invoke(self, input: dict[str, Any]) -> dict[str, Any]:
130
+ """Run the agent on a task input. Returns output dict."""
131
+ with _tracer.start_as_current_span(
132
+ "langgraph.agent.invoke",
133
+ attributes={
134
+ "framework": self.framework,
135
+ "input.length": len(str(input.get("input", ""))),
136
+ },
137
+ ) as span:
138
+ start = time.perf_counter()
139
+ messages = [
140
+ SystemMessage(content=SYSTEM_PROMPT),
141
+ HumanMessage(content=str(input.get("input", ""))),
142
+ ]
143
+ # Add RAG context if provided
144
+ if context := input.get("context"):
145
+ messages.insert(1, SystemMessage(content=f"Context:\n{context}"))
146
+
147
+ state: AgentState = {"messages": messages}
148
+ result = self._graph.invoke(
149
+ state,
150
+ config={"recursion_limit": self._max_iterations},
151
+ )
152
+ latency_ms = (time.perf_counter() - start) * 1000
153
+ span.set_attribute("latency_ms", latency_ms)
154
+
155
+ final_message = result["messages"][-1]
156
+ return {
157
+ "output": getattr(final_message, "content", str(final_message)),
158
+ "latency_ms": latency_ms,
159
+ "framework": self.framework,
160
+ }
161
+
162
+ def reset(self) -> None:
163
+ """No-op: each ``invoke()`` builds a fresh message list, so there is no
164
+ conversation state to clear. Injector state is intentionally preserved
165
+ (it is condition-wide); call ``agent.injector.reset()`` to clear it.
166
+ """
167
+
168
+ @property
169
+ def injector(self) -> ToolFailureInjector | None:
170
+ return self._injector
171
+
172
+ def get_injector_summary(self) -> dict[str, Any]:
173
+ if self._injector:
174
+ return self._injector.summary()
175
+ return {}
176
+
177
+ # ------------------------------------------------------------------
178
+ # Graph construction
179
+ # ------------------------------------------------------------------
180
+
181
+ def _build_graph(self) -> Any:
182
+ llm_with_tools = self._llm.bind_tools(self._tools)
183
+ tool_node = ToolNode(self._tools)
184
+
185
+ def agent_node(state: AgentState) -> AgentState:
186
+ with _tracer.start_as_current_span("langgraph.node.agent"):
187
+ response = llm_with_tools.invoke(state["messages"])
188
+ return {"messages": [response]}
189
+
190
+ def should_continue(state: AgentState) -> str:
191
+ last = state["messages"][-1]
192
+ if hasattr(last, "tool_calls") and last.tool_calls:
193
+ return "tools"
194
+ return END
195
+
196
+ builder = StateGraph(AgentState)
197
+ builder.add_node("agent", agent_node)
198
+ builder.add_node("tools", tool_node)
199
+ builder.set_entry_point("agent")
200
+ builder.add_conditional_edges("agent", should_continue)
201
+ builder.add_edge("tools", "agent")
202
+
203
+ return builder.compile()
@@ -0,0 +1,67 @@
1
+ """Benchmark suite for AgentProbe.
2
+
3
+ Usage::
4
+
5
+ from agentprobe.benchmark import BenchmarkConfig, BenchmarkRunner, MetricsCalculator
6
+
7
+ runner = BenchmarkRunner(agent=my_agent, config=BenchmarkConfig(n_runs=30))
8
+ results = runner.run()
9
+ metrics = MetricsCalculator().compute(results)
10
+ """
11
+
12
+ from agentprobe.benchmark.grader import BatchGradeResult, GradeResult, ResponseGrader
13
+ from agentprobe.benchmark.metrics import AgentMetrics, MetricsCalculator
14
+ from agentprobe.benchmark.runner import (
15
+ PAPER_PROMPTS,
16
+ BenchmarkConfig,
17
+ BenchmarkRunner,
18
+ RunResult,
19
+ paper_task_fn,
20
+ )
21
+ from agentprobe.benchmark.statistics import (
22
+ FrameworkData,
23
+ HypothesisTestResult,
24
+ PairwiseComparison,
25
+ StatisticalReport,
26
+ build_comparison_report,
27
+ cohens_d,
28
+ cohens_h,
29
+ compare_distributions,
30
+ compare_multiple_groups,
31
+ compare_proportions,
32
+ mean_ci,
33
+ proportion_ci,
34
+ )
35
+ from agentprobe.benchmark.tasks import corpus_task_fn, get_task
36
+
37
+ __all__ = [
38
+ # Runner
39
+ "BenchmarkConfig",
40
+ "BenchmarkRunner",
41
+ "RunResult",
42
+ "PAPER_PROMPTS",
43
+ "paper_task_fn",
44
+ # Tasks
45
+ "get_task",
46
+ "corpus_task_fn",
47
+ # Metrics
48
+ "AgentMetrics",
49
+ "MetricsCalculator",
50
+ # Grading
51
+ "ResponseGrader",
52
+ "GradeResult",
53
+ "BatchGradeResult",
54
+ # Statistics
55
+ "FrameworkData",
56
+ "PairwiseComparison",
57
+ "StatisticalReport",
58
+ "HypothesisTestResult",
59
+ "build_comparison_report",
60
+ "cohens_d",
61
+ "cohens_h",
62
+ "compare_distributions",
63
+ "compare_multiple_groups",
64
+ "compare_proportions",
65
+ "mean_ci",
66
+ "proportion_ci",
67
+ ]
@@ -0,0 +1,162 @@
1
+ """Response grading for benchmark tasks.
2
+
3
+ Evaluates agent outputs against expected keyword sets to measure
4
+ accuracy and detect silent failures (wrong answers without errors).
5
+
6
+ Usage::
7
+
8
+ from agentprobe.benchmark.grader import ResponseGrader
9
+
10
+ grader = ResponseGrader(threshold=0.5)
11
+ result = grader.grade("Tokyo has 14 million people", ["tokyo", "population", "million"])
12
+ # result.score = 1.0, result.is_correct = True
13
+
14
+ # Silent failure detection
15
+ batch = grader.grade_batch(
16
+ outputs=["Paris is the capital", "I don't know"],
17
+ expected=[["tokyo", "population"], ["berlin", "capital"]],
18
+ had_errors=[False, False],
19
+ )
20
+ # batch.silent_failure_count = 2 (wrong answers, no exceptions)
21
+ """
22
+
23
+ from __future__ import annotations
24
+
25
+ from collections.abc import Sequence
26
+ from dataclasses import dataclass
27
+
28
+
29
+ @dataclass(frozen=True)
30
+ class GradeResult:
31
+ """Result of grading a single agent response."""
32
+
33
+ score: float # 0.0-1.0, fraction of keywords matched
34
+ matched_keywords: tuple[str, ...]
35
+ total_keywords: int
36
+ is_correct: bool # score >= threshold
37
+
38
+
39
+ @dataclass(frozen=True)
40
+ class BatchGradeResult:
41
+ """Aggregated grading results for a batch of responses."""
42
+
43
+ n_total: int
44
+ n_correct: int
45
+ n_incorrect: int
46
+ accuracy: float # n_correct / n_total
47
+ mean_score: float
48
+ silent_failure_count: int # incorrect AND no exception raised
49
+ silent_failure_rate: float # silent_failure_count / n_total
50
+ grades: tuple[GradeResult, ...]
51
+
52
+
53
+ class ResponseGrader:
54
+ """Grade agent responses by keyword matching.
55
+
56
+ Args:
57
+ threshold: Minimum keyword match fraction to count as correct.
58
+ Default 0.5 means at least half the expected keywords must appear.
59
+ """
60
+
61
+ def __init__(self, threshold: float = 0.5) -> None:
62
+ if not 0.0 <= threshold <= 1.0:
63
+ raise ValueError(f"threshold must be in [0, 1], got {threshold}")
64
+ self._threshold = threshold
65
+
66
+ @property
67
+ def threshold(self) -> float:
68
+ return self._threshold
69
+
70
+ def grade(
71
+ self,
72
+ output: str,
73
+ expected_keywords: Sequence[str],
74
+ ) -> GradeResult:
75
+ """Grade a single response against expected keywords.
76
+
77
+ Args:
78
+ output: The agent's text output.
79
+ expected_keywords: Keywords that should appear in a correct response.
80
+
81
+ Returns:
82
+ GradeResult with score, matched keywords, and correctness flag.
83
+ """
84
+ if not expected_keywords:
85
+ return GradeResult(
86
+ score=1.0,
87
+ matched_keywords=(),
88
+ total_keywords=0,
89
+ is_correct=True,
90
+ )
91
+
92
+ output_lower = output.lower()
93
+ matched = tuple(kw for kw in expected_keywords if kw.lower() in output_lower)
94
+ score = len(matched) / len(expected_keywords)
95
+
96
+ return GradeResult(
97
+ score=score,
98
+ matched_keywords=matched,
99
+ total_keywords=len(expected_keywords),
100
+ is_correct=score >= self._threshold,
101
+ )
102
+
103
+ def grade_batch(
104
+ self,
105
+ outputs: Sequence[str],
106
+ expected: Sequence[Sequence[str]],
107
+ had_errors: Sequence[bool] | None = None,
108
+ ) -> BatchGradeResult:
109
+ """Grade a batch of responses and detect silent failures.
110
+
111
+ Args:
112
+ outputs: Agent output strings.
113
+ expected: Expected keyword lists, one per output.
114
+ had_errors: Whether each run raised an exception. If None,
115
+ assumes no errors (all runs completed without exception).
116
+
117
+ Returns:
118
+ BatchGradeResult with accuracy, scores, and silent failure count.
119
+ """
120
+ if len(outputs) != len(expected):
121
+ raise ValueError(
122
+ f"outputs ({len(outputs)}) and expected ({len(expected)}) must have same length"
123
+ )
124
+ if had_errors is not None and len(had_errors) != len(outputs):
125
+ raise ValueError(f"had_errors ({len(had_errors)}) must match outputs ({len(outputs)})")
126
+
127
+ if had_errors is None:
128
+ had_errors = [False] * len(outputs)
129
+
130
+ grades = tuple(self.grade(out, exp) for out, exp in zip(outputs, expected))
131
+ n_total = len(grades)
132
+
133
+ if n_total == 0:
134
+ return BatchGradeResult(
135
+ n_total=0,
136
+ n_correct=0,
137
+ n_incorrect=0,
138
+ accuracy=0.0,
139
+ mean_score=0.0,
140
+ silent_failure_count=0,
141
+ silent_failure_rate=0.0,
142
+ grades=(),
143
+ )
144
+
145
+ n_correct = sum(1 for g in grades if g.is_correct)
146
+ n_incorrect = n_total - n_correct
147
+
148
+ # Silent failure: incorrect response AND no exception was raised
149
+ silent_failures = sum(
150
+ 1 for g, err in zip(grades, had_errors) if not g.is_correct and not err
151
+ )
152
+
153
+ return BatchGradeResult(
154
+ n_total=n_total,
155
+ n_correct=n_correct,
156
+ n_incorrect=n_incorrect,
157
+ accuracy=n_correct / n_total,
158
+ mean_score=sum(g.score for g in grades) / n_total,
159
+ silent_failure_count=silent_failures,
160
+ silent_failure_rate=silent_failures / n_total,
161
+ grades=grades,
162
+ )
@@ -0,0 +1,214 @@
1
+ """Experiment metadata capture for reproducibility.
2
+
3
+ Records hardware, software, and configuration details alongside benchmark
4
+ results so that reviewers can assess the experimental context.
5
+
6
+ Usage::
7
+
8
+ from agentprobe.benchmark.metadata import capture_metadata
9
+
10
+ meta = capture_metadata(
11
+ model="llama3.1",
12
+ llm_backend="ollama",
13
+ frameworks=["langgraph", "langchain", "autogen"],
14
+ )
15
+ meta.to_dict() # JSON-serializable
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ import os
21
+ import platform
22
+ import subprocess
23
+ import sys
24
+ import time
25
+ from dataclasses import asdict, dataclass, field
26
+ from typing import Any
27
+
28
+
29
+ @dataclass(frozen=True)
30
+ class ExperimentMetadata:
31
+ """Immutable record of experiment environment and configuration."""
32
+
33
+ # Hardware
34
+ cpu_model: str
35
+ cpu_count: int
36
+ ram_gb: float
37
+ os_name: str
38
+ os_version: str
39
+ arch: str
40
+
41
+ # Software
42
+ python_version: str
43
+ agentprobe_version: str
44
+ git_hash: str
45
+ git_branch: str
46
+ git_dirty: bool
47
+
48
+ # LLM configuration
49
+ llm_backend: str
50
+ model_name: str
51
+ ollama_version: str
52
+
53
+ # Experiment configuration
54
+ frameworks: tuple[str, ...]
55
+ n_runs_per_cell: int
56
+ seed: int
57
+ timestamp_utc: str
58
+
59
+ # Dependency versions
60
+ dependency_versions: dict[str, str] = field(default_factory=dict)
61
+
62
+ def to_dict(self) -> dict[str, Any]:
63
+ d = asdict(self)
64
+ d["frameworks"] = list(d["frameworks"])
65
+ return d
66
+
67
+
68
+ def _get_git_info() -> tuple[str, str, bool]:
69
+ """Return (commit_hash, branch, is_dirty)."""
70
+ try:
71
+ commit = subprocess.check_output(
72
+ ["git", "rev-parse", "--short", "HEAD"],
73
+ stderr=subprocess.DEVNULL,
74
+ text=True,
75
+ ).strip()
76
+ branch = subprocess.check_output(
77
+ ["git", "rev-parse", "--abbrev-ref", "HEAD"],
78
+ stderr=subprocess.DEVNULL,
79
+ text=True,
80
+ ).strip()
81
+ status = subprocess.check_output(
82
+ ["git", "status", "--porcelain"],
83
+ stderr=subprocess.DEVNULL,
84
+ text=True,
85
+ ).strip()
86
+ return commit, branch, len(status) > 0
87
+ except (subprocess.CalledProcessError, FileNotFoundError):
88
+ return "unknown", "unknown", False
89
+
90
+
91
+ def _get_ollama_version() -> str:
92
+ """Return Ollama version string, or 'not available'."""
93
+ try:
94
+ out = subprocess.check_output(
95
+ ["ollama", "--version"],
96
+ stderr=subprocess.DEVNULL,
97
+ text=True,
98
+ ).strip()
99
+ return out.split()[-1] if out else "unknown"
100
+ except (subprocess.CalledProcessError, FileNotFoundError):
101
+ return "not available"
102
+
103
+
104
+ def _get_ram_gb() -> float:
105
+ """Return total system RAM in GB."""
106
+ try:
107
+ if platform.system() == "Darwin":
108
+ out = subprocess.check_output(
109
+ ["sysctl", "-n", "hw.memsize"],
110
+ stderr=subprocess.DEVNULL,
111
+ text=True,
112
+ ).strip()
113
+ return round(int(out) / (1024**3), 1)
114
+ elif platform.system() == "Linux":
115
+ with open("/proc/meminfo") as f:
116
+ for line in f:
117
+ if line.startswith("MemTotal"):
118
+ kb = int(line.split()[1])
119
+ return round(kb / (1024**2), 1)
120
+ return 0.0
121
+ except (subprocess.CalledProcessError, FileNotFoundError, ValueError):
122
+ return 0.0
123
+
124
+
125
+ def _get_cpu_model() -> str:
126
+ """Return CPU model string."""
127
+ try:
128
+ if platform.system() == "Darwin":
129
+ out = subprocess.check_output(
130
+ ["sysctl", "-n", "machdep.cpu.brand_string"],
131
+ stderr=subprocess.DEVNULL,
132
+ text=True,
133
+ ).strip()
134
+ return out
135
+ elif platform.system() == "Linux":
136
+ with open("/proc/cpuinfo") as f:
137
+ for line in f:
138
+ if line.startswith("model name"):
139
+ return line.split(":", 1)[1].strip()
140
+ return platform.processor() or "unknown"
141
+ except (subprocess.CalledProcessError, FileNotFoundError):
142
+ return platform.processor() or "unknown"
143
+
144
+
145
+ def _get_dependency_versions() -> dict[str, str]:
146
+ """Return versions of key dependencies."""
147
+ deps: dict[str, str] = {}
148
+ for pkg in [
149
+ "langgraph",
150
+ "langchain",
151
+ "langchain_core",
152
+ "langchain_community",
153
+ "opentelemetry",
154
+ "pydantic",
155
+ "numpy",
156
+ "pandas",
157
+ "scipy",
158
+ ]:
159
+ try:
160
+ mod = __import__(pkg)
161
+ deps[pkg] = getattr(mod, "__version__", "unknown")
162
+ except ImportError:
163
+ deps[pkg] = "not installed"
164
+
165
+ # pyautogen uses a different import name
166
+ try:
167
+ import autogen # type: ignore[import-untyped]
168
+
169
+ deps["pyautogen"] = getattr(autogen, "__version__", "unknown")
170
+ except ImportError:
171
+ deps["pyautogen"] = "not installed"
172
+
173
+ return deps
174
+
175
+
176
+ def capture_metadata(
177
+ model: str,
178
+ llm_backend: str = "ollama",
179
+ frameworks: list[str] | None = None,
180
+ n_runs_per_cell: int = 10,
181
+ seed: int = 42,
182
+ ) -> ExperimentMetadata:
183
+ """Capture full experiment metadata snapshot.
184
+
185
+ Call this once at experiment start and save alongside results.
186
+ """
187
+ git_hash, git_branch, git_dirty = _get_git_info()
188
+
189
+ try:
190
+ from agentprobe import __version__ as ap_version
191
+ except ImportError:
192
+ ap_version = "unknown"
193
+
194
+ return ExperimentMetadata(
195
+ cpu_model=_get_cpu_model(),
196
+ cpu_count=os.cpu_count() or 0,
197
+ ram_gb=_get_ram_gb(),
198
+ os_name=platform.system(),
199
+ os_version=platform.release(),
200
+ arch=platform.machine(),
201
+ python_version=sys.version.split()[0],
202
+ agentprobe_version=ap_version,
203
+ git_hash=git_hash,
204
+ git_branch=git_branch,
205
+ git_dirty=git_dirty,
206
+ llm_backend=llm_backend,
207
+ model_name=model,
208
+ ollama_version=_get_ollama_version() if llm_backend == "ollama" else "n/a",
209
+ frameworks=tuple(frameworks or ["langgraph", "langchain", "autogen"]),
210
+ n_runs_per_cell=n_runs_per_cell,
211
+ seed=seed,
212
+ timestamp_utc=time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
213
+ dependency_versions=_get_dependency_versions(),
214
+ )