agentprobe-toolkit 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentprobe/__init__.py +42 -0
- agentprobe/agents/__init__.py +5 -0
- agentprobe/agents/autogen_agent.py +308 -0
- agentprobe/agents/base.py +97 -0
- agentprobe/agents/langchain_agent.py +140 -0
- agentprobe/agents/langgraph_agent.py +203 -0
- agentprobe/benchmark/__init__.py +67 -0
- agentprobe/benchmark/grader.py +162 -0
- agentprobe/benchmark/metadata.py +214 -0
- agentprobe/benchmark/metrics.py +247 -0
- agentprobe/benchmark/runner.py +434 -0
- agentprobe/benchmark/statistics.py +459 -0
- agentprobe/benchmark/tasks/__init__.py +75 -0
- agentprobe/benchmark/tasks/multi_agent.py +191 -0
- agentprobe/benchmark/tasks/rag.py +309 -0
- agentprobe/benchmark/tasks/tool_use.py +228 -0
- agentprobe/config.py +67 -0
- agentprobe/injector/__init__.py +66 -0
- agentprobe/injector/context_failure.py +328 -0
- agentprobe/injector/latency_failure.py +275 -0
- agentprobe/injector/orchestration_failure.py +332 -0
- agentprobe/injector/tool_failure.py +404 -0
- agentprobe/observer/__init__.py +5 -0
- agentprobe/observer/classifier.py +320 -0
- agentprobe/observer/dashboard.py +216 -0
- agentprobe/observer/tracer.py +137 -0
- agentprobe_toolkit-1.2.0.dist-info/METADATA +405 -0
- agentprobe_toolkit-1.2.0.dist-info/RECORD +30 -0
- agentprobe_toolkit-1.2.0.dist-info/WHEEL +4 -0
- agentprobe_toolkit-1.2.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
"""LangGraph ReAct agent for AgentProbe benchmarking.
|
|
2
|
+
|
|
3
|
+
Implements a standard LangGraph ReAct agent with:
|
|
4
|
+
- Configurable LLM backend (Ollama/OpenAI/Anthropic)
|
|
5
|
+
- Two benchmark tools: web_search (mocked) and calculator
|
|
6
|
+
- Full OpenTelemetry instrumentation on every node and tool call
|
|
7
|
+
- Failure injector hooks on tool execution
|
|
8
|
+
|
|
9
|
+
The graph structure:
|
|
10
|
+
[START] -> agent_node -> tool_node -> agent_node -> ... -> [END]
|
|
11
|
+
|
|
12
|
+
Usage::
|
|
13
|
+
|
|
14
|
+
from langchain_ollama import ChatOllama
|
|
15
|
+
from agentprobe.agents.langgraph_agent import LangGraphToolAgent
|
|
16
|
+
from agentprobe.injector.tool_failure import ToolFailureConfig, ToolFailureType
|
|
17
|
+
|
|
18
|
+
llm = ChatOllama(model="llama3")
|
|
19
|
+
agent = LangGraphToolAgent(
|
|
20
|
+
llm=llm,
|
|
21
|
+
failure_config=ToolFailureConfig(
|
|
22
|
+
failure_type=ToolFailureType.TIMEOUT,
|
|
23
|
+
failure_probability=0.2,
|
|
24
|
+
),
|
|
25
|
+
)
|
|
26
|
+
result = agent.invoke({"input": "What is the capital of France?"})
|
|
27
|
+
print(result["output"], result["latency_ms"])
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
from __future__ import annotations
|
|
31
|
+
|
|
32
|
+
import time
|
|
33
|
+
from collections.abc import Sequence
|
|
34
|
+
from typing import Annotated, Any, TypedDict
|
|
35
|
+
|
|
36
|
+
from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage
|
|
37
|
+
from langchain_core.tools import tool
|
|
38
|
+
from langgraph.graph import END, StateGraph
|
|
39
|
+
from langgraph.graph.message import add_messages
|
|
40
|
+
from langgraph.prebuilt import ToolNode
|
|
41
|
+
|
|
42
|
+
from agentprobe.injector.tool_failure import ToolFailureConfig, ToolFailureInjector
|
|
43
|
+
from agentprobe.observer.tracer import get_tracer
|
|
44
|
+
|
|
45
|
+
_tracer = get_tracer(__name__)
|
|
46
|
+
|
|
47
|
+
SYSTEM_PROMPT = (
|
|
48
|
+
"You are a reliable AI assistant. Use the provided tools to answer questions "
|
|
49
|
+
"accurately. If a tool call fails, acknowledge the failure and try a different approach."
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
# ---------------------------------------------------------------------------
|
|
54
|
+
# State
|
|
55
|
+
# ---------------------------------------------------------------------------
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class AgentState(TypedDict):
|
|
59
|
+
messages: Annotated[Sequence[BaseMessage], add_messages]
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
# ---------------------------------------------------------------------------
|
|
63
|
+
# Tools (mockable for offline benchmarking)
|
|
64
|
+
# ---------------------------------------------------------------------------
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _make_tools(injector: ToolFailureInjector | None = None) -> list[Any]:
|
|
68
|
+
"""Create benchmark tools, optionally wrapping with failure injector."""
|
|
69
|
+
|
|
70
|
+
@tool
|
|
71
|
+
def web_search(query: str) -> str:
|
|
72
|
+
"""Search the web for current information."""
|
|
73
|
+
# Mock response — replace with real search in production experiments
|
|
74
|
+
return f"[MOCK WEB SEARCH] Results for: {query}\nFound 3 relevant results."
|
|
75
|
+
|
|
76
|
+
@tool
|
|
77
|
+
def calculator(expression: str) -> str:
|
|
78
|
+
"""Evaluate a mathematical expression."""
|
|
79
|
+
try:
|
|
80
|
+
# Safe eval: only allow numeric expressions
|
|
81
|
+
allowed = set("0123456789+-*/()., ")
|
|
82
|
+
if not all(c in allowed for c in expression):
|
|
83
|
+
return "Error: invalid characters in expression"
|
|
84
|
+
result = eval(expression, {"__builtins__": {}}) # noqa: S307
|
|
85
|
+
return str(result)
|
|
86
|
+
except Exception as e:
|
|
87
|
+
return f"Calculation error: {e}"
|
|
88
|
+
|
|
89
|
+
tools = [web_search, calculator]
|
|
90
|
+
|
|
91
|
+
if injector is not None:
|
|
92
|
+
tools = [injector.inject_langchain_tool(t) for t in tools]
|
|
93
|
+
|
|
94
|
+
return tools
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
# ---------------------------------------------------------------------------
|
|
98
|
+
# Agent
|
|
99
|
+
# ---------------------------------------------------------------------------
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
class LangGraphToolAgent:
|
|
103
|
+
"""LangGraph ReAct agent with AgentProbe instrumentation.
|
|
104
|
+
|
|
105
|
+
Args:
|
|
106
|
+
llm: A LangChain chat model (ChatOllama, ChatOpenAI, ChatAnthropic, etc.)
|
|
107
|
+
failure_config: Optional tool failure injection configuration.
|
|
108
|
+
max_iterations: Maximum ReAct loop iterations before forced termination.
|
|
109
|
+
"""
|
|
110
|
+
|
|
111
|
+
framework = "langgraph"
|
|
112
|
+
|
|
113
|
+
def __init__(
|
|
114
|
+
self,
|
|
115
|
+
llm: Any,
|
|
116
|
+
failure_config: ToolFailureConfig | None = None,
|
|
117
|
+
max_iterations: int = 10,
|
|
118
|
+
) -> None:
|
|
119
|
+
self._llm = llm
|
|
120
|
+
self._max_iterations = max_iterations
|
|
121
|
+
self._injector = ToolFailureInjector(failure_config) if failure_config else None
|
|
122
|
+
self._tools = _make_tools(self._injector)
|
|
123
|
+
self._graph = self._build_graph()
|
|
124
|
+
|
|
125
|
+
# ------------------------------------------------------------------
|
|
126
|
+
# Public interface
|
|
127
|
+
# ------------------------------------------------------------------
|
|
128
|
+
|
|
129
|
+
def invoke(self, input: dict[str, Any]) -> dict[str, Any]:
|
|
130
|
+
"""Run the agent on a task input. Returns output dict."""
|
|
131
|
+
with _tracer.start_as_current_span(
|
|
132
|
+
"langgraph.agent.invoke",
|
|
133
|
+
attributes={
|
|
134
|
+
"framework": self.framework,
|
|
135
|
+
"input.length": len(str(input.get("input", ""))),
|
|
136
|
+
},
|
|
137
|
+
) as span:
|
|
138
|
+
start = time.perf_counter()
|
|
139
|
+
messages = [
|
|
140
|
+
SystemMessage(content=SYSTEM_PROMPT),
|
|
141
|
+
HumanMessage(content=str(input.get("input", ""))),
|
|
142
|
+
]
|
|
143
|
+
# Add RAG context if provided
|
|
144
|
+
if context := input.get("context"):
|
|
145
|
+
messages.insert(1, SystemMessage(content=f"Context:\n{context}"))
|
|
146
|
+
|
|
147
|
+
state: AgentState = {"messages": messages}
|
|
148
|
+
result = self._graph.invoke(
|
|
149
|
+
state,
|
|
150
|
+
config={"recursion_limit": self._max_iterations},
|
|
151
|
+
)
|
|
152
|
+
latency_ms = (time.perf_counter() - start) * 1000
|
|
153
|
+
span.set_attribute("latency_ms", latency_ms)
|
|
154
|
+
|
|
155
|
+
final_message = result["messages"][-1]
|
|
156
|
+
return {
|
|
157
|
+
"output": getattr(final_message, "content", str(final_message)),
|
|
158
|
+
"latency_ms": latency_ms,
|
|
159
|
+
"framework": self.framework,
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
def reset(self) -> None:
|
|
163
|
+
"""No-op: each ``invoke()`` builds a fresh message list, so there is no
|
|
164
|
+
conversation state to clear. Injector state is intentionally preserved
|
|
165
|
+
(it is condition-wide); call ``agent.injector.reset()`` to clear it.
|
|
166
|
+
"""
|
|
167
|
+
|
|
168
|
+
@property
|
|
169
|
+
def injector(self) -> ToolFailureInjector | None:
|
|
170
|
+
return self._injector
|
|
171
|
+
|
|
172
|
+
def get_injector_summary(self) -> dict[str, Any]:
|
|
173
|
+
if self._injector:
|
|
174
|
+
return self._injector.summary()
|
|
175
|
+
return {}
|
|
176
|
+
|
|
177
|
+
# ------------------------------------------------------------------
|
|
178
|
+
# Graph construction
|
|
179
|
+
# ------------------------------------------------------------------
|
|
180
|
+
|
|
181
|
+
def _build_graph(self) -> Any:
|
|
182
|
+
llm_with_tools = self._llm.bind_tools(self._tools)
|
|
183
|
+
tool_node = ToolNode(self._tools)
|
|
184
|
+
|
|
185
|
+
def agent_node(state: AgentState) -> AgentState:
|
|
186
|
+
with _tracer.start_as_current_span("langgraph.node.agent"):
|
|
187
|
+
response = llm_with_tools.invoke(state["messages"])
|
|
188
|
+
return {"messages": [response]}
|
|
189
|
+
|
|
190
|
+
def should_continue(state: AgentState) -> str:
|
|
191
|
+
last = state["messages"][-1]
|
|
192
|
+
if hasattr(last, "tool_calls") and last.tool_calls:
|
|
193
|
+
return "tools"
|
|
194
|
+
return END
|
|
195
|
+
|
|
196
|
+
builder = StateGraph(AgentState)
|
|
197
|
+
builder.add_node("agent", agent_node)
|
|
198
|
+
builder.add_node("tools", tool_node)
|
|
199
|
+
builder.set_entry_point("agent")
|
|
200
|
+
builder.add_conditional_edges("agent", should_continue)
|
|
201
|
+
builder.add_edge("tools", "agent")
|
|
202
|
+
|
|
203
|
+
return builder.compile()
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Benchmark suite for AgentProbe.
|
|
2
|
+
|
|
3
|
+
Usage::
|
|
4
|
+
|
|
5
|
+
from agentprobe.benchmark import BenchmarkConfig, BenchmarkRunner, MetricsCalculator
|
|
6
|
+
|
|
7
|
+
runner = BenchmarkRunner(agent=my_agent, config=BenchmarkConfig(n_runs=30))
|
|
8
|
+
results = runner.run()
|
|
9
|
+
metrics = MetricsCalculator().compute(results)
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from agentprobe.benchmark.grader import BatchGradeResult, GradeResult, ResponseGrader
|
|
13
|
+
from agentprobe.benchmark.metrics import AgentMetrics, MetricsCalculator
|
|
14
|
+
from agentprobe.benchmark.runner import (
|
|
15
|
+
PAPER_PROMPTS,
|
|
16
|
+
BenchmarkConfig,
|
|
17
|
+
BenchmarkRunner,
|
|
18
|
+
RunResult,
|
|
19
|
+
paper_task_fn,
|
|
20
|
+
)
|
|
21
|
+
from agentprobe.benchmark.statistics import (
|
|
22
|
+
FrameworkData,
|
|
23
|
+
HypothesisTestResult,
|
|
24
|
+
PairwiseComparison,
|
|
25
|
+
StatisticalReport,
|
|
26
|
+
build_comparison_report,
|
|
27
|
+
cohens_d,
|
|
28
|
+
cohens_h,
|
|
29
|
+
compare_distributions,
|
|
30
|
+
compare_multiple_groups,
|
|
31
|
+
compare_proportions,
|
|
32
|
+
mean_ci,
|
|
33
|
+
proportion_ci,
|
|
34
|
+
)
|
|
35
|
+
from agentprobe.benchmark.tasks import corpus_task_fn, get_task
|
|
36
|
+
|
|
37
|
+
__all__ = [
|
|
38
|
+
# Runner
|
|
39
|
+
"BenchmarkConfig",
|
|
40
|
+
"BenchmarkRunner",
|
|
41
|
+
"RunResult",
|
|
42
|
+
"PAPER_PROMPTS",
|
|
43
|
+
"paper_task_fn",
|
|
44
|
+
# Tasks
|
|
45
|
+
"get_task",
|
|
46
|
+
"corpus_task_fn",
|
|
47
|
+
# Metrics
|
|
48
|
+
"AgentMetrics",
|
|
49
|
+
"MetricsCalculator",
|
|
50
|
+
# Grading
|
|
51
|
+
"ResponseGrader",
|
|
52
|
+
"GradeResult",
|
|
53
|
+
"BatchGradeResult",
|
|
54
|
+
# Statistics
|
|
55
|
+
"FrameworkData",
|
|
56
|
+
"PairwiseComparison",
|
|
57
|
+
"StatisticalReport",
|
|
58
|
+
"HypothesisTestResult",
|
|
59
|
+
"build_comparison_report",
|
|
60
|
+
"cohens_d",
|
|
61
|
+
"cohens_h",
|
|
62
|
+
"compare_distributions",
|
|
63
|
+
"compare_multiple_groups",
|
|
64
|
+
"compare_proportions",
|
|
65
|
+
"mean_ci",
|
|
66
|
+
"proportion_ci",
|
|
67
|
+
]
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
"""Response grading for benchmark tasks.
|
|
2
|
+
|
|
3
|
+
Evaluates agent outputs against expected keyword sets to measure
|
|
4
|
+
accuracy and detect silent failures (wrong answers without errors).
|
|
5
|
+
|
|
6
|
+
Usage::
|
|
7
|
+
|
|
8
|
+
from agentprobe.benchmark.grader import ResponseGrader
|
|
9
|
+
|
|
10
|
+
grader = ResponseGrader(threshold=0.5)
|
|
11
|
+
result = grader.grade("Tokyo has 14 million people", ["tokyo", "population", "million"])
|
|
12
|
+
# result.score = 1.0, result.is_correct = True
|
|
13
|
+
|
|
14
|
+
# Silent failure detection
|
|
15
|
+
batch = grader.grade_batch(
|
|
16
|
+
outputs=["Paris is the capital", "I don't know"],
|
|
17
|
+
expected=[["tokyo", "population"], ["berlin", "capital"]],
|
|
18
|
+
had_errors=[False, False],
|
|
19
|
+
)
|
|
20
|
+
# batch.silent_failure_count = 2 (wrong answers, no exceptions)
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
from collections.abc import Sequence
|
|
26
|
+
from dataclasses import dataclass
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True)
|
|
30
|
+
class GradeResult:
|
|
31
|
+
"""Result of grading a single agent response."""
|
|
32
|
+
|
|
33
|
+
score: float # 0.0-1.0, fraction of keywords matched
|
|
34
|
+
matched_keywords: tuple[str, ...]
|
|
35
|
+
total_keywords: int
|
|
36
|
+
is_correct: bool # score >= threshold
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dataclass(frozen=True)
|
|
40
|
+
class BatchGradeResult:
|
|
41
|
+
"""Aggregated grading results for a batch of responses."""
|
|
42
|
+
|
|
43
|
+
n_total: int
|
|
44
|
+
n_correct: int
|
|
45
|
+
n_incorrect: int
|
|
46
|
+
accuracy: float # n_correct / n_total
|
|
47
|
+
mean_score: float
|
|
48
|
+
silent_failure_count: int # incorrect AND no exception raised
|
|
49
|
+
silent_failure_rate: float # silent_failure_count / n_total
|
|
50
|
+
grades: tuple[GradeResult, ...]
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class ResponseGrader:
|
|
54
|
+
"""Grade agent responses by keyword matching.
|
|
55
|
+
|
|
56
|
+
Args:
|
|
57
|
+
threshold: Minimum keyword match fraction to count as correct.
|
|
58
|
+
Default 0.5 means at least half the expected keywords must appear.
|
|
59
|
+
"""
|
|
60
|
+
|
|
61
|
+
def __init__(self, threshold: float = 0.5) -> None:
|
|
62
|
+
if not 0.0 <= threshold <= 1.0:
|
|
63
|
+
raise ValueError(f"threshold must be in [0, 1], got {threshold}")
|
|
64
|
+
self._threshold = threshold
|
|
65
|
+
|
|
66
|
+
@property
|
|
67
|
+
def threshold(self) -> float:
|
|
68
|
+
return self._threshold
|
|
69
|
+
|
|
70
|
+
def grade(
|
|
71
|
+
self,
|
|
72
|
+
output: str,
|
|
73
|
+
expected_keywords: Sequence[str],
|
|
74
|
+
) -> GradeResult:
|
|
75
|
+
"""Grade a single response against expected keywords.
|
|
76
|
+
|
|
77
|
+
Args:
|
|
78
|
+
output: The agent's text output.
|
|
79
|
+
expected_keywords: Keywords that should appear in a correct response.
|
|
80
|
+
|
|
81
|
+
Returns:
|
|
82
|
+
GradeResult with score, matched keywords, and correctness flag.
|
|
83
|
+
"""
|
|
84
|
+
if not expected_keywords:
|
|
85
|
+
return GradeResult(
|
|
86
|
+
score=1.0,
|
|
87
|
+
matched_keywords=(),
|
|
88
|
+
total_keywords=0,
|
|
89
|
+
is_correct=True,
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
output_lower = output.lower()
|
|
93
|
+
matched = tuple(kw for kw in expected_keywords if kw.lower() in output_lower)
|
|
94
|
+
score = len(matched) / len(expected_keywords)
|
|
95
|
+
|
|
96
|
+
return GradeResult(
|
|
97
|
+
score=score,
|
|
98
|
+
matched_keywords=matched,
|
|
99
|
+
total_keywords=len(expected_keywords),
|
|
100
|
+
is_correct=score >= self._threshold,
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
def grade_batch(
|
|
104
|
+
self,
|
|
105
|
+
outputs: Sequence[str],
|
|
106
|
+
expected: Sequence[Sequence[str]],
|
|
107
|
+
had_errors: Sequence[bool] | None = None,
|
|
108
|
+
) -> BatchGradeResult:
|
|
109
|
+
"""Grade a batch of responses and detect silent failures.
|
|
110
|
+
|
|
111
|
+
Args:
|
|
112
|
+
outputs: Agent output strings.
|
|
113
|
+
expected: Expected keyword lists, one per output.
|
|
114
|
+
had_errors: Whether each run raised an exception. If None,
|
|
115
|
+
assumes no errors (all runs completed without exception).
|
|
116
|
+
|
|
117
|
+
Returns:
|
|
118
|
+
BatchGradeResult with accuracy, scores, and silent failure count.
|
|
119
|
+
"""
|
|
120
|
+
if len(outputs) != len(expected):
|
|
121
|
+
raise ValueError(
|
|
122
|
+
f"outputs ({len(outputs)}) and expected ({len(expected)}) must have same length"
|
|
123
|
+
)
|
|
124
|
+
if had_errors is not None and len(had_errors) != len(outputs):
|
|
125
|
+
raise ValueError(f"had_errors ({len(had_errors)}) must match outputs ({len(outputs)})")
|
|
126
|
+
|
|
127
|
+
if had_errors is None:
|
|
128
|
+
had_errors = [False] * len(outputs)
|
|
129
|
+
|
|
130
|
+
grades = tuple(self.grade(out, exp) for out, exp in zip(outputs, expected))
|
|
131
|
+
n_total = len(grades)
|
|
132
|
+
|
|
133
|
+
if n_total == 0:
|
|
134
|
+
return BatchGradeResult(
|
|
135
|
+
n_total=0,
|
|
136
|
+
n_correct=0,
|
|
137
|
+
n_incorrect=0,
|
|
138
|
+
accuracy=0.0,
|
|
139
|
+
mean_score=0.0,
|
|
140
|
+
silent_failure_count=0,
|
|
141
|
+
silent_failure_rate=0.0,
|
|
142
|
+
grades=(),
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
n_correct = sum(1 for g in grades if g.is_correct)
|
|
146
|
+
n_incorrect = n_total - n_correct
|
|
147
|
+
|
|
148
|
+
# Silent failure: incorrect response AND no exception was raised
|
|
149
|
+
silent_failures = sum(
|
|
150
|
+
1 for g, err in zip(grades, had_errors) if not g.is_correct and not err
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
return BatchGradeResult(
|
|
154
|
+
n_total=n_total,
|
|
155
|
+
n_correct=n_correct,
|
|
156
|
+
n_incorrect=n_incorrect,
|
|
157
|
+
accuracy=n_correct / n_total,
|
|
158
|
+
mean_score=sum(g.score for g in grades) / n_total,
|
|
159
|
+
silent_failure_count=silent_failures,
|
|
160
|
+
silent_failure_rate=silent_failures / n_total,
|
|
161
|
+
grades=grades,
|
|
162
|
+
)
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
"""Experiment metadata capture for reproducibility.
|
|
2
|
+
|
|
3
|
+
Records hardware, software, and configuration details alongside benchmark
|
|
4
|
+
results so that reviewers can assess the experimental context.
|
|
5
|
+
|
|
6
|
+
Usage::
|
|
7
|
+
|
|
8
|
+
from agentprobe.benchmark.metadata import capture_metadata
|
|
9
|
+
|
|
10
|
+
meta = capture_metadata(
|
|
11
|
+
model="llama3.1",
|
|
12
|
+
llm_backend="ollama",
|
|
13
|
+
frameworks=["langgraph", "langchain", "autogen"],
|
|
14
|
+
)
|
|
15
|
+
meta.to_dict() # JSON-serializable
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import os
|
|
21
|
+
import platform
|
|
22
|
+
import subprocess
|
|
23
|
+
import sys
|
|
24
|
+
import time
|
|
25
|
+
from dataclasses import asdict, dataclass, field
|
|
26
|
+
from typing import Any
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True)
|
|
30
|
+
class ExperimentMetadata:
|
|
31
|
+
"""Immutable record of experiment environment and configuration."""
|
|
32
|
+
|
|
33
|
+
# Hardware
|
|
34
|
+
cpu_model: str
|
|
35
|
+
cpu_count: int
|
|
36
|
+
ram_gb: float
|
|
37
|
+
os_name: str
|
|
38
|
+
os_version: str
|
|
39
|
+
arch: str
|
|
40
|
+
|
|
41
|
+
# Software
|
|
42
|
+
python_version: str
|
|
43
|
+
agentprobe_version: str
|
|
44
|
+
git_hash: str
|
|
45
|
+
git_branch: str
|
|
46
|
+
git_dirty: bool
|
|
47
|
+
|
|
48
|
+
# LLM configuration
|
|
49
|
+
llm_backend: str
|
|
50
|
+
model_name: str
|
|
51
|
+
ollama_version: str
|
|
52
|
+
|
|
53
|
+
# Experiment configuration
|
|
54
|
+
frameworks: tuple[str, ...]
|
|
55
|
+
n_runs_per_cell: int
|
|
56
|
+
seed: int
|
|
57
|
+
timestamp_utc: str
|
|
58
|
+
|
|
59
|
+
# Dependency versions
|
|
60
|
+
dependency_versions: dict[str, str] = field(default_factory=dict)
|
|
61
|
+
|
|
62
|
+
def to_dict(self) -> dict[str, Any]:
|
|
63
|
+
d = asdict(self)
|
|
64
|
+
d["frameworks"] = list(d["frameworks"])
|
|
65
|
+
return d
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _get_git_info() -> tuple[str, str, bool]:
|
|
69
|
+
"""Return (commit_hash, branch, is_dirty)."""
|
|
70
|
+
try:
|
|
71
|
+
commit = subprocess.check_output(
|
|
72
|
+
["git", "rev-parse", "--short", "HEAD"],
|
|
73
|
+
stderr=subprocess.DEVNULL,
|
|
74
|
+
text=True,
|
|
75
|
+
).strip()
|
|
76
|
+
branch = subprocess.check_output(
|
|
77
|
+
["git", "rev-parse", "--abbrev-ref", "HEAD"],
|
|
78
|
+
stderr=subprocess.DEVNULL,
|
|
79
|
+
text=True,
|
|
80
|
+
).strip()
|
|
81
|
+
status = subprocess.check_output(
|
|
82
|
+
["git", "status", "--porcelain"],
|
|
83
|
+
stderr=subprocess.DEVNULL,
|
|
84
|
+
text=True,
|
|
85
|
+
).strip()
|
|
86
|
+
return commit, branch, len(status) > 0
|
|
87
|
+
except (subprocess.CalledProcessError, FileNotFoundError):
|
|
88
|
+
return "unknown", "unknown", False
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _get_ollama_version() -> str:
|
|
92
|
+
"""Return Ollama version string, or 'not available'."""
|
|
93
|
+
try:
|
|
94
|
+
out = subprocess.check_output(
|
|
95
|
+
["ollama", "--version"],
|
|
96
|
+
stderr=subprocess.DEVNULL,
|
|
97
|
+
text=True,
|
|
98
|
+
).strip()
|
|
99
|
+
return out.split()[-1] if out else "unknown"
|
|
100
|
+
except (subprocess.CalledProcessError, FileNotFoundError):
|
|
101
|
+
return "not available"
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _get_ram_gb() -> float:
|
|
105
|
+
"""Return total system RAM in GB."""
|
|
106
|
+
try:
|
|
107
|
+
if platform.system() == "Darwin":
|
|
108
|
+
out = subprocess.check_output(
|
|
109
|
+
["sysctl", "-n", "hw.memsize"],
|
|
110
|
+
stderr=subprocess.DEVNULL,
|
|
111
|
+
text=True,
|
|
112
|
+
).strip()
|
|
113
|
+
return round(int(out) / (1024**3), 1)
|
|
114
|
+
elif platform.system() == "Linux":
|
|
115
|
+
with open("/proc/meminfo") as f:
|
|
116
|
+
for line in f:
|
|
117
|
+
if line.startswith("MemTotal"):
|
|
118
|
+
kb = int(line.split()[1])
|
|
119
|
+
return round(kb / (1024**2), 1)
|
|
120
|
+
return 0.0
|
|
121
|
+
except (subprocess.CalledProcessError, FileNotFoundError, ValueError):
|
|
122
|
+
return 0.0
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _get_cpu_model() -> str:
|
|
126
|
+
"""Return CPU model string."""
|
|
127
|
+
try:
|
|
128
|
+
if platform.system() == "Darwin":
|
|
129
|
+
out = subprocess.check_output(
|
|
130
|
+
["sysctl", "-n", "machdep.cpu.brand_string"],
|
|
131
|
+
stderr=subprocess.DEVNULL,
|
|
132
|
+
text=True,
|
|
133
|
+
).strip()
|
|
134
|
+
return out
|
|
135
|
+
elif platform.system() == "Linux":
|
|
136
|
+
with open("/proc/cpuinfo") as f:
|
|
137
|
+
for line in f:
|
|
138
|
+
if line.startswith("model name"):
|
|
139
|
+
return line.split(":", 1)[1].strip()
|
|
140
|
+
return platform.processor() or "unknown"
|
|
141
|
+
except (subprocess.CalledProcessError, FileNotFoundError):
|
|
142
|
+
return platform.processor() or "unknown"
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _get_dependency_versions() -> dict[str, str]:
|
|
146
|
+
"""Return versions of key dependencies."""
|
|
147
|
+
deps: dict[str, str] = {}
|
|
148
|
+
for pkg in [
|
|
149
|
+
"langgraph",
|
|
150
|
+
"langchain",
|
|
151
|
+
"langchain_core",
|
|
152
|
+
"langchain_community",
|
|
153
|
+
"opentelemetry",
|
|
154
|
+
"pydantic",
|
|
155
|
+
"numpy",
|
|
156
|
+
"pandas",
|
|
157
|
+
"scipy",
|
|
158
|
+
]:
|
|
159
|
+
try:
|
|
160
|
+
mod = __import__(pkg)
|
|
161
|
+
deps[pkg] = getattr(mod, "__version__", "unknown")
|
|
162
|
+
except ImportError:
|
|
163
|
+
deps[pkg] = "not installed"
|
|
164
|
+
|
|
165
|
+
# pyautogen uses a different import name
|
|
166
|
+
try:
|
|
167
|
+
import autogen # type: ignore[import-untyped]
|
|
168
|
+
|
|
169
|
+
deps["pyautogen"] = getattr(autogen, "__version__", "unknown")
|
|
170
|
+
except ImportError:
|
|
171
|
+
deps["pyautogen"] = "not installed"
|
|
172
|
+
|
|
173
|
+
return deps
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def capture_metadata(
|
|
177
|
+
model: str,
|
|
178
|
+
llm_backend: str = "ollama",
|
|
179
|
+
frameworks: list[str] | None = None,
|
|
180
|
+
n_runs_per_cell: int = 10,
|
|
181
|
+
seed: int = 42,
|
|
182
|
+
) -> ExperimentMetadata:
|
|
183
|
+
"""Capture full experiment metadata snapshot.
|
|
184
|
+
|
|
185
|
+
Call this once at experiment start and save alongside results.
|
|
186
|
+
"""
|
|
187
|
+
git_hash, git_branch, git_dirty = _get_git_info()
|
|
188
|
+
|
|
189
|
+
try:
|
|
190
|
+
from agentprobe import __version__ as ap_version
|
|
191
|
+
except ImportError:
|
|
192
|
+
ap_version = "unknown"
|
|
193
|
+
|
|
194
|
+
return ExperimentMetadata(
|
|
195
|
+
cpu_model=_get_cpu_model(),
|
|
196
|
+
cpu_count=os.cpu_count() or 0,
|
|
197
|
+
ram_gb=_get_ram_gb(),
|
|
198
|
+
os_name=platform.system(),
|
|
199
|
+
os_version=platform.release(),
|
|
200
|
+
arch=platform.machine(),
|
|
201
|
+
python_version=sys.version.split()[0],
|
|
202
|
+
agentprobe_version=ap_version,
|
|
203
|
+
git_hash=git_hash,
|
|
204
|
+
git_branch=git_branch,
|
|
205
|
+
git_dirty=git_dirty,
|
|
206
|
+
llm_backend=llm_backend,
|
|
207
|
+
model_name=model,
|
|
208
|
+
ollama_version=_get_ollama_version() if llm_backend == "ollama" else "n/a",
|
|
209
|
+
frameworks=tuple(frameworks or ["langgraph", "langchain", "autogen"]),
|
|
210
|
+
n_runs_per_cell=n_runs_per_cell,
|
|
211
|
+
seed=seed,
|
|
212
|
+
timestamp_utc=time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
213
|
+
dependency_versions=_get_dependency_versions(),
|
|
214
|
+
)
|