agentprobe-toolkit 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentprobe/__init__.py +42 -0
- agentprobe/agents/__init__.py +5 -0
- agentprobe/agents/autogen_agent.py +308 -0
- agentprobe/agents/base.py +97 -0
- agentprobe/agents/langchain_agent.py +140 -0
- agentprobe/agents/langgraph_agent.py +203 -0
- agentprobe/benchmark/__init__.py +67 -0
- agentprobe/benchmark/grader.py +162 -0
- agentprobe/benchmark/metadata.py +214 -0
- agentprobe/benchmark/metrics.py +247 -0
- agentprobe/benchmark/runner.py +434 -0
- agentprobe/benchmark/statistics.py +459 -0
- agentprobe/benchmark/tasks/__init__.py +75 -0
- agentprobe/benchmark/tasks/multi_agent.py +191 -0
- agentprobe/benchmark/tasks/rag.py +309 -0
- agentprobe/benchmark/tasks/tool_use.py +228 -0
- agentprobe/config.py +67 -0
- agentprobe/injector/__init__.py +66 -0
- agentprobe/injector/context_failure.py +328 -0
- agentprobe/injector/latency_failure.py +275 -0
- agentprobe/injector/orchestration_failure.py +332 -0
- agentprobe/injector/tool_failure.py +404 -0
- agentprobe/observer/__init__.py +5 -0
- agentprobe/observer/classifier.py +320 -0
- agentprobe/observer/dashboard.py +216 -0
- agentprobe/observer/tracer.py +137 -0
- agentprobe_toolkit-1.2.0.dist-info/METADATA +405 -0
- agentprobe_toolkit-1.2.0.dist-info/RECORD +30 -0
- agentprobe_toolkit-1.2.0.dist-info/WHEEL +4 -0
- agentprobe_toolkit-1.2.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,320 @@
|
|
|
1
|
+
"""Failure classification for AgentProbe.
|
|
2
|
+
|
|
3
|
+
The classifier takes span attributes, exception types, and log records
|
|
4
|
+
and maps them to the 5-category AgentProbe failure taxonomy:
|
|
5
|
+
|
|
6
|
+
1. TOOL_CALL — tool_failure.py exceptions / tool span errors
|
|
7
|
+
2. ORCHESTRATION — orchestration_failure.py exceptions / routing errors
|
|
8
|
+
3. CONTEXT — context_failure.py exceptions / token overflow
|
|
9
|
+
4. LATENCY — high-latency spans / tail latency detection
|
|
10
|
+
5. CONSISTENCY — non-deterministic output patterns / divergent state
|
|
11
|
+
|
|
12
|
+
Usage::
|
|
13
|
+
|
|
14
|
+
classifier = FailureClassifier()
|
|
15
|
+
|
|
16
|
+
# From an exception
|
|
17
|
+
event = classifier.classify_exception(exc, context={"tool_name": "search"})
|
|
18
|
+
|
|
19
|
+
# From a span (OpenTelemetry SpanData)
|
|
20
|
+
events = classifier.classify_spans(finished_spans)
|
|
21
|
+
|
|
22
|
+
# From a latency measurement
|
|
23
|
+
event = classifier.classify_latency(latency_ms=4800, p99_threshold_ms=2000)
|
|
24
|
+
|
|
25
|
+
# Batch classification from benchmark run output
|
|
26
|
+
report = classifier.build_report(events)
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
from __future__ import annotations
|
|
30
|
+
|
|
31
|
+
import time
|
|
32
|
+
from dataclasses import dataclass, field
|
|
33
|
+
from enum import StrEnum
|
|
34
|
+
from typing import Any
|
|
35
|
+
|
|
36
|
+
# ---------------------------------------------------------------------------
|
|
37
|
+
# Taxonomy
|
|
38
|
+
# ---------------------------------------------------------------------------
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class FailureCategory(StrEnum):
|
|
42
|
+
TOOL_CALL = "tool_call"
|
|
43
|
+
ORCHESTRATION = "orchestration"
|
|
44
|
+
CONTEXT = "context"
|
|
45
|
+
LATENCY = "latency"
|
|
46
|
+
CONSISTENCY = "consistency"
|
|
47
|
+
UNKNOWN = "unknown"
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class FailureSeverity(StrEnum):
|
|
51
|
+
LOW = "low" # Transient, likely self-healing
|
|
52
|
+
MEDIUM = "medium" # Impacts single agent turn
|
|
53
|
+
HIGH = "high" # Impacts full task completion
|
|
54
|
+
CRITICAL = "critical" # Causes cascading failures
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
# ---------------------------------------------------------------------------
|
|
58
|
+
# Failure event
|
|
59
|
+
# ---------------------------------------------------------------------------
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclass
|
|
63
|
+
class FailureEvent:
|
|
64
|
+
"""A classified failure event from an agent execution."""
|
|
65
|
+
|
|
66
|
+
category: FailureCategory
|
|
67
|
+
severity: FailureSeverity
|
|
68
|
+
description: str
|
|
69
|
+
timestamp: float = field(default_factory=time.time)
|
|
70
|
+
framework: str = "unknown"
|
|
71
|
+
task_id: str = ""
|
|
72
|
+
span_id: str = ""
|
|
73
|
+
exception_type: str = ""
|
|
74
|
+
raw_attributes: dict[str, Any] = field(default_factory=dict)
|
|
75
|
+
|
|
76
|
+
def to_dict(self) -> dict[str, Any]:
|
|
77
|
+
return {
|
|
78
|
+
"category": self.category.value,
|
|
79
|
+
"severity": self.severity.value,
|
|
80
|
+
"description": self.description,
|
|
81
|
+
"timestamp": self.timestamp,
|
|
82
|
+
"framework": self.framework,
|
|
83
|
+
"task_id": self.task_id,
|
|
84
|
+
"span_id": self.span_id,
|
|
85
|
+
"exception_type": self.exception_type,
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
# ---------------------------------------------------------------------------
|
|
90
|
+
# Classification rules
|
|
91
|
+
# ---------------------------------------------------------------------------
|
|
92
|
+
|
|
93
|
+
# Maps exception type names (or prefixes) to (category, severity)
|
|
94
|
+
_EXCEPTION_RULES: list[tuple[str, FailureCategory, FailureSeverity]] = [
|
|
95
|
+
# Tool failures
|
|
96
|
+
("ToolCallTimeout", FailureCategory.TOOL_CALL, FailureSeverity.HIGH),
|
|
97
|
+
("ToolCallMalformedOutput", FailureCategory.TOOL_CALL, FailureSeverity.MEDIUM),
|
|
98
|
+
("ToolCallRateLimited", FailureCategory.TOOL_CALL, FailureSeverity.MEDIUM),
|
|
99
|
+
("ToolCallAPIError", FailureCategory.TOOL_CALL, FailureSeverity.HIGH),
|
|
100
|
+
("ToolCallError", FailureCategory.TOOL_CALL, FailureSeverity.MEDIUM),
|
|
101
|
+
# Orchestration failures
|
|
102
|
+
("InfiniteLoopError", FailureCategory.ORCHESTRATION, FailureSeverity.CRITICAL),
|
|
103
|
+
("WrongBranchError", FailureCategory.ORCHESTRATION, FailureSeverity.HIGH),
|
|
104
|
+
("StateCorruptionError", FailureCategory.ORCHESTRATION, FailureSeverity.CRITICAL),
|
|
105
|
+
("MissingHandoffError", FailureCategory.ORCHESTRATION, FailureSeverity.HIGH),
|
|
106
|
+
("OrchestrationError", FailureCategory.ORCHESTRATION, FailureSeverity.HIGH),
|
|
107
|
+
# Context failures
|
|
108
|
+
("ContextWindowOverflow", FailureCategory.CONTEXT, FailureSeverity.HIGH),
|
|
109
|
+
("LostStateError", FailureCategory.CONTEXT, FailureSeverity.MEDIUM),
|
|
110
|
+
("HallucinatedHistoryError", FailureCategory.CONTEXT, FailureSeverity.MEDIUM),
|
|
111
|
+
("ContextError", FailureCategory.CONTEXT, FailureSeverity.MEDIUM),
|
|
112
|
+
# Common LangChain / LLM API exceptions
|
|
113
|
+
("RateLimitError", FailureCategory.TOOL_CALL, FailureSeverity.MEDIUM),
|
|
114
|
+
("APITimeoutError", FailureCategory.TOOL_CALL, FailureSeverity.HIGH),
|
|
115
|
+
("APIConnectionError", FailureCategory.TOOL_CALL, FailureSeverity.HIGH),
|
|
116
|
+
("BadRequestError", FailureCategory.CONTEXT, FailureSeverity.MEDIUM),
|
|
117
|
+
("InvalidRequestError", FailureCategory.CONTEXT, FailureSeverity.MEDIUM),
|
|
118
|
+
("GraphRecursionError", FailureCategory.ORCHESTRATION, FailureSeverity.CRITICAL),
|
|
119
|
+
]
|
|
120
|
+
|
|
121
|
+
# Span attribute keys that signal failure categories
|
|
122
|
+
_SPAN_ATTRIBUTE_RULES: dict[str, FailureCategory] = {
|
|
123
|
+
"tool.name": FailureCategory.TOOL_CALL,
|
|
124
|
+
"node.name": FailureCategory.ORCHESTRATION,
|
|
125
|
+
"router.name": FailureCategory.ORCHESTRATION,
|
|
126
|
+
"context.token_estimate": FailureCategory.CONTEXT,
|
|
127
|
+
"latency.injected_delay_ms": FailureCategory.LATENCY,
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
# ---------------------------------------------------------------------------
|
|
132
|
+
# Classifier
|
|
133
|
+
# ---------------------------------------------------------------------------
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
class FailureClassifier:
|
|
137
|
+
"""Classifies failures into the AgentProbe 5-category taxonomy.
|
|
138
|
+
|
|
139
|
+
Stateless — all methods are pure functions over their inputs.
|
|
140
|
+
"""
|
|
141
|
+
|
|
142
|
+
def classify_exception(
|
|
143
|
+
self,
|
|
144
|
+
exc: Exception,
|
|
145
|
+
context: dict[str, Any] | None = None,
|
|
146
|
+
framework: str = "unknown",
|
|
147
|
+
task_id: str = "",
|
|
148
|
+
span_id: str = "",
|
|
149
|
+
) -> FailureEvent:
|
|
150
|
+
"""Classify a Python exception into a :class:`FailureEvent`."""
|
|
151
|
+
exc_type = type(exc).__name__
|
|
152
|
+
category, severity = self._match_exception(exc_type)
|
|
153
|
+
return FailureEvent(
|
|
154
|
+
category=category,
|
|
155
|
+
severity=severity,
|
|
156
|
+
description=str(exc),
|
|
157
|
+
framework=framework,
|
|
158
|
+
task_id=task_id,
|
|
159
|
+
span_id=span_id,
|
|
160
|
+
exception_type=exc_type,
|
|
161
|
+
raw_attributes=context or {},
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
def classify_latency(
|
|
165
|
+
self,
|
|
166
|
+
latency_ms: float,
|
|
167
|
+
p99_threshold_ms: float = 2000.0,
|
|
168
|
+
framework: str = "unknown",
|
|
169
|
+
task_id: str = "",
|
|
170
|
+
) -> FailureEvent | None:
|
|
171
|
+
"""Return a LATENCY FailureEvent if latency exceeds the P99 threshold, else None."""
|
|
172
|
+
if latency_ms <= p99_threshold_ms:
|
|
173
|
+
return None
|
|
174
|
+
severity = (
|
|
175
|
+
FailureSeverity.CRITICAL
|
|
176
|
+
if latency_ms > p99_threshold_ms * 5
|
|
177
|
+
else FailureSeverity.HIGH
|
|
178
|
+
if latency_ms > p99_threshold_ms * 2
|
|
179
|
+
else FailureSeverity.MEDIUM
|
|
180
|
+
)
|
|
181
|
+
return FailureEvent(
|
|
182
|
+
category=FailureCategory.LATENCY,
|
|
183
|
+
severity=severity,
|
|
184
|
+
description=(
|
|
185
|
+
f"Latency spike: {latency_ms:.1f}ms exceeds P99 threshold of "
|
|
186
|
+
f"{p99_threshold_ms:.1f}ms"
|
|
187
|
+
),
|
|
188
|
+
framework=framework,
|
|
189
|
+
task_id=task_id,
|
|
190
|
+
raw_attributes={"latency_ms": latency_ms, "p99_threshold_ms": p99_threshold_ms},
|
|
191
|
+
)
|
|
192
|
+
|
|
193
|
+
def classify_output_pair(
|
|
194
|
+
self,
|
|
195
|
+
output_a: str,
|
|
196
|
+
output_b: str,
|
|
197
|
+
task_id: str = "",
|
|
198
|
+
framework: str = "unknown",
|
|
199
|
+
) -> FailureEvent | None:
|
|
200
|
+
"""Detect consistency failures by comparing two outputs for the same input.
|
|
201
|
+
|
|
202
|
+
Returns a CONSISTENCY event if outputs are meaningfully different, else None.
|
|
203
|
+
Uses simple normalized edit-distance heuristic — no LLM calls required.
|
|
204
|
+
"""
|
|
205
|
+
if output_a == output_b:
|
|
206
|
+
return None
|
|
207
|
+
similarity = _jaccard_similarity(output_a, output_b)
|
|
208
|
+
if similarity >= 0.85:
|
|
209
|
+
return None # Close enough — treat as consistent
|
|
210
|
+
|
|
211
|
+
severity = FailureSeverity.HIGH if similarity < 0.5 else FailureSeverity.MEDIUM
|
|
212
|
+
return FailureEvent(
|
|
213
|
+
category=FailureCategory.CONSISTENCY,
|
|
214
|
+
severity=severity,
|
|
215
|
+
description=(
|
|
216
|
+
f"Non-deterministic output detected (Jaccard similarity={similarity:.2f})"
|
|
217
|
+
),
|
|
218
|
+
framework=framework,
|
|
219
|
+
task_id=task_id,
|
|
220
|
+
raw_attributes={"jaccard_similarity": similarity},
|
|
221
|
+
)
|
|
222
|
+
|
|
223
|
+
def classify_spans(self, spans: list[Any]) -> list[FailureEvent]:
|
|
224
|
+
"""Classify a list of finished OTel SpanData objects into failure events.
|
|
225
|
+
|
|
226
|
+
Spans are expected to have .name, .attributes, .status, .events attributes
|
|
227
|
+
(matching the opentelemetry-sdk ReadableSpan interface).
|
|
228
|
+
"""
|
|
229
|
+
events: list[FailureEvent] = []
|
|
230
|
+
for span in spans:
|
|
231
|
+
event = self._classify_single_span(span)
|
|
232
|
+
if event:
|
|
233
|
+
events.append(event)
|
|
234
|
+
return events
|
|
235
|
+
|
|
236
|
+
def build_report(self, events: list[FailureEvent]) -> dict[str, Any]:
|
|
237
|
+
"""Aggregate a list of failure events into a summary report."""
|
|
238
|
+
if not events:
|
|
239
|
+
return {"total_failures": 0, "by_category": {}, "by_severity": {}}
|
|
240
|
+
|
|
241
|
+
by_cat: dict[str, int] = {}
|
|
242
|
+
by_sev: dict[str, int] = {}
|
|
243
|
+
for e in events:
|
|
244
|
+
by_cat[e.category.value] = by_cat.get(e.category.value, 0) + 1
|
|
245
|
+
by_sev[e.severity.value] = by_sev.get(e.severity.value, 0) + 1
|
|
246
|
+
|
|
247
|
+
return {
|
|
248
|
+
"total_failures": len(events),
|
|
249
|
+
"by_category": by_cat,
|
|
250
|
+
"by_severity": by_sev,
|
|
251
|
+
"critical_count": by_sev.get("critical", 0),
|
|
252
|
+
"frameworks": list({e.framework for e in events}),
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
# ------------------------------------------------------------------
|
|
256
|
+
# Internal
|
|
257
|
+
# ------------------------------------------------------------------
|
|
258
|
+
|
|
259
|
+
def _match_exception(self, exc_type: str) -> tuple[FailureCategory, FailureSeverity]:
|
|
260
|
+
for prefix, category, severity in _EXCEPTION_RULES:
|
|
261
|
+
if exc_type == prefix or exc_type.startswith(prefix):
|
|
262
|
+
return category, severity
|
|
263
|
+
return FailureCategory.UNKNOWN, FailureSeverity.MEDIUM
|
|
264
|
+
|
|
265
|
+
def _classify_single_span(self, span: Any) -> FailureEvent | None:
|
|
266
|
+
"""Classify one OTel ReadableSpan. Returns None if no failure detected."""
|
|
267
|
+
from opentelemetry.trace import StatusCode
|
|
268
|
+
|
|
269
|
+
if not hasattr(span, "status"):
|
|
270
|
+
return None
|
|
271
|
+
if span.status.status_code != StatusCode.ERROR:
|
|
272
|
+
return None
|
|
273
|
+
|
|
274
|
+
attrs = dict(span.attributes or {})
|
|
275
|
+
span_name = getattr(span, "name", "unknown")
|
|
276
|
+
|
|
277
|
+
# Determine category from attributes
|
|
278
|
+
category = FailureCategory.UNKNOWN
|
|
279
|
+
for attr_key, cat in _SPAN_ATTRIBUTE_RULES.items():
|
|
280
|
+
if attr_key in attrs:
|
|
281
|
+
category = cat
|
|
282
|
+
break
|
|
283
|
+
|
|
284
|
+
# Try to infer from span name prefix
|
|
285
|
+
if category == FailureCategory.UNKNOWN:
|
|
286
|
+
if span_name.startswith("tool."):
|
|
287
|
+
category = FailureCategory.TOOL_CALL
|
|
288
|
+
elif span_name.startswith("orchestration."):
|
|
289
|
+
category = FailureCategory.ORCHESTRATION
|
|
290
|
+
elif span_name.startswith("context."):
|
|
291
|
+
category = FailureCategory.CONTEXT
|
|
292
|
+
elif span_name.startswith("latency."):
|
|
293
|
+
category = FailureCategory.LATENCY
|
|
294
|
+
|
|
295
|
+
failure_type = attrs.get("failure.type", "unknown")
|
|
296
|
+
span_id = format(span.context.span_id, "016x") if span.context else ""
|
|
297
|
+
|
|
298
|
+
return FailureEvent(
|
|
299
|
+
category=category,
|
|
300
|
+
severity=FailureSeverity.HIGH,
|
|
301
|
+
description=f"Span '{span_name}' failed with type '{failure_type}'",
|
|
302
|
+
span_id=span_id,
|
|
303
|
+
raw_attributes=attrs,
|
|
304
|
+
)
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
# ---------------------------------------------------------------------------
|
|
308
|
+
# Helpers
|
|
309
|
+
# ---------------------------------------------------------------------------
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def _jaccard_similarity(a: str, b: str) -> float:
|
|
313
|
+
"""Token-level Jaccard similarity between two strings."""
|
|
314
|
+
tokens_a = set(a.lower().split())
|
|
315
|
+
tokens_b = set(b.lower().split())
|
|
316
|
+
if not tokens_a and not tokens_b:
|
|
317
|
+
return 1.0
|
|
318
|
+
intersection = tokens_a & tokens_b
|
|
319
|
+
union = tokens_a | tokens_b
|
|
320
|
+
return len(intersection) / len(union)
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
"""Prometheus metrics exporter for AgentProbe.
|
|
2
|
+
|
|
3
|
+
Exports agent reliability metrics to Prometheus, enabling Grafana dashboards
|
|
4
|
+
for production monitoring of agentic LLM systems.
|
|
5
|
+
|
|
6
|
+
Metrics exported:
|
|
7
|
+
agentprobe_tool_call_total — Counter of tool calls by framework/task/result
|
|
8
|
+
agentprobe_tool_failure_total — Counter of tool failures by type
|
|
9
|
+
agentprobe_latency_seconds — Histogram of agent turn latencies
|
|
10
|
+
agentprobe_mttr_seconds — Gauge of mean-time-to-recovery
|
|
11
|
+
agentprobe_failure_rate — Gauge of current failure rate [0, 1]
|
|
12
|
+
agentprobe_task_success_total — Counter of tasks completed successfully
|
|
13
|
+
agentprobe_task_failure_total — Counter of tasks that failed
|
|
14
|
+
|
|
15
|
+
Usage::
|
|
16
|
+
|
|
17
|
+
from agentprobe.observer.dashboard import PrometheusExporter
|
|
18
|
+
|
|
19
|
+
exporter = PrometheusExporter(port=8000)
|
|
20
|
+
exporter.start() # starts /metrics HTTP server on :8000
|
|
21
|
+
|
|
22
|
+
# Record metrics manually
|
|
23
|
+
exporter.record_tool_call(framework="langgraph", task="rag", success=False,
|
|
24
|
+
failure_type="timeout")
|
|
25
|
+
exporter.record_latency(framework="langgraph", latency_seconds=2.4)
|
|
26
|
+
exporter.set_failure_rate(framework="langgraph", rate=0.12)
|
|
27
|
+
|
|
28
|
+
# Or auto-record from benchmark results
|
|
29
|
+
exporter.record_benchmark_run(results)
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
from __future__ import annotations
|
|
33
|
+
|
|
34
|
+
import threading
|
|
35
|
+
from typing import Any
|
|
36
|
+
|
|
37
|
+
try:
|
|
38
|
+
from prometheus_client import (
|
|
39
|
+
Counter,
|
|
40
|
+
Gauge,
|
|
41
|
+
Histogram,
|
|
42
|
+
start_http_server,
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
_PROMETHEUS_AVAILABLE = True
|
|
46
|
+
except ImportError:
|
|
47
|
+
_PROMETHEUS_AVAILABLE = False
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
# Default latency histogram buckets (seconds) covering agent response times
|
|
51
|
+
_LATENCY_BUCKETS = (0.1, 0.25, 0.5, 1.0, 2.0, 5.0, 10.0, 30.0, 60.0, float("inf"))
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class PrometheusExporter:
|
|
55
|
+
"""Prometheus metrics exporter for AgentProbe benchmark results.
|
|
56
|
+
|
|
57
|
+
Falls back to a no-op implementation if prometheus_client is not installed,
|
|
58
|
+
so benchmarks can run without a Prometheus server.
|
|
59
|
+
|
|
60
|
+
Args:
|
|
61
|
+
port: HTTP port for the /metrics endpoint (default: 8000).
|
|
62
|
+
prefix: Metric name prefix (default: "agentprobe").
|
|
63
|
+
"""
|
|
64
|
+
|
|
65
|
+
def __init__(self, port: int = 8000, prefix: str = "agentprobe") -> None:
|
|
66
|
+
self.port = port
|
|
67
|
+
self.prefix = prefix
|
|
68
|
+
self._server_started = False
|
|
69
|
+
self._lock = threading.Lock()
|
|
70
|
+
|
|
71
|
+
if not _PROMETHEUS_AVAILABLE:
|
|
72
|
+
import warnings
|
|
73
|
+
|
|
74
|
+
warnings.warn(
|
|
75
|
+
"prometheus_client not installed. Metrics export is disabled. "
|
|
76
|
+
"Install with: pip install prometheus-client",
|
|
77
|
+
stacklevel=2,
|
|
78
|
+
)
|
|
79
|
+
self._available = False
|
|
80
|
+
return
|
|
81
|
+
|
|
82
|
+
self._available = True
|
|
83
|
+
p = prefix
|
|
84
|
+
|
|
85
|
+
self._tool_call_total = Counter(
|
|
86
|
+
f"{p}_tool_call_total",
|
|
87
|
+
"Total tool calls",
|
|
88
|
+
["framework", "task", "result"],
|
|
89
|
+
)
|
|
90
|
+
self._tool_failure_total = Counter(
|
|
91
|
+
f"{p}_tool_failure_total",
|
|
92
|
+
"Total tool failures by failure type",
|
|
93
|
+
["framework", "task", "failure_type"],
|
|
94
|
+
)
|
|
95
|
+
self._latency_histogram = Histogram(
|
|
96
|
+
f"{p}_latency_seconds",
|
|
97
|
+
"Agent turn latency in seconds",
|
|
98
|
+
["framework", "task"],
|
|
99
|
+
buckets=_LATENCY_BUCKETS,
|
|
100
|
+
)
|
|
101
|
+
self._mttr_gauge = Gauge(
|
|
102
|
+
f"{p}_mttr_seconds",
|
|
103
|
+
"Mean time to recovery in seconds",
|
|
104
|
+
["framework"],
|
|
105
|
+
)
|
|
106
|
+
self._failure_rate_gauge = Gauge(
|
|
107
|
+
f"{p}_failure_rate",
|
|
108
|
+
"Current failure rate [0, 1]",
|
|
109
|
+
["framework"],
|
|
110
|
+
)
|
|
111
|
+
self._task_success_total = Counter(
|
|
112
|
+
f"{p}_task_success_total",
|
|
113
|
+
"Total tasks completed successfully",
|
|
114
|
+
["framework", "task"],
|
|
115
|
+
)
|
|
116
|
+
self._task_failure_total = Counter(
|
|
117
|
+
f"{p}_task_failure_total",
|
|
118
|
+
"Total tasks that failed",
|
|
119
|
+
["framework", "task"],
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
# ------------------------------------------------------------------
|
|
123
|
+
# Server lifecycle
|
|
124
|
+
# ------------------------------------------------------------------
|
|
125
|
+
|
|
126
|
+
def start(self) -> None:
|
|
127
|
+
"""Start the Prometheus /metrics HTTP server (non-blocking)."""
|
|
128
|
+
if not self._available:
|
|
129
|
+
return
|
|
130
|
+
with self._lock:
|
|
131
|
+
if not self._server_started:
|
|
132
|
+
start_http_server(self.port)
|
|
133
|
+
self._server_started = True
|
|
134
|
+
|
|
135
|
+
# ------------------------------------------------------------------
|
|
136
|
+
# Recording API
|
|
137
|
+
# ------------------------------------------------------------------
|
|
138
|
+
|
|
139
|
+
def record_tool_call(
|
|
140
|
+
self,
|
|
141
|
+
framework: str,
|
|
142
|
+
task: str,
|
|
143
|
+
success: bool,
|
|
144
|
+
failure_type: str = "none",
|
|
145
|
+
) -> None:
|
|
146
|
+
if not self._available:
|
|
147
|
+
return
|
|
148
|
+
result = "success" if success else "failure"
|
|
149
|
+
self._tool_call_total.labels(framework=framework, task=task, result=result).inc()
|
|
150
|
+
if not success:
|
|
151
|
+
self._tool_failure_total.labels(
|
|
152
|
+
framework=framework, task=task, failure_type=failure_type
|
|
153
|
+
).inc()
|
|
154
|
+
|
|
155
|
+
def record_latency(
|
|
156
|
+
self,
|
|
157
|
+
framework: str,
|
|
158
|
+
latency_seconds: float,
|
|
159
|
+
task: str = "unknown",
|
|
160
|
+
) -> None:
|
|
161
|
+
if not self._available:
|
|
162
|
+
return
|
|
163
|
+
self._latency_histogram.labels(framework=framework, task=task).observe(latency_seconds)
|
|
164
|
+
|
|
165
|
+
def set_mttr(self, framework: str, mttr_seconds: float) -> None:
|
|
166
|
+
if not self._available:
|
|
167
|
+
return
|
|
168
|
+
self._mttr_gauge.labels(framework=framework).set(mttr_seconds)
|
|
169
|
+
|
|
170
|
+
def set_failure_rate(self, framework: str, rate: float) -> None:
|
|
171
|
+
if not self._available:
|
|
172
|
+
return
|
|
173
|
+
self._failure_rate_gauge.labels(framework=framework).set(rate)
|
|
174
|
+
|
|
175
|
+
def record_task_result(self, framework: str, task: str, success: bool) -> None:
|
|
176
|
+
if not self._available:
|
|
177
|
+
return
|
|
178
|
+
if success:
|
|
179
|
+
self._task_success_total.labels(framework=framework, task=task).inc()
|
|
180
|
+
else:
|
|
181
|
+
self._task_failure_total.labels(framework=framework, task=task).inc()
|
|
182
|
+
|
|
183
|
+
def record_benchmark_run(self, results: dict[str, Any]) -> None:
|
|
184
|
+
"""Record a complete benchmark run result dict (as produced by BenchmarkRunner).
|
|
185
|
+
|
|
186
|
+
Expected keys: framework, task, runs (list of RunResult dicts).
|
|
187
|
+
"""
|
|
188
|
+
if not self._available:
|
|
189
|
+
return
|
|
190
|
+
framework = results.get("framework", "unknown")
|
|
191
|
+
task = results.get("task", "unknown")
|
|
192
|
+
runs: list[dict[str, Any]] = results.get("runs", [])
|
|
193
|
+
|
|
194
|
+
for run in runs:
|
|
195
|
+
success = run.get("success", False)
|
|
196
|
+
latency = run.get("latency_ms", 0.0) / 1000.0
|
|
197
|
+
failure_type = run.get("failure_type", "none")
|
|
198
|
+
|
|
199
|
+
self.record_tool_call(framework, task, success, failure_type)
|
|
200
|
+
self.record_latency(framework, latency, task)
|
|
201
|
+
self.record_task_result(framework, task, success)
|
|
202
|
+
|
|
203
|
+
# Compute aggregate metrics
|
|
204
|
+
total = len(runs)
|
|
205
|
+
if total > 0:
|
|
206
|
+
failures = sum(1 for r in runs if not r.get("success", True))
|
|
207
|
+
self.set_failure_rate(framework, failures / total)
|
|
208
|
+
|
|
209
|
+
# MTTR: average latency of failed runs that eventually recovered
|
|
210
|
+
recovery_latencies = [
|
|
211
|
+
r.get("recovery_latency_ms", 0.0) / 1000.0
|
|
212
|
+
for r in runs
|
|
213
|
+
if not r.get("success", True) and r.get("recovered", False)
|
|
214
|
+
]
|
|
215
|
+
if recovery_latencies:
|
|
216
|
+
self.set_mttr(framework, sum(recovery_latencies) / len(recovery_latencies))
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
"""OpenTelemetry tracing integration for AgentProbe.
|
|
2
|
+
|
|
3
|
+
Sets up a tracer provider with:
|
|
4
|
+
- Console span exporter (for local dev / CI)
|
|
5
|
+
- OTLP gRPC exporter (for production / Jaeger / Tempo)
|
|
6
|
+
|
|
7
|
+
Usage::
|
|
8
|
+
|
|
9
|
+
from agentprobe.observer.tracer import AgentProbeTracer
|
|
10
|
+
|
|
11
|
+
tracer = AgentProbeTracer(
|
|
12
|
+
service_name="my-agent",
|
|
13
|
+
otlp_endpoint="http://localhost:4317", # optional
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
with tracer.span("my.operation", attributes={"key": "value"}) as span:
|
|
17
|
+
result = do_work()
|
|
18
|
+
span.set_attribute("result.count", len(result))
|
|
19
|
+
|
|
20
|
+
# Get a module-level tracer (for use inside library code)
|
|
21
|
+
from agentprobe.observer.tracer import get_tracer
|
|
22
|
+
_tracer = get_tracer(__name__)
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import os
|
|
28
|
+
from collections.abc import Generator
|
|
29
|
+
from contextlib import contextmanager
|
|
30
|
+
from typing import Any
|
|
31
|
+
|
|
32
|
+
from opentelemetry import trace
|
|
33
|
+
from opentelemetry.sdk.resources import SERVICE_NAME, Resource
|
|
34
|
+
from opentelemetry.sdk.trace import TracerProvider
|
|
35
|
+
from opentelemetry.sdk.trace.export import (
|
|
36
|
+
BatchSpanProcessor,
|
|
37
|
+
ConsoleSpanExporter,
|
|
38
|
+
SimpleSpanProcessor,
|
|
39
|
+
)
|
|
40
|
+
from opentelemetry.trace import Span, Tracer
|
|
41
|
+
|
|
42
|
+
# Optional OTLP exporter — gracefully unavailable if grpc not installed
|
|
43
|
+
try:
|
|
44
|
+
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
|
|
45
|
+
|
|
46
|
+
_OTLP_AVAILABLE = True
|
|
47
|
+
except ImportError:
|
|
48
|
+
_OTLP_AVAILABLE = False
|
|
49
|
+
|
|
50
|
+
_DEFAULT_SERVICE = "agentprobe"
|
|
51
|
+
_GLOBAL_PROVIDER: TracerProvider | None = None
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _build_provider(
|
|
55
|
+
service_name: str,
|
|
56
|
+
otlp_endpoint: str | None = None,
|
|
57
|
+
console: bool = True,
|
|
58
|
+
) -> TracerProvider:
|
|
59
|
+
resource = Resource(attributes={SERVICE_NAME: service_name})
|
|
60
|
+
provider = TracerProvider(resource=resource)
|
|
61
|
+
|
|
62
|
+
if console:
|
|
63
|
+
provider.add_span_processor(SimpleSpanProcessor(ConsoleSpanExporter()))
|
|
64
|
+
|
|
65
|
+
if otlp_endpoint and _OTLP_AVAILABLE:
|
|
66
|
+
otlp_exporter = OTLPSpanExporter(endpoint=otlp_endpoint, insecure=True)
|
|
67
|
+
provider.add_span_processor(BatchSpanProcessor(otlp_exporter))
|
|
68
|
+
elif otlp_endpoint and not _OTLP_AVAILABLE:
|
|
69
|
+
import warnings
|
|
70
|
+
|
|
71
|
+
warnings.warn(
|
|
72
|
+
"opentelemetry-exporter-otlp-proto-grpc is not installed. "
|
|
73
|
+
"OTLP export is disabled. Install with: "
|
|
74
|
+
"pip install opentelemetry-exporter-otlp-proto-grpc",
|
|
75
|
+
stacklevel=2,
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
return provider
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class AgentProbeTracer:
|
|
82
|
+
"""Configured OpenTelemetry tracer for AgentProbe experiments.
|
|
83
|
+
|
|
84
|
+
Registers as the global OTel tracer provider so that all
|
|
85
|
+
``get_tracer(__name__)`` calls within the library resolve to this provider.
|
|
86
|
+
|
|
87
|
+
Args:
|
|
88
|
+
service_name: Service name shown in trace UIs (Jaeger, Tempo, etc.)
|
|
89
|
+
otlp_endpoint: gRPC endpoint for OTLP export (e.g. "http://localhost:4317").
|
|
90
|
+
If None, only console export is active.
|
|
91
|
+
console: Whether to print spans to stdout (useful for local dev).
|
|
92
|
+
"""
|
|
93
|
+
|
|
94
|
+
def __init__(
|
|
95
|
+
self,
|
|
96
|
+
service_name: str = _DEFAULT_SERVICE,
|
|
97
|
+
otlp_endpoint: str | None = None,
|
|
98
|
+
console: bool = True,
|
|
99
|
+
) -> None:
|
|
100
|
+
# Allow env-var override
|
|
101
|
+
endpoint = otlp_endpoint or os.getenv("AGENTPROBE_OTLP_ENDPOINT")
|
|
102
|
+
console_on = console and os.getenv("AGENTPROBE_NO_CONSOLE_TRACE") != "1"
|
|
103
|
+
|
|
104
|
+
self._provider = _build_provider(service_name, endpoint, console_on)
|
|
105
|
+
trace.set_tracer_provider(self._provider)
|
|
106
|
+
self._tracer: Tracer = self._provider.get_tracer(service_name)
|
|
107
|
+
self.service_name = service_name
|
|
108
|
+
|
|
109
|
+
@contextmanager
|
|
110
|
+
def span(
|
|
111
|
+
self,
|
|
112
|
+
name: str,
|
|
113
|
+
attributes: dict[str, Any] | None = None,
|
|
114
|
+
) -> Generator[Span, None, None]:
|
|
115
|
+
"""Context manager: start a new span with optional attributes."""
|
|
116
|
+
with self._tracer.start_as_current_span(name) as span:
|
|
117
|
+
if attributes:
|
|
118
|
+
for k, v in attributes.items():
|
|
119
|
+
span.set_attribute(k, v)
|
|
120
|
+
yield span
|
|
121
|
+
|
|
122
|
+
def get_tracer(self, name: str) -> Tracer:
|
|
123
|
+
return self._provider.get_tracer(name)
|
|
124
|
+
|
|
125
|
+
def shutdown(self) -> None:
|
|
126
|
+
"""Flush and shut down the span processor (call at end of experiments)."""
|
|
127
|
+
self._provider.shutdown()
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def get_tracer(name: str) -> Tracer:
|
|
131
|
+
"""Return a module-level tracer.
|
|
132
|
+
|
|
133
|
+
Uses the globally registered OTel provider if :class:`AgentProbeTracer`
|
|
134
|
+
has been initialized; otherwise uses a no-op tracer (safe for unit tests
|
|
135
|
+
that don't need trace output).
|
|
136
|
+
"""
|
|
137
|
+
return trace.get_tracer(name)
|