agentloop-profiler 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of agentloop-profiler might be problematic. Click here for more details.
- agentloop/__init__.py +68 -0
- agentloop/audit.py +34 -0
- agentloop/autoinstrument.py +102 -0
- agentloop/ci.py +135 -0
- agentloop/cli.py +876 -0
- agentloop/client.py +170 -0
- agentloop/config.py +50 -0
- agentloop/costs.py +15 -0
- agentloop/decorators.py +160 -0
- agentloop/demo.py +283 -0
- agentloop/doctor.py +344 -0
- agentloop/events.py +49 -0
- agentloop/exporters.py +60 -0
- agentloop/findings.py +215 -0
- agentloop/graph.py +312 -0
- agentloop/integrations/__init__.py +1 -0
- agentloop/integrations/crewai.py +144 -0
- agentloop/integrations/langgraph.py +140 -0
- agentloop/integrations/openai.py +369 -0
- agentloop/integrations/openai_agents.py +212 -0
- agentloop/integrations/sdk.py +82 -0
- agentloop/integrations/vercel_ai.py +83 -0
- agentloop/issues.py +88 -0
- agentloop/markdown.py +102 -0
- agentloop/metrics.py +123 -0
- agentloop/optimizer.py +323 -0
- agentloop/otel.py +302 -0
- agentloop/otel_ids.py +81 -0
- agentloop/patches.py +583 -0
- agentloop/plan_export.py +72 -0
- agentloop/quality.py +222 -0
- agentloop/replay.py +424 -0
- agentloop/runtime.py +244 -0
- agentloop/savings.py +135 -0
- agentloop/server.py +292 -0
- agentloop/store.py +840 -0
- agentloop/timing.py +124 -0
- agentloop/tracer.py +356 -0
- agentloop/value.py +243 -0
- agentloop/version.py +3 -0
- agentloop_profiler-0.4.0.dist-info/METADATA +598 -0
- agentloop_profiler-0.4.0.dist-info/RECORD +50 -0
- agentloop_profiler-0.4.0.dist-info/WHEEL +5 -0
- agentloop_profiler-0.4.0.dist-info/entry_points.txt +2 -0
- agentloop_profiler-0.4.0.dist-info/licenses/LICENSE +201 -0
- agentloop_profiler-0.4.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +54 -0
- agentloop_profiler-0.4.0.dist-info/top_level.txt +2 -0
- dashboard/__init__.py +1 -0
- dashboard/app.py +698 -0
- dashboard/value_view.py +101 -0
agentloop/optimizer.py
ADDED
|
@@ -0,0 +1,323 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from enum import Enum
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from agentloop.graph import ExecutionGraph
|
|
8
|
+
from agentloop.savings import SavingsItem, select_compatible
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class RecommendationType(str, Enum):
|
|
12
|
+
PARALLELIZE_TOOLS = "parallelize_tools"
|
|
13
|
+
CACHE_CONTEXT = "cache_context"
|
|
14
|
+
BATCH_MODEL_CALLS = "batch_model_calls"
|
|
15
|
+
ROUTE_TO_SMALLER_MODEL = "route_to_smaller_model"
|
|
16
|
+
REMOVE_RETRY_LOOP = "remove_retry_loop"
|
|
17
|
+
ADD_SCHEMA_VALIDATION = "add_schema_validation"
|
|
18
|
+
SPLIT_LARGE_STEP = "split_large_step"
|
|
19
|
+
RUNAWAY_LOOP = "runaway_loop"
|
|
20
|
+
TOOL_OSCILLATION = "tool_oscillation"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass
|
|
24
|
+
class OptimizationCard:
|
|
25
|
+
type: RecommendationType
|
|
26
|
+
title: str
|
|
27
|
+
why: str
|
|
28
|
+
rewrite_hint: str
|
|
29
|
+
confidence: str
|
|
30
|
+
estimated_latency_savings_ms: float = 0.0
|
|
31
|
+
estimated_cost_savings_usd: float = 0.0
|
|
32
|
+
affected_nodes: list[str] | None = None
|
|
33
|
+
|
|
34
|
+
def to_dict(self) -> dict[str, Any]:
|
|
35
|
+
return {
|
|
36
|
+
"type": self.type.value,
|
|
37
|
+
"title": self.title,
|
|
38
|
+
"why": self.why,
|
|
39
|
+
"rewrite_hint": self.rewrite_hint,
|
|
40
|
+
"confidence": self.confidence,
|
|
41
|
+
"estimated_latency_savings_ms": round(self.estimated_latency_savings_ms, 3),
|
|
42
|
+
"estimated_cost_savings_usd": round(self.estimated_cost_savings_usd, 6),
|
|
43
|
+
"affected_nodes": self.affected_nodes or [],
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def build_optimization_plan(trace: Any, report: dict[str, Any] | None = None) -> dict[str, Any]:
|
|
48
|
+
report = report or trace.report()
|
|
49
|
+
graph = ExecutionGraph.from_trace(trace)
|
|
50
|
+
cards: list[OptimizationCard] = []
|
|
51
|
+
cards.extend(_parallelization_cards(graph))
|
|
52
|
+
cards.extend(_context_cache_cards(report, graph))
|
|
53
|
+
cards.extend(_retry_cards(graph))
|
|
54
|
+
cards.extend(_batch_model_cards(graph))
|
|
55
|
+
cards.extend(_routing_cards(graph))
|
|
56
|
+
cards.extend(_split_large_step_cards(graph))
|
|
57
|
+
cards.extend(_runaway_loop_cards(graph))
|
|
58
|
+
cards.extend(_tool_oscillation_cards(graph))
|
|
59
|
+
|
|
60
|
+
current_runtime = report.get("total_runtime_ms", graph.total_runtime_ms())
|
|
61
|
+
current_cost = report.get("estimated_cost_usd", 0.0)
|
|
62
|
+
aggregate = _aggregate_savings(cards, current_runtime, current_cost)
|
|
63
|
+
total_latency_savings = aggregate["latency_savings_ms"]
|
|
64
|
+
total_cost_savings = aggregate["cost_savings_usd"]
|
|
65
|
+
|
|
66
|
+
return {
|
|
67
|
+
"run_id": trace.run_id,
|
|
68
|
+
"name": trace.name,
|
|
69
|
+
"current": {
|
|
70
|
+
"runtime_ms": current_runtime,
|
|
71
|
+
"estimated_cost_usd": current_cost,
|
|
72
|
+
"input_tokens": report.get("input_tokens", 0),
|
|
73
|
+
"output_tokens": report.get("output_tokens", 0),
|
|
74
|
+
"retry_count": report.get("retry_count", 0),
|
|
75
|
+
"repeated_context_ratio": report.get("repeated_context_ratio", 0.0),
|
|
76
|
+
},
|
|
77
|
+
"estimated_after": {
|
|
78
|
+
"runtime_ms": round(max(0.0, current_runtime - total_latency_savings), 3),
|
|
79
|
+
"estimated_cost_usd": round(max(0.0, current_cost - total_cost_savings), 6),
|
|
80
|
+
"latency_reduction_pct": round((total_latency_savings / current_runtime) * 100, 2)
|
|
81
|
+
if current_runtime
|
|
82
|
+
else 0.0,
|
|
83
|
+
"cost_reduction_pct": round((total_cost_savings / current_cost) * 100, 2)
|
|
84
|
+
if current_cost
|
|
85
|
+
else 0.0,
|
|
86
|
+
},
|
|
87
|
+
"savings_aggregation": aggregate["explanation"],
|
|
88
|
+
"graph": graph.to_dict(),
|
|
89
|
+
"optimization_cards": [card.to_dict() for card in cards],
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _aggregate_savings(
|
|
94
|
+
cards: list[OptimizationCard], current_runtime: float, current_cost: float
|
|
95
|
+
) -> dict[str, Any]:
|
|
96
|
+
"""Combine per-card savings without double-counting overlapping spans.
|
|
97
|
+
|
|
98
|
+
Cards that share any affected span compete for the same work, so their
|
|
99
|
+
estimates are mutually exclusive alternatives, not additive. The plan
|
|
100
|
+
totals come from the compatible (span-disjoint) subset of cards that
|
|
101
|
+
maximizes latency savings, breaking ties by cost savings — so the reported
|
|
102
|
+
latency and cost pair is achievable by one concrete set of changes, never a
|
|
103
|
+
mix of incompatible alternatives. The totals are finally capped at the
|
|
104
|
+
current runtime/cost, because no plan can save more wall-clock time (or
|
|
105
|
+
money) than the run actually spent. This keeps ``latency_reduction_pct`` at
|
|
106
|
+
or below 100% and internally consistent with ``runtime_ms``, while per-card
|
|
107
|
+
estimates stay untouched and explainable.
|
|
108
|
+
"""
|
|
109
|
+
items = [
|
|
110
|
+
SavingsItem(
|
|
111
|
+
spans=frozenset(card.affected_nodes or []),
|
|
112
|
+
latency_ms=float(card.estimated_latency_savings_ms or 0.0),
|
|
113
|
+
cost_usd=float(card.estimated_cost_savings_usd or 0.0),
|
|
114
|
+
)
|
|
115
|
+
for card in cards
|
|
116
|
+
]
|
|
117
|
+
selection = select_compatible(items)
|
|
118
|
+
capped_latency = min(selection.latency_ms, current_runtime) if current_runtime else 0.0
|
|
119
|
+
capped_cost = min(selection.cost_usd, current_cost) if current_cost else 0.0
|
|
120
|
+
raw_latency = sum(card.estimated_latency_savings_ms for card in cards)
|
|
121
|
+
raw_cost = sum(card.estimated_cost_savings_usd for card in cards)
|
|
122
|
+
return {
|
|
123
|
+
"latency_savings_ms": capped_latency,
|
|
124
|
+
"cost_savings_usd": capped_cost,
|
|
125
|
+
"explanation": {
|
|
126
|
+
"rule": (
|
|
127
|
+
"cards sharing affected spans are mutually exclusive alternatives; "
|
|
128
|
+
"totals come from the compatible (span-disjoint) subset of cards "
|
|
129
|
+
"that maximizes latency savings, breaking ties by cost savings, "
|
|
130
|
+
"capped at current runtime/cost"
|
|
131
|
+
),
|
|
132
|
+
"card_count": len(cards),
|
|
133
|
+
"selected_card_indexes": list(selection.indices),
|
|
134
|
+
"raw_latency_savings_ms": round(raw_latency, 3),
|
|
135
|
+
"effective_latency_savings_ms": round(capped_latency, 3),
|
|
136
|
+
"raw_cost_savings_usd": round(raw_cost, 6),
|
|
137
|
+
"effective_cost_savings_usd": round(capped_cost, 6),
|
|
138
|
+
},
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _parallelization_cards(graph: ExecutionGraph) -> list[OptimizationCard]:
|
|
143
|
+
cards = []
|
|
144
|
+
for group in graph.parallelizable_groups():
|
|
145
|
+
cards.append(
|
|
146
|
+
OptimizationCard(
|
|
147
|
+
type=RecommendationType.PARALLELIZE_TOOLS,
|
|
148
|
+
title=f"Parallelize repeated `{group['name']}` tool calls",
|
|
149
|
+
why=f"{group['count']} `{group['name']}` calls appear serial and independent.",
|
|
150
|
+
rewrite_hint="Use asyncio.gather or ThreadPoolExecutor around independent tool calls.",
|
|
151
|
+
confidence="medium",
|
|
152
|
+
estimated_latency_savings_ms=group["estimated_savings_ms"],
|
|
153
|
+
affected_nodes=group["node_ids"],
|
|
154
|
+
)
|
|
155
|
+
)
|
|
156
|
+
return cards
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _context_cache_cards(report: dict[str, Any], graph: ExecutionGraph) -> list[OptimizationCard]:
|
|
160
|
+
ratio = report.get("repeated_context_ratio", 0.0)
|
|
161
|
+
if ratio < 0.10:
|
|
162
|
+
return []
|
|
163
|
+
model_nodes = [node.node_id for node in graph.nodes if node.event_type == "model_call"]
|
|
164
|
+
current_cost = report.get("estimated_cost_usd", 0.0)
|
|
165
|
+
return [
|
|
166
|
+
OptimizationCard(
|
|
167
|
+
type=RecommendationType.CACHE_CONTEXT,
|
|
168
|
+
title="Cache repeated prompt/context prefix",
|
|
169
|
+
why=f"Repeated context ratio is {ratio:.1%}, which suggests stable instructions or source text are being resent.",
|
|
170
|
+
rewrite_hint="Move stable instructions into cached prefixes, summaries, or framework-level memory instead of resending full text.",
|
|
171
|
+
confidence="high" if ratio >= 0.20 else "medium",
|
|
172
|
+
estimated_cost_savings_usd=current_cost * min(0.5, ratio),
|
|
173
|
+
affected_nodes=model_nodes,
|
|
174
|
+
)
|
|
175
|
+
]
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _retry_cards(graph: ExecutionGraph) -> list[OptimizationCard]:
|
|
179
|
+
retry_nodes = [node for node in graph.nodes if node.event_type == "retry"]
|
|
180
|
+
if not retry_nodes:
|
|
181
|
+
return []
|
|
182
|
+
retry_time = sum(node.duration_ms for node in retry_nodes)
|
|
183
|
+
return [
|
|
184
|
+
OptimizationCard(
|
|
185
|
+
type=RecommendationType.ADD_SCHEMA_VALIDATION,
|
|
186
|
+
title="Reduce retry loop with structured outputs",
|
|
187
|
+
why=f"Observed {len(retry_nodes)} retry span(s), costing {retry_time / 1000:.2f}s.",
|
|
188
|
+
rewrite_hint="Add schema validation, JSON mode, constrained decoding, or a cheap repair prompt before rerunning a full step.",
|
|
189
|
+
confidence="high",
|
|
190
|
+
estimated_latency_savings_ms=retry_time * 0.8,
|
|
191
|
+
affected_nodes=[node.node_id for node in retry_nodes],
|
|
192
|
+
)
|
|
193
|
+
]
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def _batch_model_cards(graph: ExecutionGraph) -> list[OptimizationCard]:
|
|
197
|
+
groups: dict[str, list[Any]] = {}
|
|
198
|
+
for node in graph.nodes:
|
|
199
|
+
if node.event_type == "model_call":
|
|
200
|
+
groups.setdefault(node.name, []).append(node)
|
|
201
|
+
cards = []
|
|
202
|
+
for name, nodes in groups.items():
|
|
203
|
+
if len(nodes) < 3:
|
|
204
|
+
continue
|
|
205
|
+
duration = sum(node.duration_ms for node in nodes)
|
|
206
|
+
cards.append(
|
|
207
|
+
OptimizationCard(
|
|
208
|
+
type=RecommendationType.BATCH_MODEL_CALLS,
|
|
209
|
+
title=f"Batch repeated `{name}` model calls",
|
|
210
|
+
why=f"{len(nodes)} `{name}` calls have the same role and may be batchable.",
|
|
211
|
+
rewrite_hint="Batch documents/items into one prompt or use map-reduce only when outputs truly need independent reasoning.",
|
|
212
|
+
confidence="medium",
|
|
213
|
+
estimated_latency_savings_ms=duration * 0.35,
|
|
214
|
+
affected_nodes=[node.node_id for node in nodes],
|
|
215
|
+
)
|
|
216
|
+
)
|
|
217
|
+
return cards
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def _routing_cards(graph: ExecutionGraph) -> list[OptimizationCard]:
|
|
221
|
+
cards = []
|
|
222
|
+
for node in graph.nodes:
|
|
223
|
+
if node.event_type != "model_call" or not node.model:
|
|
224
|
+
continue
|
|
225
|
+
if "mini" in node.model.lower():
|
|
226
|
+
continue
|
|
227
|
+
if node.output_tokens <= 400 and node.input_tokens <= 2500:
|
|
228
|
+
cards.append(
|
|
229
|
+
OptimizationCard(
|
|
230
|
+
type=RecommendationType.ROUTE_TO_SMALLER_MODEL,
|
|
231
|
+
title=f"Route `{node.name}` to a cheaper model",
|
|
232
|
+
why=f"`{node.name}` is a relatively small model step using {node.total_tokens} tokens on {node.model}.",
|
|
233
|
+
rewrite_hint="Try a smaller model for planning, summarization, extraction, or verification steps and keep the larger model for final synthesis.",
|
|
234
|
+
confidence="low",
|
|
235
|
+
estimated_latency_savings_ms=node.duration_ms * 0.25,
|
|
236
|
+
affected_nodes=[node.node_id],
|
|
237
|
+
)
|
|
238
|
+
)
|
|
239
|
+
return cards[:3]
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _split_large_step_cards(graph: ExecutionGraph) -> list[OptimizationCard]:
|
|
243
|
+
cards = []
|
|
244
|
+
for node in graph.nodes:
|
|
245
|
+
if node.event_type == "model_call" and node.total_tokens >= 4000:
|
|
246
|
+
cards.append(
|
|
247
|
+
OptimizationCard(
|
|
248
|
+
type=RecommendationType.SPLIT_LARGE_STEP,
|
|
249
|
+
title=f"Split or compress `{node.name}`",
|
|
250
|
+
why=f"`{node.name}` used {node.total_tokens} tokens, making it a large and fragile step.",
|
|
251
|
+
rewrite_hint="Split into retrieve-filter-summarize or compress the context before the final reasoning call.",
|
|
252
|
+
confidence="medium",
|
|
253
|
+
estimated_latency_savings_ms=node.duration_ms * 0.20,
|
|
254
|
+
affected_nodes=[node.node_id],
|
|
255
|
+
)
|
|
256
|
+
)
|
|
257
|
+
return cards
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _runaway_loop_cards(graph: ExecutionGraph) -> list[OptimizationCard]:
|
|
261
|
+
cards = []
|
|
262
|
+
groups: dict[tuple[str, str], list[Any]] = {}
|
|
263
|
+
for node in graph.nodes:
|
|
264
|
+
groups.setdefault((node.event_type, node.name), []).append(node)
|
|
265
|
+
|
|
266
|
+
for (event_type, name), nodes in groups.items():
|
|
267
|
+
if len(nodes) < 8:
|
|
268
|
+
continue
|
|
269
|
+
duration = sum(node.duration_ms for node in nodes)
|
|
270
|
+
cards.append(
|
|
271
|
+
OptimizationCard(
|
|
272
|
+
type=RecommendationType.RUNAWAY_LOOP,
|
|
273
|
+
title=f"Add a loop guardrail around `{name}`",
|
|
274
|
+
why=f"`{name}` ran {len(nodes)} times in one trace, which suggests an unbounded or weakly bounded agent loop.",
|
|
275
|
+
rewrite_hint="Add max-iteration, max-cost, and unchanged-state guards before the workflow can keep looping.",
|
|
276
|
+
confidence="high" if len(nodes) >= 12 else "medium",
|
|
277
|
+
estimated_latency_savings_ms=duration * 0.30,
|
|
278
|
+
affected_nodes=[node.node_id for node in nodes],
|
|
279
|
+
)
|
|
280
|
+
)
|
|
281
|
+
return cards[:3]
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def _tool_oscillation_cards(graph: ExecutionGraph) -> list[OptimizationCard]:
|
|
285
|
+
tool_nodes = [node for node in graph.nodes if node.event_type == "tool_call"]
|
|
286
|
+
if len(tool_nodes) < 4:
|
|
287
|
+
return []
|
|
288
|
+
|
|
289
|
+
cards = []
|
|
290
|
+
index = 0
|
|
291
|
+
while index <= len(tool_nodes) - 4:
|
|
292
|
+
first = tool_nodes[index].name
|
|
293
|
+
second = tool_nodes[index + 1].name
|
|
294
|
+
if first == second:
|
|
295
|
+
index += 1
|
|
296
|
+
continue
|
|
297
|
+
|
|
298
|
+
window = tool_nodes[index : index + 4]
|
|
299
|
+
if [node.name for node in window] != [first, second, first, second]:
|
|
300
|
+
index += 1
|
|
301
|
+
continue
|
|
302
|
+
|
|
303
|
+
end = index + 4
|
|
304
|
+
while end < len(tool_nodes) and tool_nodes[end].name == (
|
|
305
|
+
first if (end - index) % 2 == 0 else second
|
|
306
|
+
):
|
|
307
|
+
end += 1
|
|
308
|
+
oscillating = tool_nodes[index:end]
|
|
309
|
+
duration = sum(node.duration_ms for node in oscillating)
|
|
310
|
+
cards.append(
|
|
311
|
+
OptimizationCard(
|
|
312
|
+
type=RecommendationType.TOOL_OSCILLATION,
|
|
313
|
+
title=f"Stop `{first}`/`{second}` tool oscillation",
|
|
314
|
+
why=f"Observed {len(oscillating)} alternating `{first}` and `{second}` tool calls.",
|
|
315
|
+
rewrite_hint="Add a state-change check or decision memo so the agent does not repeat equivalent tool transitions.",
|
|
316
|
+
confidence="medium",
|
|
317
|
+
estimated_latency_savings_ms=duration * 0.40,
|
|
318
|
+
affected_nodes=[node.node_id for node in oscillating],
|
|
319
|
+
)
|
|
320
|
+
)
|
|
321
|
+
index = end
|
|
322
|
+
|
|
323
|
+
return sorted(cards, key=lambda card: card.estimated_latency_savings_ms, reverse=True)[:3]
|
agentloop/otel.py
ADDED
|
@@ -0,0 +1,302 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from datetime import datetime, timezone
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from agentloop.events import AgentEvent
|
|
7
|
+
from agentloop.otel_ids import to_span_id, to_trace_id
|
|
8
|
+
from agentloop.tracer import AgentTrace
|
|
9
|
+
from agentloop.version import __version__
|
|
10
|
+
|
|
11
|
+
_NANOSECONDS_PER_MILLISECOND = 1_000_000
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def trace_from_otel(
|
|
15
|
+
payload: dict[str, Any] | list[dict[str, Any]], name: str | None = None
|
|
16
|
+
) -> AgentTrace:
|
|
17
|
+
"""Convert OTLP/GenAI-style JSON spans into an AgentLoop trace.
|
|
18
|
+
|
|
19
|
+
This accepts the standard OTLP JSON shape (`resourceSpans -> scopeSpans -> spans`),
|
|
20
|
+
a dictionary with a top-level `spans` list, or a raw list of span dictionaries.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
spans = list(_iter_spans(payload))
|
|
24
|
+
run_id = _run_id_from_spans(spans)
|
|
25
|
+
trace = AgentTrace(name=name or _trace_name(spans), run_id=run_id, metadata={"source": "otel"})
|
|
26
|
+
for span in spans:
|
|
27
|
+
trace.add_event(_event_from_span(span, run_id))
|
|
28
|
+
bounds = _trace_bounds_ns(spans)
|
|
29
|
+
if bounds is not None:
|
|
30
|
+
started_ns, ended_ns = bounds
|
|
31
|
+
trace.started_at = _iso_from_ns(started_ns)
|
|
32
|
+
trace.ended_at = _iso_from_ns(ended_ns)
|
|
33
|
+
trace.elapsed_ms = (ended_ns - started_ns) / _NANOSECONDS_PER_MILLISECOND
|
|
34
|
+
return trace
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def trace_to_otel(trace: AgentTrace) -> dict[str, Any]:
|
|
38
|
+
"""Export an AgentLoop trace as dependency-free OTLP-like GenAI JSON."""
|
|
39
|
+
|
|
40
|
+
spans = [_span_from_event(trace, event) for event in trace.events]
|
|
41
|
+
return {
|
|
42
|
+
"resourceSpans": [
|
|
43
|
+
{
|
|
44
|
+
"resource": {
|
|
45
|
+
"attributes": [
|
|
46
|
+
_attribute("service.name", "agentloop"),
|
|
47
|
+
_attribute("agentloop.trace.name", trace.name),
|
|
48
|
+
_attribute("agentloop.run_id", trace.run_id),
|
|
49
|
+
]
|
|
50
|
+
},
|
|
51
|
+
"scopeSpans": [
|
|
52
|
+
{
|
|
53
|
+
"scope": {"name": "agentloop", "version": __version__},
|
|
54
|
+
"spans": spans,
|
|
55
|
+
}
|
|
56
|
+
],
|
|
57
|
+
}
|
|
58
|
+
]
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _iter_spans(payload: dict[str, Any] | list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
63
|
+
if isinstance(payload, list):
|
|
64
|
+
return payload
|
|
65
|
+
if "spans" in payload and isinstance(payload["spans"], list):
|
|
66
|
+
return payload["spans"]
|
|
67
|
+
|
|
68
|
+
spans: list[dict[str, Any]] = []
|
|
69
|
+
for resource_span in payload.get("resourceSpans", []):
|
|
70
|
+
for scope_span in resource_span.get("scopeSpans", []):
|
|
71
|
+
spans.extend(scope_span.get("spans", []))
|
|
72
|
+
return spans
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _run_id_from_spans(spans: list[dict[str, Any]]) -> str:
|
|
76
|
+
for span in spans:
|
|
77
|
+
trace_id = str(span.get("traceId") or span.get("trace_id") or "")
|
|
78
|
+
if trace_id:
|
|
79
|
+
# Preserve the full trace id (not just the last 16 chars) so a valid
|
|
80
|
+
# OTLP id round-trips back out unchanged via to_trace_id().
|
|
81
|
+
return "run_" + trace_id.lower()
|
|
82
|
+
return "run_otel_import"
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _trace_name(spans: list[dict[str, Any]]) -> str:
|
|
86
|
+
for span in spans:
|
|
87
|
+
attrs = _attributes(span)
|
|
88
|
+
workflow = attrs.get("gen_ai.workflow.name") or attrs.get("agentloop.trace.name")
|
|
89
|
+
if workflow:
|
|
90
|
+
return str(workflow)
|
|
91
|
+
return "otel_trace"
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _event_from_span(span: dict[str, Any], run_id: str) -> AgentEvent:
|
|
95
|
+
attrs = _attributes(span)
|
|
96
|
+
operation = str(attrs.get("gen_ai.operation.name") or attrs.get("agentloop.event_type") or "")
|
|
97
|
+
event_type = _event_type(operation, span.get("name"))
|
|
98
|
+
started_ns = _int_or_none(span.get("startTimeUnixNano") or span.get("start_time_unix_nano"))
|
|
99
|
+
ended_ns = _int_or_none(span.get("endTimeUnixNano") or span.get("end_time_unix_nano"))
|
|
100
|
+
duration_ms = _duration_ms(started_ns, ended_ns, span)
|
|
101
|
+
span_id = str(span.get("spanId") or span.get("span_id") or "")
|
|
102
|
+
parent_id = str(span.get("parentSpanId") or span.get("parent_span_id") or "") or None
|
|
103
|
+
status = _status(span)
|
|
104
|
+
error = _error(span)
|
|
105
|
+
|
|
106
|
+
metadata = {
|
|
107
|
+
key: value
|
|
108
|
+
for key, value in attrs.items()
|
|
109
|
+
if not key.startswith("gen_ai.usage.") and key not in _direct_attribute_keys()
|
|
110
|
+
}
|
|
111
|
+
metadata["otel_span_id"] = span_id
|
|
112
|
+
metadata["otel_trace_id"] = span.get("traceId") or span.get("trace_id")
|
|
113
|
+
|
|
114
|
+
return AgentEvent(
|
|
115
|
+
event_id="span_" + span_id if span_id else "",
|
|
116
|
+
run_id=run_id,
|
|
117
|
+
event_type=event_type,
|
|
118
|
+
name=str(
|
|
119
|
+
attrs.get("gen_ai.tool.name")
|
|
120
|
+
or attrs.get("agentloop.name")
|
|
121
|
+
or span.get("name")
|
|
122
|
+
or event_type
|
|
123
|
+
),
|
|
124
|
+
started_at=_iso_from_ns(started_ns),
|
|
125
|
+
ended_at=_iso_from_ns(ended_ns),
|
|
126
|
+
duration_ms=duration_ms,
|
|
127
|
+
parent_id="span_" + parent_id if parent_id else None,
|
|
128
|
+
model=_first_string(attrs, "gen_ai.request.model", "gen_ai.response.model"),
|
|
129
|
+
input_tokens=int(
|
|
130
|
+
attrs.get("gen_ai.usage.input_tokens") or attrs.get("llm.usage.prompt_tokens") or 0
|
|
131
|
+
),
|
|
132
|
+
output_tokens=int(
|
|
133
|
+
attrs.get("gen_ai.usage.output_tokens") or attrs.get("llm.usage.completion_tokens") or 0
|
|
134
|
+
),
|
|
135
|
+
status=status,
|
|
136
|
+
error=error,
|
|
137
|
+
metadata=metadata,
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _span_from_event(trace: AgentTrace, event: AgentEvent) -> dict[str, Any]:
|
|
142
|
+
start_ns = _ns_from_iso(event.started_at)
|
|
143
|
+
end_ns = _ns_from_iso(event.ended_at)
|
|
144
|
+
attrs = [
|
|
145
|
+
_attribute("gen_ai.operation.name", _operation_name(event.event_type)),
|
|
146
|
+
_attribute("agentloop.event_type", event.event_type),
|
|
147
|
+
_attribute("agentloop.name", event.name),
|
|
148
|
+
_attribute("agentloop.run_id", trace.run_id),
|
|
149
|
+
_attribute("gen_ai.usage.input_tokens", event.input_tokens),
|
|
150
|
+
_attribute("gen_ai.usage.output_tokens", event.output_tokens),
|
|
151
|
+
]
|
|
152
|
+
if event.model:
|
|
153
|
+
attrs.append(_attribute("gen_ai.request.model", event.model))
|
|
154
|
+
for key, value in sorted((event.metadata or {}).items()):
|
|
155
|
+
if isinstance(value, str | int | float | bool):
|
|
156
|
+
attrs.append(_attribute(f"agentloop.metadata.{key}", value))
|
|
157
|
+
|
|
158
|
+
# Keep the original native ids in attributes so a remapped (non-hex/custom)
|
|
159
|
+
# id stays diagnosable; the run id is already carried as agentloop.run_id.
|
|
160
|
+
attrs.append(_attribute("agentloop.native_event_id", event.event_id))
|
|
161
|
+
if event.parent_id:
|
|
162
|
+
attrs.append(_attribute("agentloop.native_parent_id", event.parent_id))
|
|
163
|
+
|
|
164
|
+
span = {
|
|
165
|
+
"traceId": to_trace_id(trace.run_id),
|
|
166
|
+
"spanId": to_span_id(event.event_id),
|
|
167
|
+
"name": event.name,
|
|
168
|
+
"kind": "SPAN_KIND_INTERNAL",
|
|
169
|
+
"startTimeUnixNano": str(start_ns),
|
|
170
|
+
"endTimeUnixNano": str(end_ns),
|
|
171
|
+
"attributes": attrs,
|
|
172
|
+
"status": {"code": "STATUS_CODE_OK" if event.status == "ok" else "STATUS_CODE_ERROR"},
|
|
173
|
+
}
|
|
174
|
+
if event.parent_id:
|
|
175
|
+
span["parentSpanId"] = to_span_id(event.parent_id)
|
|
176
|
+
return span
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _event_type(operation: str, name: Any) -> str:
|
|
180
|
+
op = operation.lower()
|
|
181
|
+
span_name = str(name or "").lower()
|
|
182
|
+
if op in {"chat", "text_completion", "embeddings", "generate_content"}:
|
|
183
|
+
return "model_call"
|
|
184
|
+
if op in {"execute_tool"} or "tool" in span_name:
|
|
185
|
+
return "tool_call"
|
|
186
|
+
if op == "retry" or "retry" in span_name:
|
|
187
|
+
return "retry"
|
|
188
|
+
return "tool_call" if op in {"invoke_agent", "invoke_workflow"} else "model_call"
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _operation_name(event_type: str) -> str:
|
|
192
|
+
if event_type == "model_call":
|
|
193
|
+
return "chat"
|
|
194
|
+
if event_type == "tool_call":
|
|
195
|
+
return "execute_tool"
|
|
196
|
+
return event_type
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _attributes(span: dict[str, Any]) -> dict[str, Any]:
|
|
200
|
+
raw = span.get("attributes", {})
|
|
201
|
+
if isinstance(raw, dict):
|
|
202
|
+
return raw.copy()
|
|
203
|
+
attrs: dict[str, Any] = {}
|
|
204
|
+
for item in raw or []:
|
|
205
|
+
key = item.get("key")
|
|
206
|
+
if key:
|
|
207
|
+
attrs[key] = _attribute_value(item.get("value", {}))
|
|
208
|
+
return attrs
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _attribute_value(value: dict[str, Any]) -> Any:
|
|
212
|
+
if not isinstance(value, dict):
|
|
213
|
+
return value
|
|
214
|
+
for key in ("stringValue", "intValue", "doubleValue", "boolValue"):
|
|
215
|
+
if key in value:
|
|
216
|
+
return value[key]
|
|
217
|
+
if "arrayValue" in value:
|
|
218
|
+
values = value.get("arrayValue", {}).get("values", [])
|
|
219
|
+
return [_attribute_value(item) for item in values]
|
|
220
|
+
return None
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _attribute(key: str, value: Any) -> dict[str, Any]:
|
|
224
|
+
if isinstance(value, bool):
|
|
225
|
+
encoded = {"boolValue": value}
|
|
226
|
+
elif isinstance(value, int):
|
|
227
|
+
encoded = {"intValue": value}
|
|
228
|
+
elif isinstance(value, float):
|
|
229
|
+
encoded = {"doubleValue": value}
|
|
230
|
+
else:
|
|
231
|
+
encoded = {"stringValue": str(value)}
|
|
232
|
+
return {"key": key, "value": encoded}
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def _duration_ms(started_ns: int | None, ended_ns: int | None, span: dict[str, Any]) -> float:
|
|
236
|
+
if started_ns is not None and ended_ns is not None and ended_ns >= started_ns:
|
|
237
|
+
return (ended_ns - started_ns) / _NANOSECONDS_PER_MILLISECOND
|
|
238
|
+
return float(span.get("duration_ms") or span.get("durationMs") or 0.0)
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def _trace_bounds_ns(spans: list[dict[str, Any]]) -> tuple[int, int] | None:
|
|
242
|
+
if not spans:
|
|
243
|
+
return None
|
|
244
|
+
bounds: list[tuple[int, int]] = []
|
|
245
|
+
for span in spans:
|
|
246
|
+
started_ns = _int_or_none(span.get("startTimeUnixNano") or span.get("start_time_unix_nano"))
|
|
247
|
+
ended_ns = _int_or_none(span.get("endTimeUnixNano") or span.get("end_time_unix_nano"))
|
|
248
|
+
if started_ns is None or ended_ns is None or ended_ns < started_ns:
|
|
249
|
+
return None
|
|
250
|
+
bounds.append((started_ns, ended_ns))
|
|
251
|
+
return min(started for started, _ in bounds), max(ended for _, ended in bounds)
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def _iso_from_ns(value: int | None) -> str:
|
|
255
|
+
if value is None:
|
|
256
|
+
return datetime.now(timezone.utc).isoformat()
|
|
257
|
+
return datetime.fromtimestamp(value / 1_000_000_000, tz=timezone.utc).isoformat()
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _ns_from_iso(value: str) -> int:
|
|
261
|
+
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
262
|
+
return int(parsed.timestamp() * 1_000_000_000)
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def _int_or_none(value: Any) -> int | None:
|
|
266
|
+
if value is None or value == "":
|
|
267
|
+
return None
|
|
268
|
+
return int(value)
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def _first_string(attrs: dict[str, Any], *keys: str) -> str | None:
|
|
272
|
+
for key in keys:
|
|
273
|
+
value = attrs.get(key)
|
|
274
|
+
if value:
|
|
275
|
+
return str(value)
|
|
276
|
+
return None
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def _status(span: dict[str, Any]) -> str:
|
|
280
|
+
status = span.get("status", {})
|
|
281
|
+
if isinstance(status, dict) and str(status.get("code", "")).endswith("ERROR"):
|
|
282
|
+
return "error"
|
|
283
|
+
return "ok"
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def _error(span: dict[str, Any]) -> str | None:
|
|
287
|
+
status = span.get("status", {})
|
|
288
|
+
if isinstance(status, dict):
|
|
289
|
+
return status.get("message")
|
|
290
|
+
return None
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def _direct_attribute_keys() -> set[str]:
|
|
294
|
+
return {
|
|
295
|
+
"agentloop.event_type",
|
|
296
|
+
"agentloop.name",
|
|
297
|
+
"agentloop.trace.name",
|
|
298
|
+
"gen_ai.operation.name",
|
|
299
|
+
"gen_ai.request.model",
|
|
300
|
+
"gen_ai.response.model",
|
|
301
|
+
"gen_ai.tool.name",
|
|
302
|
+
}
|