agent-observability-trace-cli 0.1.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_observability_trace_cli-0.1.2.dist-info/METADATA +336 -0
- agent_observability_trace_cli-0.1.2.dist-info/RECORD +51 -0
- agent_observability_trace_cli-0.1.2.dist-info/WHEEL +4 -0
- agent_observability_trace_cli-0.1.2.dist-info/entry_points.txt +2 -0
- agent_observability_trace_cli-0.1.2.dist-info/licenses/LICENSE +198 -0
- agent_trace/__init__.py +1182 -0
- agent_trace/_cli.py +1377 -0
- agent_trace/_inspect.py +2020 -0
- agent_trace/_replay/__init__.py +1 -0
- agent_trace/_replay/engine.py +268 -0
- agent_trace/_replay/fixture.py +868 -0
- agent_trace/core/__init__.py +1 -0
- agent_trace/core/clock.py +91 -0
- agent_trace/core/exceptions.py +51 -0
- agent_trace/core/span.py +271 -0
- agent_trace/core/trace.py +92 -0
- agent_trace/exporters/__init__.py +1 -0
- agent_trace/exporters/file.py +80 -0
- agent_trace/exporters/otlp.py +199 -0
- agent_trace/exporters/remote_fixture.py +307 -0
- agent_trace/exporters/stdout.py +258 -0
- agent_trace/integrations/__init__.py +69 -0
- agent_trace/integrations/agno.py +489 -0
- agent_trace/integrations/autogen.py +581 -0
- agent_trace/integrations/crewai.py +486 -0
- agent_trace/integrations/google_genai.py +731 -0
- agent_trace/integrations/haystack.py +254 -0
- agent_trace/integrations/langchain_core.py +361 -0
- agent_trace/integrations/langgraph.py +2093 -0
- agent_trace/integrations/langgraph_checkpoint.py +763 -0
- agent_trace/integrations/langgraph_state_diff.py +345 -0
- agent_trace/integrations/langgraph_stream_debug.py +202 -0
- agent_trace/integrations/llama_index.py +472 -0
- agent_trace/integrations/mcp.py +281 -0
- agent_trace/integrations/openai_agents.py +811 -0
- agent_trace/integrations/pydantic_ai.py +504 -0
- agent_trace/integrations/streaming.py +218 -0
- agent_trace/interceptor/__init__.py +64 -0
- agent_trace/interceptor/aiohttp_hook.py +152 -0
- agent_trace/interceptor/botocore_hook.py +223 -0
- agent_trace/interceptor/grpc_hook.py +651 -0
- agent_trace/interceptor/httpx_hook.py +866 -0
- agent_trace/interceptor/logging_hook.py +137 -0
- agent_trace/interceptor/requests_patch.py +184 -0
- agent_trace/interceptor/sse.py +176 -0
- agent_trace/interceptor/stdio_hook.py +245 -0
- agent_trace/interceptor/warnings_hook.py +132 -0
- agent_trace/interceptor/websocket_hook.py +323 -0
- agent_trace/plugins/__init__.py +35 -0
- agent_trace/plugins/base.py +111 -0
- agent_trace/py.typed +0 -0
agent_trace/_cli.py
ADDED
|
@@ -0,0 +1,1377 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Command-line interface for agent-trace.
|
|
3
|
+
|
|
4
|
+
Commands:
|
|
5
|
+
agent-trace replay <run_id> — replay a recorded run and print span tree
|
|
6
|
+
agent-trace list — list all recorded runs in trace_dir
|
|
7
|
+
agent-trace show <run_id> — show the stored trace.json for a run
|
|
8
|
+
agent-trace inspect <run_id> — auto-flag anomalies in captured bodies
|
|
9
|
+
agent-trace diff <run_id_a> <b> — diff two recorded runs' exchanges
|
|
10
|
+
agent-trace run -- <command> — exec a child process, recording pre-enabled
|
|
11
|
+
agent-trace version — print version
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import argparse
|
|
17
|
+
import datetime
|
|
18
|
+
import difflib
|
|
19
|
+
import json
|
|
20
|
+
import os
|
|
21
|
+
import subprocess
|
|
22
|
+
import sys
|
|
23
|
+
import uuid
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
from typing import Any
|
|
26
|
+
|
|
27
|
+
__all__ = ["main"]
|
|
28
|
+
|
|
29
|
+
_VERSION = "0.1.0"
|
|
30
|
+
_DEFAULT_TRACE_DIR = Path.home() / ".agent-trace" / "runs"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
# ---------------------------------------------------------------------------
|
|
34
|
+
# Helpers
|
|
35
|
+
# ---------------------------------------------------------------------------
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _trace_dir() -> Path:
|
|
39
|
+
"""Return the active trace directory, honouring the env override."""
|
|
40
|
+
env_dir = os.environ.get("AGENT_TRACE_TRACE_DIR")
|
|
41
|
+
if env_dir:
|
|
42
|
+
return Path(env_dir)
|
|
43
|
+
return _DEFAULT_TRACE_DIR
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _run_dir(run_id: str) -> Path:
|
|
47
|
+
base = _trace_dir().resolve()
|
|
48
|
+
candidate = (base / run_id).resolve()
|
|
49
|
+
try:
|
|
50
|
+
candidate.relative_to(base)
|
|
51
|
+
except ValueError:
|
|
52
|
+
raise ValueError(
|
|
53
|
+
f"Invalid run_id {run_id!r}: path traversal detected"
|
|
54
|
+
) from None
|
|
55
|
+
return candidate
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _fixture_path(run_id: str) -> Path:
|
|
59
|
+
return _run_dir(run_id) / "fixture.db"
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _trace_json_path(run_id: str) -> Path:
|
|
63
|
+
return _run_dir(run_id) / "trace.json"
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _require_run_dir(run_id: str) -> Path:
|
|
67
|
+
"""Return the run directory or exit with a clear message."""
|
|
68
|
+
d = _run_dir(run_id)
|
|
69
|
+
if not d.exists():
|
|
70
|
+
sys.exit(
|
|
71
|
+
f"error: no run directory found for {run_id!r}\n"
|
|
72
|
+
f"Expected: {d}\n"
|
|
73
|
+
"Run 'agent-trace list' to see all recorded runs."
|
|
74
|
+
)
|
|
75
|
+
return d
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
# ---------------------------------------------------------------------------
|
|
79
|
+
# Subcommand: version
|
|
80
|
+
# ---------------------------------------------------------------------------
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def cmd_version(_args: argparse.Namespace) -> None:
|
|
84
|
+
print(f"agent-trace {_VERSION}")
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
# ---------------------------------------------------------------------------
|
|
88
|
+
# Subcommand: list
|
|
89
|
+
# ---------------------------------------------------------------------------
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def cmd_list(_args: argparse.Namespace) -> None:
|
|
93
|
+
"""List all recorded runs in the trace directory."""
|
|
94
|
+
trace_dir = _trace_dir()
|
|
95
|
+
if not trace_dir.exists():
|
|
96
|
+
print(f"No runs yet. Trace directory does not exist: {trace_dir}")
|
|
97
|
+
return
|
|
98
|
+
|
|
99
|
+
run_dirs = sorted(
|
|
100
|
+
(d for d in trace_dir.iterdir() if d.is_dir()),
|
|
101
|
+
key=lambda d: d.stat().st_mtime,
|
|
102
|
+
reverse=True,
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
if not run_dirs:
|
|
106
|
+
print(f"No recorded runs in {trace_dir}")
|
|
107
|
+
return
|
|
108
|
+
|
|
109
|
+
# Header
|
|
110
|
+
print(f"{'RUN ID':<30} {'EXCHANGES':>9} {'SPANS':>6} RECORDED AT")
|
|
111
|
+
print("-" * 70)
|
|
112
|
+
|
|
113
|
+
for run_dir in run_dirs:
|
|
114
|
+
run_id = run_dir.name
|
|
115
|
+
exchanges = _count_exchanges(run_dir)
|
|
116
|
+
spans = _count_spans(run_dir)
|
|
117
|
+
mtime = _format_mtime(run_dir.stat().st_mtime)
|
|
118
|
+
print(f"{run_id:<30} {exchanges:>9} {spans:>6} {mtime}")
|
|
119
|
+
|
|
120
|
+
print()
|
|
121
|
+
print(f"Total: {len(run_dirs)} run(s) in {trace_dir}")
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _count_exchanges(run_dir: Path) -> str:
|
|
125
|
+
fixture_db = run_dir / "fixture.db"
|
|
126
|
+
if not fixture_db.exists():
|
|
127
|
+
return "-"
|
|
128
|
+
try:
|
|
129
|
+
from agent_trace._replay.fixture import Fixture
|
|
130
|
+
|
|
131
|
+
with Fixture(fixture_db) as f:
|
|
132
|
+
return str(f.exchange_count())
|
|
133
|
+
except Exception:
|
|
134
|
+
return "?"
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _count_spans(run_dir: Path) -> str:
|
|
138
|
+
trace_json = run_dir / "trace.json"
|
|
139
|
+
if not trace_json.exists():
|
|
140
|
+
return "-"
|
|
141
|
+
try:
|
|
142
|
+
data = json.loads(trace_json.read_text(encoding="utf-8"))
|
|
143
|
+
return str(len(data.get("spans", [])))
|
|
144
|
+
except Exception:
|
|
145
|
+
return "?"
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _format_mtime(ts: float) -> str:
|
|
149
|
+
dt = datetime.datetime.fromtimestamp(ts, tz=datetime.timezone.utc)
|
|
150
|
+
return dt.strftime("%Y-%m-%d %H:%M:%S UTC")
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
# ---------------------------------------------------------------------------
|
|
154
|
+
# Error classification summary — surfaces error.origin/error.known_pattern
|
|
155
|
+
# (set by the LangGraph integration's exception classifier) as a one-line
|
|
156
|
+
# pointer per ERROR span, instead of requiring a developer to read the raw
|
|
157
|
+
# trace.json blob and find those attributes by eye.
|
|
158
|
+
# ---------------------------------------------------------------------------
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _error_classification_rows(spans: list[dict[str, object]]) -> list[dict[str, str]]:
|
|
162
|
+
"""Return one row per ERROR-status span with its origin/known-pattern
|
|
163
|
+
classification (empty string for a field that wasn't set)."""
|
|
164
|
+
rows: list[dict[str, str]] = []
|
|
165
|
+
for span in spans:
|
|
166
|
+
if span.get("status") != "ERROR":
|
|
167
|
+
continue
|
|
168
|
+
attrs = span.get("attributes") or {}
|
|
169
|
+
if not isinstance(attrs, dict):
|
|
170
|
+
continue
|
|
171
|
+
rows.append(
|
|
172
|
+
{
|
|
173
|
+
"name": str(span.get("name", "?")),
|
|
174
|
+
"origin": str(attrs.get("error.origin", "")),
|
|
175
|
+
"known_pattern": str(attrs.get("error.known_pattern", "")),
|
|
176
|
+
}
|
|
177
|
+
)
|
|
178
|
+
return rows
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def _print_error_classification(spans: list[dict[str, object]]) -> None:
|
|
182
|
+
rows = _error_classification_rows(spans)
|
|
183
|
+
if not rows:
|
|
184
|
+
return
|
|
185
|
+
print()
|
|
186
|
+
print("Error classification:")
|
|
187
|
+
for row in rows:
|
|
188
|
+
origin = row["origin"] or "unclassified"
|
|
189
|
+
line = f" {row['name']:<30} origin={origin}"
|
|
190
|
+
if row["known_pattern"]:
|
|
191
|
+
line += f" known_pattern={row['known_pattern']}"
|
|
192
|
+
print(line)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
# ---------------------------------------------------------------------------
|
|
196
|
+
# Duplicate node span detection — flags a node:<name> span that appears more
|
|
197
|
+
# than once within a single trace, the shape produced when a resumed
|
|
198
|
+
# graph.stream()/.invoke() call re-executes a task that should have reused a
|
|
199
|
+
# cached checkpointed write (see issue #6050). Heuristic, not a definitive
|
|
200
|
+
# bug detector: an intentionally looping node (e.g. a ReAct agent) also
|
|
201
|
+
# produces repeated node:<name> spans by design — this surfaces the count as
|
|
202
|
+
# a fact for a developer to interpret, rather than claiming a verdict.
|
|
203
|
+
# ---------------------------------------------------------------------------
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def _duplicate_node_span_counts(spans: list[dict[str, object]]) -> dict[str, int]:
|
|
207
|
+
"""Return {node span name: occurrence count} for every node:<name> span
|
|
208
|
+
name that appears more than once in *spans*."""
|
|
209
|
+
counts: dict[str, int] = {}
|
|
210
|
+
for span in spans:
|
|
211
|
+
name = span.get("name")
|
|
212
|
+
if isinstance(name, str) and name.startswith("node:"):
|
|
213
|
+
counts[name] = counts.get(name, 0) + 1
|
|
214
|
+
return {name: n for name, n in counts.items() if n > 1}
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _print_duplicate_node_spans(spans: list[dict[str, object]]) -> None:
|
|
218
|
+
duplicates = _duplicate_node_span_counts(spans)
|
|
219
|
+
if not duplicates:
|
|
220
|
+
return
|
|
221
|
+
print()
|
|
222
|
+
print("Duplicate node spans (same node:<name> executed more than once):")
|
|
223
|
+
for name, count in sorted(duplicates.items()):
|
|
224
|
+
print(f" {name:<30} executed {count} times")
|
|
225
|
+
print(
|
|
226
|
+
" (Expected for an intentionally looping node — e.g. a ReAct agent. "
|
|
227
|
+
"Unexpected for a task that should have reused a cached checkpointed "
|
|
228
|
+
"write across a resume — see issue #6050.)"
|
|
229
|
+
)
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
# ---------------------------------------------------------------------------
|
|
233
|
+
# Streaming timing diagnostic — surfaces time-to-first-chunk and max
|
|
234
|
+
# inter-chunk gap for any exchange recorded via a pass-through streaming
|
|
235
|
+
# transport (RecordingTransport(..., stream=True) /
|
|
236
|
+
# AsyncRecordingTransport(..., stream=True)). Exchanges recorded the default
|
|
237
|
+
# (eager-buffering) way carry no chunk_timestamps and are silently skipped —
|
|
238
|
+
# absence means "not captured this way", not "instant delivery".
|
|
239
|
+
# ---------------------------------------------------------------------------
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _streaming_timing_rows(
|
|
243
|
+
exchanges: list[dict[str, object]],
|
|
244
|
+
) -> list[dict[str, object]]:
|
|
245
|
+
"""One row per exchange with recorded per-chunk timestamps."""
|
|
246
|
+
from agent_trace._replay.fixture import (
|
|
247
|
+
max_inter_chunk_gap_ms,
|
|
248
|
+
time_to_first_chunk_ms,
|
|
249
|
+
)
|
|
250
|
+
|
|
251
|
+
rows: list[dict[str, object]] = []
|
|
252
|
+
for exchange in exchanges:
|
|
253
|
+
if not exchange.get("chunk_timestamps"):
|
|
254
|
+
continue
|
|
255
|
+
rows.append(
|
|
256
|
+
{
|
|
257
|
+
"url": exchange.get("url", "?"),
|
|
258
|
+
"method": exchange.get("method", "?"),
|
|
259
|
+
"chunk_count": len(exchange["chunk_timestamps"]), # type: ignore[arg-type]
|
|
260
|
+
"time_to_first_chunk_ms": time_to_first_chunk_ms(exchange),
|
|
261
|
+
"max_inter_chunk_gap_ms": max_inter_chunk_gap_ms(exchange),
|
|
262
|
+
}
|
|
263
|
+
)
|
|
264
|
+
return rows
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def _print_streaming_timing(exchanges: list[dict[str, object]]) -> None:
|
|
268
|
+
rows = _streaming_timing_rows(exchanges)
|
|
269
|
+
if not rows:
|
|
270
|
+
return
|
|
271
|
+
print()
|
|
272
|
+
print("Streaming timing (time-to-first-chunk / max inter-chunk gap):")
|
|
273
|
+
for row in rows:
|
|
274
|
+
ttfc = row["time_to_first_chunk_ms"]
|
|
275
|
+
gap = row["max_inter_chunk_gap_ms"]
|
|
276
|
+
ttfc_str = f"{ttfc:.1f}ms" if isinstance(ttfc, (int, float)) else "?"
|
|
277
|
+
gap_str = f"{gap:.1f}ms" if isinstance(gap, (int, float)) else "?"
|
|
278
|
+
print(
|
|
279
|
+
f" {row['method']:<6} {row['url']:<50} "
|
|
280
|
+
f"chunks={row['chunk_count']:>4} "
|
|
281
|
+
f"first_chunk={ttfc_str:>10} max_gap={gap_str:>10}"
|
|
282
|
+
)
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
# ---------------------------------------------------------------------------
|
|
286
|
+
# HTTP error exchange summary — surfaces 4xx/5xx exchanges as flagged errors
|
|
287
|
+
# in `agent-trace replay` output instead of leaving them as undifferentiated
|
|
288
|
+
# raw rows indistinguishable from a normal 200 (a captured error response
|
|
289
|
+
# like Azure's content_filter 400 sits in the fixture with nothing in the
|
|
290
|
+
# CLI distinguishing it today).
|
|
291
|
+
# ---------------------------------------------------------------------------
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def _print_http_error_exchanges(exchanges: list[dict[str, object]]) -> None:
|
|
295
|
+
from agent_trace import _inspect as ins
|
|
296
|
+
|
|
297
|
+
flags = ins.flag_4xx_5xx_exchanges(exchanges)
|
|
298
|
+
if not flags:
|
|
299
|
+
return
|
|
300
|
+
print()
|
|
301
|
+
print(f"HTTP error exchanges ({len(flags)}):")
|
|
302
|
+
for flag in flags:
|
|
303
|
+
print(f" {flag['method']:<6} {flag['url']:<50} HTTP {flag['status']}")
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
# ---------------------------------------------------------------------------
|
|
307
|
+
# Checkpoint durability diagnostic — correlates checkpoint:put/put_writes
|
|
308
|
+
# spans (recorded by TracingCheckpointSaver,
|
|
309
|
+
# agent_trace.integrations.langgraph_checkpoint) against the rest of the
|
|
310
|
+
# trace, surfacing a terminal durable|partial|abandoned status per the shape
|
|
311
|
+
# independently proposed in the #5672 thread (cancellation_requested,
|
|
312
|
+
# writes_enqueued_count, writes_flushed_count, checkpoint_status) instead of
|
|
313
|
+
# requiring a developer to manually diff two span sets by hand to answer
|
|
314
|
+
# "did what the user saw actually get durably persisted before the run
|
|
315
|
+
# ended".
|
|
316
|
+
# ---------------------------------------------------------------------------
|
|
317
|
+
|
|
318
|
+
_CHECKPOINT_WRITE_SPAN_NAMES = frozenset(
|
|
319
|
+
{
|
|
320
|
+
"checkpoint:put",
|
|
321
|
+
"checkpoint:aput",
|
|
322
|
+
"checkpoint:put_writes",
|
|
323
|
+
"checkpoint:aput_writes",
|
|
324
|
+
}
|
|
325
|
+
)
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def _checkpoint_durability_summary(
|
|
329
|
+
spans: list[dict[str, object]],
|
|
330
|
+
) -> dict[str, object] | None:
|
|
331
|
+
"""Return the durability summary for *spans*, or None if no
|
|
332
|
+
checkpoint-write spans were recorded in this trace at all (the
|
|
333
|
+
TracingCheckpointSaver wrapper wasn't wired in — nothing to report,
|
|
334
|
+
distinct from "wired in but zero writes flushed")."""
|
|
335
|
+
write_spans = [s for s in spans if s.get("name") in _CHECKPOINT_WRITE_SPAN_NAMES]
|
|
336
|
+
if not write_spans:
|
|
337
|
+
return None
|
|
338
|
+
|
|
339
|
+
enqueued = len(write_spans)
|
|
340
|
+
flushed = 0
|
|
341
|
+
for span in write_spans:
|
|
342
|
+
attrs = span.get("attributes") or {}
|
|
343
|
+
if isinstance(attrs, dict) and attrs.get("checkpoint.completed") is True:
|
|
344
|
+
flushed += 1
|
|
345
|
+
|
|
346
|
+
cancellation_requested = any(s.get("status") == "CANCELLED" for s in spans)
|
|
347
|
+
|
|
348
|
+
if flushed == enqueued and not cancellation_requested:
|
|
349
|
+
status = "durable"
|
|
350
|
+
elif flushed == 0:
|
|
351
|
+
status = "abandoned"
|
|
352
|
+
else:
|
|
353
|
+
status = "partial"
|
|
354
|
+
|
|
355
|
+
return {
|
|
356
|
+
"cancellation_requested": cancellation_requested,
|
|
357
|
+
"writes_enqueued_count": enqueued,
|
|
358
|
+
"writes_flushed_count": flushed,
|
|
359
|
+
"checkpoint_status": status,
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
def _print_checkpoint_durability(spans: list[dict[str, object]]) -> None:
|
|
364
|
+
summary = _checkpoint_durability_summary(spans)
|
|
365
|
+
if summary is None:
|
|
366
|
+
return
|
|
367
|
+
print()
|
|
368
|
+
print("Checkpoint durability:")
|
|
369
|
+
print(f" checkpoint_status: {summary['checkpoint_status']}")
|
|
370
|
+
print(f" writes_enqueued_count: {summary['writes_enqueued_count']}")
|
|
371
|
+
print(f" writes_flushed_count: {summary['writes_flushed_count']}")
|
|
372
|
+
print(f" cancellation_requested: {summary['cancellation_requested']}")
|
|
373
|
+
if summary["checkpoint_status"] != "durable":
|
|
374
|
+
print(
|
|
375
|
+
" (Not every checkpoint write span closed checkpoint.completed=True "
|
|
376
|
+
"before the run ended — state observed during the run may not "
|
|
377
|
+
"match what was actually persisted. See issue #5672.)"
|
|
378
|
+
)
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
# ---------------------------------------------------------------------------
|
|
382
|
+
# Zero-tasks-scheduled diagnostic — flags a checkpoint:update_state span
|
|
383
|
+
# (recorded by traced_update_state()/traced_aupdate_state(),
|
|
384
|
+
# agent_trace.integrations.langgraph_checkpoint) whose post-write task
|
|
385
|
+
# schedule came back empty, the exact silent-no-op-resume shape behind issue
|
|
386
|
+
# #4217 (a missing/incorrect as_node on an external state write).
|
|
387
|
+
# ---------------------------------------------------------------------------
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
def _zero_task_update_rows(spans: list[dict[str, object]]) -> list[dict[str, object]]:
|
|
391
|
+
rows: list[dict[str, object]] = []
|
|
392
|
+
for span in spans:
|
|
393
|
+
if span.get("name") != "checkpoint:update_state":
|
|
394
|
+
continue
|
|
395
|
+
attrs = span.get("attributes") or {}
|
|
396
|
+
if not isinstance(attrs, dict):
|
|
397
|
+
continue
|
|
398
|
+
if attrs.get("checkpoint.zero_tasks_scheduled") is True:
|
|
399
|
+
rows.append(
|
|
400
|
+
{
|
|
401
|
+
"as_node": attrs.get("checkpoint.as_node", "<not provided>"),
|
|
402
|
+
"as_node_provided": attrs.get("checkpoint.as_node_provided", False),
|
|
403
|
+
}
|
|
404
|
+
)
|
|
405
|
+
return rows
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
def _print_zero_task_updates(spans: list[dict[str, object]]) -> None:
|
|
409
|
+
rows = _zero_task_update_rows(spans)
|
|
410
|
+
if not rows:
|
|
411
|
+
return
|
|
412
|
+
print()
|
|
413
|
+
print("Zero tasks scheduled after a state update (likely misattributed write):")
|
|
414
|
+
for row in rows:
|
|
415
|
+
provided = "yes" if row["as_node_provided"] else "no"
|
|
416
|
+
print(f" as_node={row['as_node']} (explicitly provided: {provided})")
|
|
417
|
+
print(
|
|
418
|
+
" (An update_state() call was followed by an empty scheduled-task "
|
|
419
|
+
"list — the graph will not advance from this state. Usually a "
|
|
420
|
+
"missing/incorrect as_node. See issue #4217.)"
|
|
421
|
+
)
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
# ---------------------------------------------------------------------------
|
|
425
|
+
# Exception-text surfacing — pulls the exception.message off any ERROR-status
|
|
426
|
+
# span (already captured by Span.record_exception) into a print-ready view,
|
|
427
|
+
# for both `agent-trace show --errors-only` and the plain-text summary block
|
|
428
|
+
# both `show`/`replay` print after their main output.
|
|
429
|
+
# ---------------------------------------------------------------------------
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
def _span_exception_message(span: dict[str, object]) -> str | None:
|
|
433
|
+
"""Return the exception.message text for *span*, or None if it has no
|
|
434
|
+
recorded exception event."""
|
|
435
|
+
events = span.get("events")
|
|
436
|
+
if not isinstance(events, list):
|
|
437
|
+
return None
|
|
438
|
+
for event in events:
|
|
439
|
+
if not isinstance(event, dict) or event.get("name") != "exception":
|
|
440
|
+
continue
|
|
441
|
+
attrs = event.get("attributes") or {}
|
|
442
|
+
if isinstance(attrs, dict) and "exception.message" in attrs:
|
|
443
|
+
return str(attrs["exception.message"])
|
|
444
|
+
return None
|
|
445
|
+
|
|
446
|
+
|
|
447
|
+
def _span_exception_http_detail(span: dict[str, object]) -> str | None:
|
|
448
|
+
"""Return "HTTP <status>: <body preview>" for *span*'s recorded
|
|
449
|
+
exception event when it carried an HTTP error response body (see
|
|
450
|
+
Span.record_exception's exception.http_response_body/
|
|
451
|
+
exception.http_status_code attributes, core/span.py) — the actual
|
|
452
|
+
provider/proxy error text (#4940), not just the generic one-line
|
|
453
|
+
str(exc) message which is all `exception.message` alone gives you for
|
|
454
|
+
e.g. requests.exceptions.HTTPError."""
|
|
455
|
+
events = span.get("events")
|
|
456
|
+
if not isinstance(events, list):
|
|
457
|
+
return None
|
|
458
|
+
for event in events:
|
|
459
|
+
if not isinstance(event, dict) or event.get("name") != "exception":
|
|
460
|
+
continue
|
|
461
|
+
attrs = event.get("attributes") or {}
|
|
462
|
+
if not isinstance(attrs, dict):
|
|
463
|
+
continue
|
|
464
|
+
body = attrs.get("exception.http_response_body")
|
|
465
|
+
if not body:
|
|
466
|
+
continue
|
|
467
|
+
status = attrs.get("exception.http_status_code")
|
|
468
|
+
prefix = f"HTTP {status}: " if status is not None else "HTTP: "
|
|
469
|
+
return f"{prefix}{body}"
|
|
470
|
+
return None
|
|
471
|
+
|
|
472
|
+
|
|
473
|
+
def _error_spans(spans: list[dict[str, object]]) -> list[dict[str, object]]:
|
|
474
|
+
return [s for s in spans if s.get("status") == "ERROR"]
|
|
475
|
+
|
|
476
|
+
|
|
477
|
+
def _print_errors_only(spans: list[dict[str, object]]) -> None:
|
|
478
|
+
"""`agent-trace show --errors-only` — filter to ERROR-status spans and
|
|
479
|
+
print each with its captured exception text inline, instead of dumping
|
|
480
|
+
the full trace.json and requiring a manual grep for
|
|
481
|
+
"exception.stacktrace"."""
|
|
482
|
+
errors = _error_spans(spans)
|
|
483
|
+
print(f"Error spans: {len(errors)} of {len(spans)} total")
|
|
484
|
+
if not errors:
|
|
485
|
+
return
|
|
486
|
+
print()
|
|
487
|
+
for span in errors:
|
|
488
|
+
print(f"[ERR] {span.get('name', '?')}")
|
|
489
|
+
message = _span_exception_message(span)
|
|
490
|
+
if message:
|
|
491
|
+
print(f" {message}")
|
|
492
|
+
http_detail = _span_exception_http_detail(span)
|
|
493
|
+
if http_detail:
|
|
494
|
+
print(f" {http_detail}")
|
|
495
|
+
_print_error_classification(spans)
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
# ---------------------------------------------------------------------------
|
|
499
|
+
# Retry-storm detection — flags a single node:* span with more than one
|
|
500
|
+
# llm:* child span, the shape produced by application code that silently
|
|
501
|
+
# re-invokes the LLM in a loop when a response is empty/malformed (#2920):
|
|
502
|
+
# an invisible cause of "sometimes fast, sometimes very slow" with no error
|
|
503
|
+
# and no user-facing signal.
|
|
504
|
+
# ---------------------------------------------------------------------------
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
def _retry_storm_rows(spans: list[dict[str, object]]) -> list[dict[str, object]]:
|
|
508
|
+
children_by_parent: dict[str | None, list[dict[str, object]]] = {}
|
|
509
|
+
for span in spans:
|
|
510
|
+
parent_id = span.get("parent_id")
|
|
511
|
+
parent_key = parent_id if isinstance(parent_id, str) else None
|
|
512
|
+
children_by_parent.setdefault(parent_key, []).append(span)
|
|
513
|
+
|
|
514
|
+
rows: list[dict[str, object]] = []
|
|
515
|
+
for span in spans:
|
|
516
|
+
name = span.get("name")
|
|
517
|
+
if not isinstance(name, str) or not name.startswith("node:"):
|
|
518
|
+
continue
|
|
519
|
+
span_id = span.get("span_id")
|
|
520
|
+
children = (
|
|
521
|
+
children_by_parent.get(span_id, []) if isinstance(span_id, str) else []
|
|
522
|
+
)
|
|
523
|
+
llm_children = [
|
|
524
|
+
c for c in children if str(c.get("name", "")).startswith("llm:")
|
|
525
|
+
]
|
|
526
|
+
if len(llm_children) > 1:
|
|
527
|
+
rows.append({"node": name, "llm_child_count": len(llm_children)})
|
|
528
|
+
return rows
|
|
529
|
+
|
|
530
|
+
|
|
531
|
+
def _print_retry_storms(spans: list[dict[str, object]]) -> None:
|
|
532
|
+
rows = _retry_storm_rows(spans)
|
|
533
|
+
if not rows:
|
|
534
|
+
return
|
|
535
|
+
print()
|
|
536
|
+
print("Repeated LLM calls under one node span (possible retry storm):")
|
|
537
|
+
for row in rows:
|
|
538
|
+
print(f" {row['node']:<30} {row['llm_child_count']} llm:* child spans")
|
|
539
|
+
print(
|
|
540
|
+
" (A node re-invoking the LLM multiple times per call produces no "
|
|
541
|
+
"error and no user-facing signal, but explains 'sometimes fast, "
|
|
542
|
+
"sometimes very slow' behavior. See issue #2920.)"
|
|
543
|
+
)
|
|
544
|
+
|
|
545
|
+
|
|
546
|
+
# ---------------------------------------------------------------------------
|
|
547
|
+
# Orphaned/misattributed span detection — flags a span whose callback-derived
|
|
548
|
+
# parent_id looks suspicious: a root span (parent_id is None) that is not the
|
|
549
|
+
# trace's earliest root and whose start_time falls inside another span's
|
|
550
|
+
# active [start, end) window. This is the exact visible symptom of
|
|
551
|
+
# langgraph#3975 (ChatOpenAI calls flattened to the trace root instead of
|
|
552
|
+
# nested under their originating node when a compiled graph is piped into a
|
|
553
|
+
# raw lambda) — a reconciliation pass against the trace's own chronological
|
|
554
|
+
# span ordering (the callback-independent signal LangGraphTracer's
|
|
555
|
+
# parent_run_id-registry lookup has no visibility into) surfaces it
|
|
556
|
+
# automatically instead of requiring a developer to eyeball trace.json.
|
|
557
|
+
# ---------------------------------------------------------------------------
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
def _numeric_start_time(span: dict[str, object]) -> float | None:
|
|
561
|
+
start = span.get("start_time")
|
|
562
|
+
return float(start) if isinstance(start, (int, float)) else None
|
|
563
|
+
|
|
564
|
+
|
|
565
|
+
def _http_sequence_confirms(
|
|
566
|
+
suspect: dict[str, object],
|
|
567
|
+
likely_parent: dict[str, object],
|
|
568
|
+
exchanges: list[dict[str, object]],
|
|
569
|
+
) -> bool | None:
|
|
570
|
+
"""Cross-check a *suspect*/*likely_parent* guess — built purely from
|
|
571
|
+
wall-clock span `start_time`/`end_time` — against the independent,
|
|
572
|
+
always-correct total ordering already captured in fixture.db's
|
|
573
|
+
`sequence_num` column (see `_replay/fixture.py`'s module docstring: "a
|
|
574
|
+
monotonically increasing sequence_num" assigned once per HTTP round-trip
|
|
575
|
+
at record time, immune to the callback-registry misses that produce
|
|
576
|
+
misattributed spans in the first place). This is the reconciliation
|
|
577
|
+
pass the "unaccounted-for root span" heuristic above needs: two spans'
|
|
578
|
+
wall-clock timestamps can only be trusted so far, but the HTTP capture
|
|
579
|
+
layer's sequence_num ordering is a second, structurally independent
|
|
580
|
+
signal for the same underlying chronology.
|
|
581
|
+
|
|
582
|
+
Returns True when at least one HTTP exchange was recorded during
|
|
583
|
+
*likely_parent*'s activity strictly before *suspect* started (i.e. the
|
|
584
|
+
HTTP layer independently confirms *likely_parent* was already doing
|
|
585
|
+
work before *suspect* began — corroborating the timestamp-based guess),
|
|
586
|
+
False when every such exchange has a `sequence_num` *greater than* an
|
|
587
|
+
exchange recorded during *suspect*'s own window (the HTTP layer
|
|
588
|
+
contradicts the guess), or None when there isn't enough HTTP-layer data
|
|
589
|
+
on either side to judge (note *likely_parent*'s window necessarily
|
|
590
|
+
contains *suspect*'s, since that's how it was chosen — so only the
|
|
591
|
+
portion of *likely_parent*'s window strictly before *suspect* started is
|
|
592
|
+
used, otherwise every exchange inside *suspect*'s own window would also
|
|
593
|
+
trivially count as "inside likely_parent's window" too).
|
|
594
|
+
"""
|
|
595
|
+
|
|
596
|
+
def _sequence_nums_in(start: float, end_bound: float) -> list[int]:
|
|
597
|
+
seqs: list[int] = []
|
|
598
|
+
for exchange in exchanges:
|
|
599
|
+
recorded_at = exchange.get("recorded_at")
|
|
600
|
+
seq = exchange.get("sequence_num")
|
|
601
|
+
if not isinstance(recorded_at, (int, float)) or not isinstance(
|
|
602
|
+
seq, int
|
|
603
|
+
):
|
|
604
|
+
continue
|
|
605
|
+
if start <= recorded_at < end_bound:
|
|
606
|
+
seqs.append(seq)
|
|
607
|
+
return seqs
|
|
608
|
+
|
|
609
|
+
parent_start = _numeric_start_time(likely_parent)
|
|
610
|
+
suspect_start = _numeric_start_time(suspect)
|
|
611
|
+
if parent_start is None or suspect_start is None:
|
|
612
|
+
return None
|
|
613
|
+
|
|
614
|
+
suspect_end = suspect.get("end_time")
|
|
615
|
+
suspect_end_bound = (
|
|
616
|
+
suspect_end if isinstance(suspect_end, (int, float)) else float("inf")
|
|
617
|
+
)
|
|
618
|
+
|
|
619
|
+
parent_exclusive_seqs = _sequence_nums_in(parent_start, suspect_start)
|
|
620
|
+
suspect_seqs = _sequence_nums_in(suspect_start, suspect_end_bound)
|
|
621
|
+
if not parent_exclusive_seqs or not suspect_seqs:
|
|
622
|
+
return None
|
|
623
|
+
return max(parent_exclusive_seqs) < min(suspect_seqs)
|
|
624
|
+
|
|
625
|
+
|
|
626
|
+
def _misattributed_span_rows(
|
|
627
|
+
spans: list[dict[str, object]],
|
|
628
|
+
exchanges: list[dict[str, object]] | None = None,
|
|
629
|
+
) -> list[dict[str, object]]:
|
|
630
|
+
rootlike = [s for s in spans if s.get("parent_id") is None]
|
|
631
|
+
timed_roots = [
|
|
632
|
+
(s, t) for s in rootlike if (t := _numeric_start_time(s)) is not None
|
|
633
|
+
]
|
|
634
|
+
roots = [s for s, _t in sorted(timed_roots, key=lambda pair: pair[1])]
|
|
635
|
+
if len(roots) <= 1:
|
|
636
|
+
return []
|
|
637
|
+
|
|
638
|
+
suspicious_root_ids = {id(s) for s in roots[1:]}
|
|
639
|
+
suspicious_roots = roots[1:]
|
|
640
|
+
|
|
641
|
+
# Spans that could plausibly be the "real" parent: anything open (no
|
|
642
|
+
# end_time, or start <= suspect.start < end) at the moment the
|
|
643
|
+
# suspicious root started. Includes the trace's earliest/genuine root —
|
|
644
|
+
# the langgraph#3975 shape is exactly "this call should have nested
|
|
645
|
+
# under the run's actual root/node span but didn't". Excludes other
|
|
646
|
+
# suspicious roots (one flattened call is never the "real" parent of
|
|
647
|
+
# another).
|
|
648
|
+
candidates = [s for s in spans if id(s) not in suspicious_root_ids]
|
|
649
|
+
|
|
650
|
+
rows: list[dict[str, object]] = []
|
|
651
|
+
for suspect in suspicious_roots:
|
|
652
|
+
start = _numeric_start_time(suspect)
|
|
653
|
+
if start is None:
|
|
654
|
+
continue
|
|
655
|
+
best: dict[str, object] | None = None
|
|
656
|
+
best_start: float | None = None
|
|
657
|
+
for candidate in candidates:
|
|
658
|
+
if candidate is suspect:
|
|
659
|
+
continue
|
|
660
|
+
c_start = _numeric_start_time(candidate)
|
|
661
|
+
if c_start is None:
|
|
662
|
+
continue
|
|
663
|
+
c_end = candidate.get("end_time")
|
|
664
|
+
if isinstance(c_end, (int, float)):
|
|
665
|
+
still_open = c_start <= start < c_end
|
|
666
|
+
else:
|
|
667
|
+
# No end_time (still open) or a malformed non-numeric value —
|
|
668
|
+
# treat both as "open" rather than excluding the candidate.
|
|
669
|
+
still_open = c_start <= start
|
|
670
|
+
if still_open and (best_start is None or c_start > best_start):
|
|
671
|
+
best = candidate
|
|
672
|
+
best_start = c_start
|
|
673
|
+
if best is not None:
|
|
674
|
+
row: dict[str, object] = {
|
|
675
|
+
"span": suspect.get("name"),
|
|
676
|
+
"likely_parent": best.get("name"),
|
|
677
|
+
}
|
|
678
|
+
if exchanges is not None:
|
|
679
|
+
row["http_sequence_confirmed"] = _http_sequence_confirms(
|
|
680
|
+
suspect, best, exchanges
|
|
681
|
+
)
|
|
682
|
+
rows.append(row)
|
|
683
|
+
return rows
|
|
684
|
+
|
|
685
|
+
|
|
686
|
+
def _print_misattributed_spans(
|
|
687
|
+
spans: list[dict[str, object]],
|
|
688
|
+
exchanges: list[dict[str, object]] | None = None,
|
|
689
|
+
) -> None:
|
|
690
|
+
rows = _misattributed_span_rows(spans, exchanges)
|
|
691
|
+
if not rows:
|
|
692
|
+
return
|
|
693
|
+
print()
|
|
694
|
+
print(
|
|
695
|
+
"Possibly misattributed spans (unexpected root, chronologically "
|
|
696
|
+
"overlaps another span):"
|
|
697
|
+
)
|
|
698
|
+
for row in rows:
|
|
699
|
+
suffix = ""
|
|
700
|
+
confirmed = row.get("http_sequence_confirmed")
|
|
701
|
+
if confirmed is True:
|
|
702
|
+
suffix = " [confirmed via HTTP sequence_num ordering]"
|
|
703
|
+
elif confirmed is False:
|
|
704
|
+
suffix = " [HTTP sequence_num ordering does NOT confirm this guess]"
|
|
705
|
+
print(
|
|
706
|
+
f" {row['span']:<30} likely belongs under "
|
|
707
|
+
f"{row['likely_parent']}{suffix}"
|
|
708
|
+
)
|
|
709
|
+
print(
|
|
710
|
+
" (A span with no parent that started while another span was still "
|
|
711
|
+
"open may have been flattened to the trace root instead of nested "
|
|
712
|
+
"under its originating node — the shape behind langgraph#3975. "
|
|
713
|
+
"Reconciled against fixture.db's sequence_num-ordered HTTP capture "
|
|
714
|
+
"where available, per the timestamp-independent cross-check above.)"
|
|
715
|
+
)
|
|
716
|
+
|
|
717
|
+
|
|
718
|
+
# ---------------------------------------------------------------------------
|
|
719
|
+
# Subcommand: show
|
|
720
|
+
# ---------------------------------------------------------------------------
|
|
721
|
+
|
|
722
|
+
|
|
723
|
+
def cmd_show(args: argparse.Namespace) -> None:
|
|
724
|
+
"""Print the trace.json for a run, pretty-printed."""
|
|
725
|
+
run_id: str = args.run_id
|
|
726
|
+
_require_run_dir(run_id)
|
|
727
|
+
|
|
728
|
+
trace_path = _trace_json_path(run_id)
|
|
729
|
+
if not trace_path.exists():
|
|
730
|
+
sys.exit(
|
|
731
|
+
f"error: no trace.json found for run {run_id!r}\n"
|
|
732
|
+
f"Expected: {trace_path}\n"
|
|
733
|
+
"The run directory exists but no trace was written. "
|
|
734
|
+
"Did the trace context exit normally?"
|
|
735
|
+
)
|
|
736
|
+
|
|
737
|
+
try:
|
|
738
|
+
data = json.loads(trace_path.read_text(encoding="utf-8"))
|
|
739
|
+
except json.JSONDecodeError as exc:
|
|
740
|
+
sys.exit(f"error: trace.json is not valid JSON: {exc}")
|
|
741
|
+
|
|
742
|
+
spans = data.get("spans", [])
|
|
743
|
+
|
|
744
|
+
if getattr(args, "errors_only", False):
|
|
745
|
+
_print_errors_only(spans)
|
|
746
|
+
return
|
|
747
|
+
|
|
748
|
+
# Try rich for colored output; fall back to plain json.dumps
|
|
749
|
+
try:
|
|
750
|
+
from rich.console import Console
|
|
751
|
+
from rich.syntax import Syntax
|
|
752
|
+
|
|
753
|
+
console = Console()
|
|
754
|
+
syntax = Syntax(
|
|
755
|
+
json.dumps(data, indent=2),
|
|
756
|
+
"json",
|
|
757
|
+
theme="monokai",
|
|
758
|
+
line_numbers=False,
|
|
759
|
+
)
|
|
760
|
+
console.print(syntax)
|
|
761
|
+
except ImportError:
|
|
762
|
+
print(json.dumps(data, indent=2))
|
|
763
|
+
|
|
764
|
+
print()
|
|
765
|
+
print(f"File: {trace_path}")
|
|
766
|
+
print(f"Spans: {len(spans)}")
|
|
767
|
+
|
|
768
|
+
# Load fixture.db's exchanges (if this run was recorded with
|
|
769
|
+
# record=True) purely so _print_misattributed_spans can reconcile its
|
|
770
|
+
# wall-clock-timestamp guess against the independent, always-correct
|
|
771
|
+
# sequence_num ordering the HTTP capture layer already has.
|
|
772
|
+
exchanges: list[dict[str, object]] | None = None
|
|
773
|
+
fixture_db = _fixture_path(run_id)
|
|
774
|
+
if fixture_db.exists():
|
|
775
|
+
try:
|
|
776
|
+
from agent_trace._replay.fixture import Fixture
|
|
777
|
+
|
|
778
|
+
with Fixture(fixture_db) as f:
|
|
779
|
+
exchanges = f.all_exchanges()
|
|
780
|
+
except Exception:
|
|
781
|
+
exchanges = None
|
|
782
|
+
|
|
783
|
+
_print_error_classification(spans)
|
|
784
|
+
_print_duplicate_node_spans(spans)
|
|
785
|
+
_print_retry_storms(spans)
|
|
786
|
+
_print_misattributed_spans(spans, exchanges)
|
|
787
|
+
_print_checkpoint_durability(spans)
|
|
788
|
+
_print_zero_task_updates(spans)
|
|
789
|
+
|
|
790
|
+
|
|
791
|
+
# ---------------------------------------------------------------------------
|
|
792
|
+
# Subcommand: replay
|
|
793
|
+
# ---------------------------------------------------------------------------
|
|
794
|
+
|
|
795
|
+
|
|
796
|
+
def _print_replay_span_diagnostics(
|
|
797
|
+
spans: list[dict[str, object]],
|
|
798
|
+
exchanges: list[dict[str, object]] | None = None,
|
|
799
|
+
) -> None:
|
|
800
|
+
"""The full diagnostic block `agent-trace replay` prints after the span
|
|
801
|
+
tree — split out from cmd_replay() purely to keep that function's
|
|
802
|
+
statement count manageable, not for reuse elsewhere."""
|
|
803
|
+
_print_error_classification(spans)
|
|
804
|
+
_print_duplicate_node_spans(spans)
|
|
805
|
+
_print_retry_storms(spans)
|
|
806
|
+
_print_misattributed_spans(spans, exchanges)
|
|
807
|
+
_print_checkpoint_durability(spans)
|
|
808
|
+
_print_zero_task_updates(spans)
|
|
809
|
+
|
|
810
|
+
|
|
811
|
+
def cmd_replay(args: argparse.Namespace) -> None:
|
|
812
|
+
"""Replay a recorded run and print the resulting span tree."""
|
|
813
|
+
run_id: str = args.run_id
|
|
814
|
+
_require_run_dir(run_id)
|
|
815
|
+
|
|
816
|
+
fixture = _fixture_path(run_id)
|
|
817
|
+
if not fixture.exists():
|
|
818
|
+
sys.exit(
|
|
819
|
+
f"error: no fixture.db found for run {run_id!r}\n"
|
|
820
|
+
f"Expected: {fixture}\n"
|
|
821
|
+
"Did you record this run with record=True?\n"
|
|
822
|
+
"Without record=True only trace.json is written, not fixture.db."
|
|
823
|
+
)
|
|
824
|
+
|
|
825
|
+
trace_json_path = _trace_json_path(run_id)
|
|
826
|
+
|
|
827
|
+
print(f"Replaying run: {run_id}")
|
|
828
|
+
print(f"Fixture: {fixture}")
|
|
829
|
+
|
|
830
|
+
# Load the original trace to find span names
|
|
831
|
+
original_spans: list[dict[str, object]] = []
|
|
832
|
+
if trace_json_path.exists():
|
|
833
|
+
try:
|
|
834
|
+
data = json.loads(trace_json_path.read_text(encoding="utf-8"))
|
|
835
|
+
original_spans = data.get("spans", [])
|
|
836
|
+
except Exception: # noqa: S110
|
|
837
|
+
pass
|
|
838
|
+
|
|
839
|
+
print(f"Original span count: {len(original_spans)}")
|
|
840
|
+
|
|
841
|
+
# Count exchanges via Fixture (honours WAL mode + schema)
|
|
842
|
+
from agent_trace._replay.fixture import Fixture
|
|
843
|
+
|
|
844
|
+
try:
|
|
845
|
+
with Fixture(fixture) as f:
|
|
846
|
+
exchange_count = f.exchange_count()
|
|
847
|
+
all_exchanges = f.all_exchanges()
|
|
848
|
+
except Exception:
|
|
849
|
+
exchange_count = 0
|
|
850
|
+
all_exchanges = []
|
|
851
|
+
|
|
852
|
+
print(f"Recorded exchanges: {exchange_count}")
|
|
853
|
+
_print_streaming_timing(all_exchanges)
|
|
854
|
+
_print_http_error_exchanges(all_exchanges)
|
|
855
|
+
print()
|
|
856
|
+
|
|
857
|
+
# Enter replay mode
|
|
858
|
+
from agent_trace import replay as _replay
|
|
859
|
+
|
|
860
|
+
with _replay(run_id, trace_dir=_trace_dir()) as ctx:
|
|
861
|
+
print(f"Replay active — {ctx.fixture.exchange_count()} exchange(s) available")
|
|
862
|
+
print("(No HTTP requests were made — fixture is ready for agent code)")
|
|
863
|
+
print()
|
|
864
|
+
|
|
865
|
+
# Load and print the original trace
|
|
866
|
+
if trace_json_path.exists():
|
|
867
|
+
try:
|
|
868
|
+
from agent_trace.core.trace import Trace
|
|
869
|
+
from agent_trace.exporters.stdout import StdoutExporter
|
|
870
|
+
|
|
871
|
+
trace_data = json.loads(trace_json_path.read_text(encoding="utf-8"))
|
|
872
|
+
trace_obj = Trace.from_dict(trace_data)
|
|
873
|
+
print("--- Original span tree (from trace.json) ---")
|
|
874
|
+
StdoutExporter().export(trace_obj)
|
|
875
|
+
_print_replay_span_diagnostics(trace_data.get("spans", []), all_exchanges)
|
|
876
|
+
except Exception as exc:
|
|
877
|
+
print(f"Could not render span tree: {exc}")
|
|
878
|
+
else:
|
|
879
|
+
print(f"(No trace.json — run 'agent-trace show {run_id}' after recording)")
|
|
880
|
+
|
|
881
|
+
|
|
882
|
+
# ---------------------------------------------------------------------------
|
|
883
|
+
# Subcommand: inspect
|
|
884
|
+
# ---------------------------------------------------------------------------
|
|
885
|
+
#
|
|
886
|
+
# The one-stop diagnosis command: decodes fixture.db request/response bodies
|
|
887
|
+
# and runs every pattern check in agent_trace._inspect against them,
|
|
888
|
+
# printing a flagged summary instead of requiring a developer to hand-write
|
|
889
|
+
# a comparison script against Fixture.all_exchanges() (the exact manual step
|
|
890
|
+
# this command replaces, closing the gap reported in upstream issue #531).
|
|
891
|
+
# ---------------------------------------------------------------------------
|
|
892
|
+
|
|
893
|
+
|
|
894
|
+
def _load_run_exchanges_and_spans(
|
|
895
|
+
run_id: str,
|
|
896
|
+
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
|
897
|
+
"""Return (exchanges, spans) for *run_id* — empty lists for whichever
|
|
898
|
+
file (fixture.db / trace.json) doesn't exist rather than raising, since
|
|
899
|
+
a run may have been recorded with record=False (spans only) or may not
|
|
900
|
+
have a trace.json yet."""
|
|
901
|
+
from agent_trace._replay.fixture import Fixture
|
|
902
|
+
|
|
903
|
+
exchanges: list[dict[str, Any]] = []
|
|
904
|
+
fixture_path = _fixture_path(run_id)
|
|
905
|
+
if fixture_path.exists():
|
|
906
|
+
with Fixture(fixture_path) as f:
|
|
907
|
+
exchanges = f.all_exchanges()
|
|
908
|
+
|
|
909
|
+
spans: list[dict[str, Any]] = []
|
|
910
|
+
trace_path = _trace_json_path(run_id)
|
|
911
|
+
if trace_path.exists():
|
|
912
|
+
try:
|
|
913
|
+
data = json.loads(trace_path.read_text(encoding="utf-8"))
|
|
914
|
+
spans = data.get("spans", [])
|
|
915
|
+
except json.JSONDecodeError:
|
|
916
|
+
pass
|
|
917
|
+
|
|
918
|
+
return exchanges, spans
|
|
919
|
+
|
|
920
|
+
|
|
921
|
+
def _print_flags(title: str, flags: list[dict[str, Any]]) -> None:
|
|
922
|
+
if not flags:
|
|
923
|
+
return
|
|
924
|
+
print()
|
|
925
|
+
print(f"{title} ({len(flags)}):")
|
|
926
|
+
for flag in flags:
|
|
927
|
+
print(f" - {flag.get('detail', flag)}")
|
|
928
|
+
|
|
929
|
+
|
|
930
|
+
def cmd_inspect(args: argparse.Namespace) -> None:
|
|
931
|
+
"""Auto-flag/search raw request-response bodies for known malformed
|
|
932
|
+
shapes, plus a set of cross-span diagnostics — a CLI command so a
|
|
933
|
+
developer doesn't have to hand-write a comparison script against
|
|
934
|
+
Fixture.all_exchanges()."""
|
|
935
|
+
from agent_trace import _inspect as ins
|
|
936
|
+
|
|
937
|
+
run_id: str = args.run_id
|
|
938
|
+
_require_run_dir(run_id)
|
|
939
|
+
exchanges, spans = _load_run_exchanges_and_spans(run_id)
|
|
940
|
+
|
|
941
|
+
print(f"Inspecting run: {run_id}")
|
|
942
|
+
print(f"Exchanges: {len(exchanges)} Spans: {len(spans)}")
|
|
943
|
+
|
|
944
|
+
results = ins.run_all_exchange_checks(exchanges)
|
|
945
|
+
for check_name, flags in results.items():
|
|
946
|
+
_print_flags(check_name, flags)
|
|
947
|
+
|
|
948
|
+
_print_flags(
|
|
949
|
+
"known_error_pattern",
|
|
950
|
+
ins.match_known_error_patterns(spans),
|
|
951
|
+
)
|
|
952
|
+
_print_flags(
|
|
953
|
+
"reserved_kwarg_collision",
|
|
954
|
+
ins.check_reserved_kwarg_collision(spans),
|
|
955
|
+
)
|
|
956
|
+
_print_flags(
|
|
957
|
+
"near_duplicate_sibling_content",
|
|
958
|
+
ins.find_near_duplicate_sibling_content(spans),
|
|
959
|
+
)
|
|
960
|
+
_print_flags(
|
|
961
|
+
"phantom_tool_call",
|
|
962
|
+
ins.check_phantom_tool_call(spans, exchanges),
|
|
963
|
+
)
|
|
964
|
+
_print_flags(
|
|
965
|
+
"system_prompt_dropped",
|
|
966
|
+
ins.check_system_prompt_dropped(spans),
|
|
967
|
+
)
|
|
968
|
+
|
|
969
|
+
if args.registered_tools:
|
|
970
|
+
registered = set(args.registered_tools.split(","))
|
|
971
|
+
_print_flags(
|
|
972
|
+
"tool_call_name_fuzzy_match",
|
|
973
|
+
ins.check_tool_call_name_fuzzy_match(exchanges, registered),
|
|
974
|
+
)
|
|
975
|
+
_print_flags(
|
|
976
|
+
"tool_call_name_dotted_compound",
|
|
977
|
+
ins.check_tool_call_name_dotted_compound(exchanges, registered),
|
|
978
|
+
)
|
|
979
|
+
_print_flags(
|
|
980
|
+
"action_name_not_registered",
|
|
981
|
+
ins.check_action_name_not_registered(exchanges, registered),
|
|
982
|
+
)
|
|
983
|
+
_print_flags(
|
|
984
|
+
"tool_call_name_not_registered",
|
|
985
|
+
ins.check_tool_call_name_not_registered(exchanges, registered),
|
|
986
|
+
)
|
|
987
|
+
|
|
988
|
+
if args.configured_host:
|
|
989
|
+
_print_flags(
|
|
990
|
+
"endpoint_host_mismatch",
|
|
991
|
+
ins.check_endpoint_host_mismatch(exchanges, args.configured_host),
|
|
992
|
+
)
|
|
993
|
+
|
|
994
|
+
if args.check_kwarg:
|
|
995
|
+
_print_flags(
|
|
996
|
+
"missing_extra_kwarg",
|
|
997
|
+
ins.check_missing_extra_kwarg(exchanges, args.check_kwarg),
|
|
998
|
+
)
|
|
999
|
+
|
|
1000
|
+
if args.diff_field:
|
|
1001
|
+
_print_flags(
|
|
1002
|
+
"field_present_on_wire_absent_downstream",
|
|
1003
|
+
ins.field_present_on_wire_absent_downstream(
|
|
1004
|
+
exchanges, spans, args.diff_field
|
|
1005
|
+
),
|
|
1006
|
+
)
|
|
1007
|
+
|
|
1008
|
+
if args.diff_get_post_field:
|
|
1009
|
+
_print_flags(
|
|
1010
|
+
"get_post_field_mismatch",
|
|
1011
|
+
ins.check_get_post_field_mismatch(
|
|
1012
|
+
exchanges,
|
|
1013
|
+
args.diff_get_post_field,
|
|
1014
|
+
get_id_field=args.diff_get_post_id_field,
|
|
1015
|
+
post_id_field=args.diff_get_post_post_id_field,
|
|
1016
|
+
),
|
|
1017
|
+
)
|
|
1018
|
+
|
|
1019
|
+
if not results and not spans:
|
|
1020
|
+
print()
|
|
1021
|
+
print("No anomalies flagged (or nothing recorded for this run).")
|
|
1022
|
+
|
|
1023
|
+
|
|
1024
|
+
# ---------------------------------------------------------------------------
|
|
1025
|
+
# Subcommand: diff
|
|
1026
|
+
# ---------------------------------------------------------------------------
|
|
1027
|
+
#
|
|
1028
|
+
# `agent-trace diff <run_id_a> <run_id_b>` — the CLI never previously
|
|
1029
|
+
# exposed raw exchange bodies at all, let alone a comparison between two
|
|
1030
|
+
# runs. cmd_show only pretty-prints trace.json span metadata; cmd_replay
|
|
1031
|
+
# only prints exchange counts and the span tree.
|
|
1032
|
+
# ---------------------------------------------------------------------------
|
|
1033
|
+
|
|
1034
|
+
|
|
1035
|
+
def _exchanges_by_url(
|
|
1036
|
+
exchanges: list[dict[str, Any]],
|
|
1037
|
+
) -> dict[str, list[dict[str, Any]]]:
|
|
1038
|
+
by_url: dict[str, list[dict[str, Any]]] = {}
|
|
1039
|
+
for exchange in exchanges:
|
|
1040
|
+
by_url.setdefault(str(exchange.get("url")), []).append(exchange)
|
|
1041
|
+
return by_url
|
|
1042
|
+
|
|
1043
|
+
|
|
1044
|
+
def _diff_text(label_a: str, text_a: str, label_b: str, text_b: str) -> list[str]:
|
|
1045
|
+
return list(
|
|
1046
|
+
difflib.unified_diff(
|
|
1047
|
+
text_a.splitlines(keepends=True),
|
|
1048
|
+
text_b.splitlines(keepends=True),
|
|
1049
|
+
fromfile=label_a,
|
|
1050
|
+
tofile=label_b,
|
|
1051
|
+
lineterm="",
|
|
1052
|
+
)
|
|
1053
|
+
)
|
|
1054
|
+
|
|
1055
|
+
|
|
1056
|
+
def _print_exchange_diff(
|
|
1057
|
+
exchanges_a: list[dict[str, Any]],
|
|
1058
|
+
exchanges_b: list[dict[str, Any]],
|
|
1059
|
+
run_id_a: str,
|
|
1060
|
+
run_id_b: str,
|
|
1061
|
+
) -> None:
|
|
1062
|
+
"""Print the field-level diff of two runs' exchanges, matched by URL."""
|
|
1063
|
+
by_url_a = _exchanges_by_url(exchanges_a)
|
|
1064
|
+
by_url_b = _exchanges_by_url(exchanges_b)
|
|
1065
|
+
|
|
1066
|
+
all_urls = sorted(set(by_url_a) | set(by_url_b))
|
|
1067
|
+
if not all_urls:
|
|
1068
|
+
print("No exchanges recorded in either run.")
|
|
1069
|
+
return
|
|
1070
|
+
|
|
1071
|
+
any_diff = False
|
|
1072
|
+
for url in all_urls:
|
|
1073
|
+
rows_a = by_url_a.get(url, [])
|
|
1074
|
+
rows_b = by_url_b.get(url, [])
|
|
1075
|
+
if not rows_a:
|
|
1076
|
+
print(f"\n{url}: only present in {run_id_b} ({len(rows_b)} exchange(s))")
|
|
1077
|
+
any_diff = True
|
|
1078
|
+
continue
|
|
1079
|
+
if not rows_b:
|
|
1080
|
+
print(f"\n{url}: only present in {run_id_a} ({len(rows_a)} exchange(s))")
|
|
1081
|
+
any_diff = True
|
|
1082
|
+
continue
|
|
1083
|
+
|
|
1084
|
+
for i in range(max(len(rows_a), len(rows_b))):
|
|
1085
|
+
row_a = rows_a[i] if i < len(rows_a) else None
|
|
1086
|
+
row_b = rows_b[i] if i < len(rows_b) else None
|
|
1087
|
+
if row_a is None or row_b is None:
|
|
1088
|
+
print(f"\n{url} [{i}]: exchange count differs between runs")
|
|
1089
|
+
any_diff = True
|
|
1090
|
+
continue
|
|
1091
|
+
for field in ("request_body", "response_body"):
|
|
1092
|
+
text_a = str(row_a.get(field, ""))
|
|
1093
|
+
text_b = str(row_b.get(field, ""))
|
|
1094
|
+
if text_a == text_b:
|
|
1095
|
+
continue
|
|
1096
|
+
any_diff = True
|
|
1097
|
+
print(f"\n{url} [{i}] {field}:")
|
|
1098
|
+
for line in _diff_text(f"{run_id_a}", text_a, f"{run_id_b}", text_b):
|
|
1099
|
+
print(f" {line}")
|
|
1100
|
+
|
|
1101
|
+
if not any_diff:
|
|
1102
|
+
print("\nNo differences found — every matched exchange is byte-identical.")
|
|
1103
|
+
|
|
1104
|
+
|
|
1105
|
+
def cmd_diff(args: argparse.Namespace) -> None:
|
|
1106
|
+
"""Print a structured diff of two recorded runs' exchanges, matched by
|
|
1107
|
+
URL, highlighting field-level differences between request/response
|
|
1108
|
+
bodies."""
|
|
1109
|
+
from agent_trace import _inspect as ins
|
|
1110
|
+
|
|
1111
|
+
run_id_a: str = args.run_id_a
|
|
1112
|
+
run_id_b: str = args.run_id_b
|
|
1113
|
+
_require_run_dir(run_id_a)
|
|
1114
|
+
_require_run_dir(run_id_b)
|
|
1115
|
+
|
|
1116
|
+
exchanges_a, spans_a = _load_run_exchanges_and_spans(run_id_a)
|
|
1117
|
+
exchanges_b, spans_b = _load_run_exchanges_and_spans(run_id_b)
|
|
1118
|
+
|
|
1119
|
+
print(f"Diffing {run_id_a} ({len(exchanges_a)} exchanges) vs "
|
|
1120
|
+
f"{run_id_b} ({len(exchanges_b)} exchanges)")
|
|
1121
|
+
|
|
1122
|
+
_print_exchange_diff(exchanges_a, exchanges_b, run_id_a, run_id_b)
|
|
1123
|
+
|
|
1124
|
+
# Restart-vs-resume: does run_b's root chain span for a shared
|
|
1125
|
+
# LangGraph thread_id continue from run_a's last recorded
|
|
1126
|
+
# langgraph_step, or did the graph start over from scratch (#161)?
|
|
1127
|
+
_print_flags(
|
|
1128
|
+
"restart_vs_resume",
|
|
1129
|
+
ins.check_restart_vs_resume(spans_a, spans_b),
|
|
1130
|
+
)
|
|
1131
|
+
|
|
1132
|
+
|
|
1133
|
+
# ---------------------------------------------------------------------------
|
|
1134
|
+
# Subcommand: run
|
|
1135
|
+
# ---------------------------------------------------------------------------
|
|
1136
|
+
#
|
|
1137
|
+
# Wraps an arbitrary child process (e.g. `langgraph dev`) with recording
|
|
1138
|
+
# pre-enabled process-wide, via the AGENT_TRACE_AUTO_RECORD env var
|
|
1139
|
+
# (Tracer.start_auto_record() / agent_trace._activate_auto_record_from_env())
|
|
1140
|
+
# — the supported mechanism for capturing a framework-managed dev server
|
|
1141
|
+
# process whose entrypoint the developer doesn't own, and doesn't want to
|
|
1142
|
+
# hand-instrument with a `with tracer.start_trace(...)` block.
|
|
1143
|
+
# ---------------------------------------------------------------------------
|
|
1144
|
+
|
|
1145
|
+
|
|
1146
|
+
def _strip_leading_separator(command: list[str]) -> list[str]:
|
|
1147
|
+
"""`agent-trace run -- langgraph dev` — argparse.REMAINDER captures the
|
|
1148
|
+
`--` separator verbatim as the first token of the remainder. Strip it
|
|
1149
|
+
(when present) so the child process is exec'd with the real command
|
|
1150
|
+
only. A bare `agent-trace run langgraph dev` (no `--`) works too — there
|
|
1151
|
+
is simply nothing to strip in that case."""
|
|
1152
|
+
if command and command[0] == "--":
|
|
1153
|
+
return command[1:]
|
|
1154
|
+
return command
|
|
1155
|
+
|
|
1156
|
+
|
|
1157
|
+
def cmd_run(args: argparse.Namespace) -> None:
|
|
1158
|
+
"""Exec a child process with recording pre-enabled process-wide.
|
|
1159
|
+
|
|
1160
|
+
Sets ``AGENT_TRACE_AUTO_RECORD=1`` (plus ``AGENT_TRACE_RUN_ID``/
|
|
1161
|
+
``AGENT_TRACE_AUTO_RECORD_NAME``/``AGENT_TRACE_TRACE_DIR``) in the
|
|
1162
|
+
child's environment before launching it, so the *first* `import
|
|
1163
|
+
agent_trace` anywhere inside that process — including one owned
|
|
1164
|
+
entirely by a third-party CLI like `langgraph dev` that the developer
|
|
1165
|
+
never touches — activates process-wide recording with zero code
|
|
1166
|
+
changes required in the developer's own graph/agent code. See
|
|
1167
|
+
``agent_trace.Tracer.start_auto_record`` and
|
|
1168
|
+
``agent_trace._activate_auto_record_from_env`` for the activation path
|
|
1169
|
+
this env var feeds into.
|
|
1170
|
+
|
|
1171
|
+
Exits with the child process's own exit code.
|
|
1172
|
+
"""
|
|
1173
|
+
command = _strip_leading_separator(args.child_command)
|
|
1174
|
+
if not command:
|
|
1175
|
+
sys.exit(
|
|
1176
|
+
"error: no command given.\n"
|
|
1177
|
+
"Usage: agent-trace run -- <command> [args...]\n"
|
|
1178
|
+
"Example: agent-trace run -- langgraph dev"
|
|
1179
|
+
)
|
|
1180
|
+
|
|
1181
|
+
run_id = args.run_id or f"run_{uuid.uuid4().hex[:12]}"
|
|
1182
|
+
trace_dir = _trace_dir()
|
|
1183
|
+
|
|
1184
|
+
env = os.environ.copy()
|
|
1185
|
+
env["AGENT_TRACE_AUTO_RECORD"] = "1"
|
|
1186
|
+
env["AGENT_TRACE_RUN_ID"] = run_id
|
|
1187
|
+
env["AGENT_TRACE_AUTO_RECORD_NAME"] = args.name
|
|
1188
|
+
env["AGENT_TRACE_TRACE_DIR"] = str(trace_dir)
|
|
1189
|
+
|
|
1190
|
+
print("agent-trace: recording enabled (AGENT_TRACE_AUTO_RECORD=1)")
|
|
1191
|
+
print(f"agent-trace: run_id: {run_id}")
|
|
1192
|
+
print(f"agent-trace: trace_dir: {trace_dir}")
|
|
1193
|
+
print(f"agent-trace: command: {' '.join(command)}")
|
|
1194
|
+
print()
|
|
1195
|
+
|
|
1196
|
+
try:
|
|
1197
|
+
result = subprocess.run(command, env=env, check=False) # noqa: S603
|
|
1198
|
+
except FileNotFoundError as exc:
|
|
1199
|
+
sys.exit(f"error: could not exec {command[0]!r}: {exc}")
|
|
1200
|
+
except KeyboardInterrupt:
|
|
1201
|
+
sys.exit(130)
|
|
1202
|
+
|
|
1203
|
+
print()
|
|
1204
|
+
print(f"agent-trace: child process exited with code {result.returncode}")
|
|
1205
|
+
print(f"agent-trace: inspect this run with: agent-trace show {run_id}")
|
|
1206
|
+
sys.exit(result.returncode)
|
|
1207
|
+
|
|
1208
|
+
|
|
1209
|
+
# ---------------------------------------------------------------------------
|
|
1210
|
+
# Entry point
|
|
1211
|
+
# ---------------------------------------------------------------------------
|
|
1212
|
+
|
|
1213
|
+
|
|
1214
|
+
def main() -> None:
|
|
1215
|
+
parser = argparse.ArgumentParser(
|
|
1216
|
+
prog="agent-trace",
|
|
1217
|
+
description="agent-trace — AI agent observability with record/replay",
|
|
1218
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
1219
|
+
epilog=(
|
|
1220
|
+
"Examples:\n"
|
|
1221
|
+
" agent-trace list\n"
|
|
1222
|
+
" agent-trace show run_abc123def456\n"
|
|
1223
|
+
" agent-trace show run_abc123def456 --errors-only\n"
|
|
1224
|
+
" agent-trace replay run_abc123def456\n"
|
|
1225
|
+
" agent-trace inspect run_abc123def456\n"
|
|
1226
|
+
" agent-trace diff run_a run_b\n"
|
|
1227
|
+
" agent-trace run -- langgraph dev\n"
|
|
1228
|
+
" agent-trace version\n"
|
|
1229
|
+
),
|
|
1230
|
+
)
|
|
1231
|
+
|
|
1232
|
+
sub = parser.add_subparsers(dest="command", metavar="COMMAND")
|
|
1233
|
+
sub.required = True
|
|
1234
|
+
|
|
1235
|
+
# version
|
|
1236
|
+
sub.add_parser("version", help="Print version and exit")
|
|
1237
|
+
|
|
1238
|
+
# list
|
|
1239
|
+
sub.add_parser("list", help="List all recorded runs")
|
|
1240
|
+
|
|
1241
|
+
# show
|
|
1242
|
+
show_p = sub.add_parser("show", help="Pretty-print trace.json for a run")
|
|
1243
|
+
show_p.add_argument("run_id", help="Run ID (e.g. run_abc123def456)")
|
|
1244
|
+
show_p.add_argument(
|
|
1245
|
+
"--errors-only",
|
|
1246
|
+
dest="errors_only",
|
|
1247
|
+
action="store_true",
|
|
1248
|
+
help="Only print ERROR-status spans with their captured exception text",
|
|
1249
|
+
)
|
|
1250
|
+
|
|
1251
|
+
# replay
|
|
1252
|
+
replay_p = sub.add_parser(
|
|
1253
|
+
"replay",
|
|
1254
|
+
help="Enter replay mode for a run and print the span tree",
|
|
1255
|
+
)
|
|
1256
|
+
replay_p.add_argument("run_id", help="Run ID (e.g. run_abc123def456)")
|
|
1257
|
+
|
|
1258
|
+
# inspect
|
|
1259
|
+
inspect_p = sub.add_parser(
|
|
1260
|
+
"inspect",
|
|
1261
|
+
help="Auto-flag/search raw request-response bodies for known malformed shapes",
|
|
1262
|
+
)
|
|
1263
|
+
inspect_p.add_argument("run_id", help="Run ID (e.g. run_abc123def456)")
|
|
1264
|
+
inspect_p.add_argument(
|
|
1265
|
+
"--registered-tools",
|
|
1266
|
+
dest="registered_tools",
|
|
1267
|
+
default=None,
|
|
1268
|
+
help="Comma-separated list of registered tool names, enables tool-call "
|
|
1269
|
+
"name fuzzy-match/dotted-compound/ReAct-action-name checks",
|
|
1270
|
+
)
|
|
1271
|
+
inspect_p.add_argument(
|
|
1272
|
+
"--configured-host",
|
|
1273
|
+
dest="configured_host",
|
|
1274
|
+
default=None,
|
|
1275
|
+
help="Framework's configured LLM endpoint host, enables the "
|
|
1276
|
+
"endpoint-host-mismatch check",
|
|
1277
|
+
)
|
|
1278
|
+
inspect_p.add_argument(
|
|
1279
|
+
"--check-kwarg",
|
|
1280
|
+
dest="check_kwarg",
|
|
1281
|
+
default=None,
|
|
1282
|
+
help="Dotted kwarg path (e.g. extra_body.chat_template_kwargs.thinking) "
|
|
1283
|
+
"expected to be present on the wire; flags requests where it's absent",
|
|
1284
|
+
)
|
|
1285
|
+
inspect_p.add_argument(
|
|
1286
|
+
"--diff-field",
|
|
1287
|
+
dest="diff_field",
|
|
1288
|
+
default=None,
|
|
1289
|
+
help="Response field to check for wire-present-but-downstream-"
|
|
1290
|
+
"absent — a top-level key (e.g. usage) or a dotted/nested path "
|
|
1291
|
+
"with numeric list-index segments (e.g. "
|
|
1292
|
+
"choices.0.message.reasoning_content, for provider fields nested "
|
|
1293
|
+
"inside the response body like DeepSeek's reasoning_content)",
|
|
1294
|
+
)
|
|
1295
|
+
inspect_p.add_argument(
|
|
1296
|
+
"--diff-get-post-field",
|
|
1297
|
+
dest="diff_get_post_field",
|
|
1298
|
+
default=None,
|
|
1299
|
+
help="Dotted field path (e.g. instructions) to compare between an "
|
|
1300
|
+
"earlier GET response and a later, causally-related POST request "
|
|
1301
|
+
"body referencing the same resource id (#2620); flags stale-value "
|
|
1302
|
+
"mismatches such as a GPTAssistantAgent POST /runs still sending "
|
|
1303
|
+
"instructions that no longer match what GET /assistants/{id} "
|
|
1304
|
+
"returns",
|
|
1305
|
+
)
|
|
1306
|
+
inspect_p.add_argument(
|
|
1307
|
+
"--diff-get-post-id-field",
|
|
1308
|
+
dest="diff_get_post_id_field",
|
|
1309
|
+
default="id",
|
|
1310
|
+
help="Field name the GET response uses for the resource id "
|
|
1311
|
+
"(default: id)",
|
|
1312
|
+
)
|
|
1313
|
+
inspect_p.add_argument(
|
|
1314
|
+
"--diff-get-post-post-id-field",
|
|
1315
|
+
dest="diff_get_post_post_id_field",
|
|
1316
|
+
default=None,
|
|
1317
|
+
help="Field name the POST request body uses to reference the same "
|
|
1318
|
+
"resource id, if different from --diff-get-post-id-field (e.g. "
|
|
1319
|
+
"assistant_id)",
|
|
1320
|
+
)
|
|
1321
|
+
|
|
1322
|
+
# diff
|
|
1323
|
+
diff_p = sub.add_parser(
|
|
1324
|
+
"diff",
|
|
1325
|
+
help="Diff two recorded runs' exchanges (matched by URL)",
|
|
1326
|
+
)
|
|
1327
|
+
diff_p.add_argument("run_id_a", help="First run ID")
|
|
1328
|
+
diff_p.add_argument("run_id_b", help="Second run ID")
|
|
1329
|
+
|
|
1330
|
+
# run
|
|
1331
|
+
run_p = sub.add_parser(
|
|
1332
|
+
"run",
|
|
1333
|
+
help=(
|
|
1334
|
+
"Exec a child process with recording pre-enabled process-wide "
|
|
1335
|
+
"(e.g. agent-trace run -- langgraph dev)"
|
|
1336
|
+
),
|
|
1337
|
+
)
|
|
1338
|
+
run_p.add_argument(
|
|
1339
|
+
"--run-id",
|
|
1340
|
+
dest="run_id",
|
|
1341
|
+
default=None,
|
|
1342
|
+
help="Explicit run ID (default: random, printed on start)",
|
|
1343
|
+
)
|
|
1344
|
+
run_p.add_argument(
|
|
1345
|
+
"--name",
|
|
1346
|
+
dest="name",
|
|
1347
|
+
default="auto-record",
|
|
1348
|
+
help="Trace name recorded in trace.json metadata (default: auto-record)",
|
|
1349
|
+
)
|
|
1350
|
+
run_p.add_argument(
|
|
1351
|
+
"child_command",
|
|
1352
|
+
nargs=argparse.REMAINDER,
|
|
1353
|
+
help="Command to exec, e.g. -- langgraph dev",
|
|
1354
|
+
)
|
|
1355
|
+
|
|
1356
|
+
args = parser.parse_args()
|
|
1357
|
+
|
|
1358
|
+
dispatch = {
|
|
1359
|
+
"version": cmd_version,
|
|
1360
|
+
"list": cmd_list,
|
|
1361
|
+
"show": cmd_show,
|
|
1362
|
+
"replay": cmd_replay,
|
|
1363
|
+
"inspect": cmd_inspect,
|
|
1364
|
+
"diff": cmd_diff,
|
|
1365
|
+
"run": cmd_run,
|
|
1366
|
+
}
|
|
1367
|
+
|
|
1368
|
+
handler = dispatch.get(args.command)
|
|
1369
|
+
if handler is None:
|
|
1370
|
+
parser.print_help()
|
|
1371
|
+
sys.exit(1)
|
|
1372
|
+
|
|
1373
|
+
handler(args)
|
|
1374
|
+
|
|
1375
|
+
|
|
1376
|
+
if __name__ == "__main__":
|
|
1377
|
+
main()
|