agent-observability-trace-cli 0.1.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. agent_observability_trace_cli-0.1.2.dist-info/METADATA +336 -0
  2. agent_observability_trace_cli-0.1.2.dist-info/RECORD +51 -0
  3. agent_observability_trace_cli-0.1.2.dist-info/WHEEL +4 -0
  4. agent_observability_trace_cli-0.1.2.dist-info/entry_points.txt +2 -0
  5. agent_observability_trace_cli-0.1.2.dist-info/licenses/LICENSE +198 -0
  6. agent_trace/__init__.py +1182 -0
  7. agent_trace/_cli.py +1377 -0
  8. agent_trace/_inspect.py +2020 -0
  9. agent_trace/_replay/__init__.py +1 -0
  10. agent_trace/_replay/engine.py +268 -0
  11. agent_trace/_replay/fixture.py +868 -0
  12. agent_trace/core/__init__.py +1 -0
  13. agent_trace/core/clock.py +91 -0
  14. agent_trace/core/exceptions.py +51 -0
  15. agent_trace/core/span.py +271 -0
  16. agent_trace/core/trace.py +92 -0
  17. agent_trace/exporters/__init__.py +1 -0
  18. agent_trace/exporters/file.py +80 -0
  19. agent_trace/exporters/otlp.py +199 -0
  20. agent_trace/exporters/remote_fixture.py +307 -0
  21. agent_trace/exporters/stdout.py +258 -0
  22. agent_trace/integrations/__init__.py +69 -0
  23. agent_trace/integrations/agno.py +489 -0
  24. agent_trace/integrations/autogen.py +581 -0
  25. agent_trace/integrations/crewai.py +486 -0
  26. agent_trace/integrations/google_genai.py +731 -0
  27. agent_trace/integrations/haystack.py +254 -0
  28. agent_trace/integrations/langchain_core.py +361 -0
  29. agent_trace/integrations/langgraph.py +2093 -0
  30. agent_trace/integrations/langgraph_checkpoint.py +763 -0
  31. agent_trace/integrations/langgraph_state_diff.py +345 -0
  32. agent_trace/integrations/langgraph_stream_debug.py +202 -0
  33. agent_trace/integrations/llama_index.py +472 -0
  34. agent_trace/integrations/mcp.py +281 -0
  35. agent_trace/integrations/openai_agents.py +811 -0
  36. agent_trace/integrations/pydantic_ai.py +504 -0
  37. agent_trace/integrations/streaming.py +218 -0
  38. agent_trace/interceptor/__init__.py +64 -0
  39. agent_trace/interceptor/aiohttp_hook.py +152 -0
  40. agent_trace/interceptor/botocore_hook.py +223 -0
  41. agent_trace/interceptor/grpc_hook.py +651 -0
  42. agent_trace/interceptor/httpx_hook.py +866 -0
  43. agent_trace/interceptor/logging_hook.py +137 -0
  44. agent_trace/interceptor/requests_patch.py +184 -0
  45. agent_trace/interceptor/sse.py +176 -0
  46. agent_trace/interceptor/stdio_hook.py +245 -0
  47. agent_trace/interceptor/warnings_hook.py +132 -0
  48. agent_trace/interceptor/websocket_hook.py +323 -0
  49. agent_trace/plugins/__init__.py +35 -0
  50. agent_trace/plugins/base.py +111 -0
  51. agent_trace/py.typed +0 -0
agent_trace/_cli.py ADDED
@@ -0,0 +1,1377 @@
1
+ """
2
+ Command-line interface for agent-trace.
3
+
4
+ Commands:
5
+ agent-trace replay <run_id> — replay a recorded run and print span tree
6
+ agent-trace list — list all recorded runs in trace_dir
7
+ agent-trace show <run_id> — show the stored trace.json for a run
8
+ agent-trace inspect <run_id> — auto-flag anomalies in captured bodies
9
+ agent-trace diff <run_id_a> <b> — diff two recorded runs' exchanges
10
+ agent-trace run -- <command> — exec a child process, recording pre-enabled
11
+ agent-trace version — print version
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import argparse
17
+ import datetime
18
+ import difflib
19
+ import json
20
+ import os
21
+ import subprocess
22
+ import sys
23
+ import uuid
24
+ from pathlib import Path
25
+ from typing import Any
26
+
27
+ __all__ = ["main"]
28
+
29
+ _VERSION = "0.1.0"
30
+ _DEFAULT_TRACE_DIR = Path.home() / ".agent-trace" / "runs"
31
+
32
+
33
+ # ---------------------------------------------------------------------------
34
+ # Helpers
35
+ # ---------------------------------------------------------------------------
36
+
37
+
38
+ def _trace_dir() -> Path:
39
+ """Return the active trace directory, honouring the env override."""
40
+ env_dir = os.environ.get("AGENT_TRACE_TRACE_DIR")
41
+ if env_dir:
42
+ return Path(env_dir)
43
+ return _DEFAULT_TRACE_DIR
44
+
45
+
46
+ def _run_dir(run_id: str) -> Path:
47
+ base = _trace_dir().resolve()
48
+ candidate = (base / run_id).resolve()
49
+ try:
50
+ candidate.relative_to(base)
51
+ except ValueError:
52
+ raise ValueError(
53
+ f"Invalid run_id {run_id!r}: path traversal detected"
54
+ ) from None
55
+ return candidate
56
+
57
+
58
+ def _fixture_path(run_id: str) -> Path:
59
+ return _run_dir(run_id) / "fixture.db"
60
+
61
+
62
+ def _trace_json_path(run_id: str) -> Path:
63
+ return _run_dir(run_id) / "trace.json"
64
+
65
+
66
+ def _require_run_dir(run_id: str) -> Path:
67
+ """Return the run directory or exit with a clear message."""
68
+ d = _run_dir(run_id)
69
+ if not d.exists():
70
+ sys.exit(
71
+ f"error: no run directory found for {run_id!r}\n"
72
+ f"Expected: {d}\n"
73
+ "Run 'agent-trace list' to see all recorded runs."
74
+ )
75
+ return d
76
+
77
+
78
+ # ---------------------------------------------------------------------------
79
+ # Subcommand: version
80
+ # ---------------------------------------------------------------------------
81
+
82
+
83
+ def cmd_version(_args: argparse.Namespace) -> None:
84
+ print(f"agent-trace {_VERSION}")
85
+
86
+
87
+ # ---------------------------------------------------------------------------
88
+ # Subcommand: list
89
+ # ---------------------------------------------------------------------------
90
+
91
+
92
+ def cmd_list(_args: argparse.Namespace) -> None:
93
+ """List all recorded runs in the trace directory."""
94
+ trace_dir = _trace_dir()
95
+ if not trace_dir.exists():
96
+ print(f"No runs yet. Trace directory does not exist: {trace_dir}")
97
+ return
98
+
99
+ run_dirs = sorted(
100
+ (d for d in trace_dir.iterdir() if d.is_dir()),
101
+ key=lambda d: d.stat().st_mtime,
102
+ reverse=True,
103
+ )
104
+
105
+ if not run_dirs:
106
+ print(f"No recorded runs in {trace_dir}")
107
+ return
108
+
109
+ # Header
110
+ print(f"{'RUN ID':<30} {'EXCHANGES':>9} {'SPANS':>6} RECORDED AT")
111
+ print("-" * 70)
112
+
113
+ for run_dir in run_dirs:
114
+ run_id = run_dir.name
115
+ exchanges = _count_exchanges(run_dir)
116
+ spans = _count_spans(run_dir)
117
+ mtime = _format_mtime(run_dir.stat().st_mtime)
118
+ print(f"{run_id:<30} {exchanges:>9} {spans:>6} {mtime}")
119
+
120
+ print()
121
+ print(f"Total: {len(run_dirs)} run(s) in {trace_dir}")
122
+
123
+
124
+ def _count_exchanges(run_dir: Path) -> str:
125
+ fixture_db = run_dir / "fixture.db"
126
+ if not fixture_db.exists():
127
+ return "-"
128
+ try:
129
+ from agent_trace._replay.fixture import Fixture
130
+
131
+ with Fixture(fixture_db) as f:
132
+ return str(f.exchange_count())
133
+ except Exception:
134
+ return "?"
135
+
136
+
137
+ def _count_spans(run_dir: Path) -> str:
138
+ trace_json = run_dir / "trace.json"
139
+ if not trace_json.exists():
140
+ return "-"
141
+ try:
142
+ data = json.loads(trace_json.read_text(encoding="utf-8"))
143
+ return str(len(data.get("spans", [])))
144
+ except Exception:
145
+ return "?"
146
+
147
+
148
+ def _format_mtime(ts: float) -> str:
149
+ dt = datetime.datetime.fromtimestamp(ts, tz=datetime.timezone.utc)
150
+ return dt.strftime("%Y-%m-%d %H:%M:%S UTC")
151
+
152
+
153
+ # ---------------------------------------------------------------------------
154
+ # Error classification summary — surfaces error.origin/error.known_pattern
155
+ # (set by the LangGraph integration's exception classifier) as a one-line
156
+ # pointer per ERROR span, instead of requiring a developer to read the raw
157
+ # trace.json blob and find those attributes by eye.
158
+ # ---------------------------------------------------------------------------
159
+
160
+
161
+ def _error_classification_rows(spans: list[dict[str, object]]) -> list[dict[str, str]]:
162
+ """Return one row per ERROR-status span with its origin/known-pattern
163
+ classification (empty string for a field that wasn't set)."""
164
+ rows: list[dict[str, str]] = []
165
+ for span in spans:
166
+ if span.get("status") != "ERROR":
167
+ continue
168
+ attrs = span.get("attributes") or {}
169
+ if not isinstance(attrs, dict):
170
+ continue
171
+ rows.append(
172
+ {
173
+ "name": str(span.get("name", "?")),
174
+ "origin": str(attrs.get("error.origin", "")),
175
+ "known_pattern": str(attrs.get("error.known_pattern", "")),
176
+ }
177
+ )
178
+ return rows
179
+
180
+
181
+ def _print_error_classification(spans: list[dict[str, object]]) -> None:
182
+ rows = _error_classification_rows(spans)
183
+ if not rows:
184
+ return
185
+ print()
186
+ print("Error classification:")
187
+ for row in rows:
188
+ origin = row["origin"] or "unclassified"
189
+ line = f" {row['name']:<30} origin={origin}"
190
+ if row["known_pattern"]:
191
+ line += f" known_pattern={row['known_pattern']}"
192
+ print(line)
193
+
194
+
195
+ # ---------------------------------------------------------------------------
196
+ # Duplicate node span detection — flags a node:<name> span that appears more
197
+ # than once within a single trace, the shape produced when a resumed
198
+ # graph.stream()/.invoke() call re-executes a task that should have reused a
199
+ # cached checkpointed write (see issue #6050). Heuristic, not a definitive
200
+ # bug detector: an intentionally looping node (e.g. a ReAct agent) also
201
+ # produces repeated node:<name> spans by design — this surfaces the count as
202
+ # a fact for a developer to interpret, rather than claiming a verdict.
203
+ # ---------------------------------------------------------------------------
204
+
205
+
206
+ def _duplicate_node_span_counts(spans: list[dict[str, object]]) -> dict[str, int]:
207
+ """Return {node span name: occurrence count} for every node:<name> span
208
+ name that appears more than once in *spans*."""
209
+ counts: dict[str, int] = {}
210
+ for span in spans:
211
+ name = span.get("name")
212
+ if isinstance(name, str) and name.startswith("node:"):
213
+ counts[name] = counts.get(name, 0) + 1
214
+ return {name: n for name, n in counts.items() if n > 1}
215
+
216
+
217
+ def _print_duplicate_node_spans(spans: list[dict[str, object]]) -> None:
218
+ duplicates = _duplicate_node_span_counts(spans)
219
+ if not duplicates:
220
+ return
221
+ print()
222
+ print("Duplicate node spans (same node:<name> executed more than once):")
223
+ for name, count in sorted(duplicates.items()):
224
+ print(f" {name:<30} executed {count} times")
225
+ print(
226
+ " (Expected for an intentionally looping node — e.g. a ReAct agent. "
227
+ "Unexpected for a task that should have reused a cached checkpointed "
228
+ "write across a resume — see issue #6050.)"
229
+ )
230
+
231
+
232
+ # ---------------------------------------------------------------------------
233
+ # Streaming timing diagnostic — surfaces time-to-first-chunk and max
234
+ # inter-chunk gap for any exchange recorded via a pass-through streaming
235
+ # transport (RecordingTransport(..., stream=True) /
236
+ # AsyncRecordingTransport(..., stream=True)). Exchanges recorded the default
237
+ # (eager-buffering) way carry no chunk_timestamps and are silently skipped —
238
+ # absence means "not captured this way", not "instant delivery".
239
+ # ---------------------------------------------------------------------------
240
+
241
+
242
+ def _streaming_timing_rows(
243
+ exchanges: list[dict[str, object]],
244
+ ) -> list[dict[str, object]]:
245
+ """One row per exchange with recorded per-chunk timestamps."""
246
+ from agent_trace._replay.fixture import (
247
+ max_inter_chunk_gap_ms,
248
+ time_to_first_chunk_ms,
249
+ )
250
+
251
+ rows: list[dict[str, object]] = []
252
+ for exchange in exchanges:
253
+ if not exchange.get("chunk_timestamps"):
254
+ continue
255
+ rows.append(
256
+ {
257
+ "url": exchange.get("url", "?"),
258
+ "method": exchange.get("method", "?"),
259
+ "chunk_count": len(exchange["chunk_timestamps"]), # type: ignore[arg-type]
260
+ "time_to_first_chunk_ms": time_to_first_chunk_ms(exchange),
261
+ "max_inter_chunk_gap_ms": max_inter_chunk_gap_ms(exchange),
262
+ }
263
+ )
264
+ return rows
265
+
266
+
267
+ def _print_streaming_timing(exchanges: list[dict[str, object]]) -> None:
268
+ rows = _streaming_timing_rows(exchanges)
269
+ if not rows:
270
+ return
271
+ print()
272
+ print("Streaming timing (time-to-first-chunk / max inter-chunk gap):")
273
+ for row in rows:
274
+ ttfc = row["time_to_first_chunk_ms"]
275
+ gap = row["max_inter_chunk_gap_ms"]
276
+ ttfc_str = f"{ttfc:.1f}ms" if isinstance(ttfc, (int, float)) else "?"
277
+ gap_str = f"{gap:.1f}ms" if isinstance(gap, (int, float)) else "?"
278
+ print(
279
+ f" {row['method']:<6} {row['url']:<50} "
280
+ f"chunks={row['chunk_count']:>4} "
281
+ f"first_chunk={ttfc_str:>10} max_gap={gap_str:>10}"
282
+ )
283
+
284
+
285
+ # ---------------------------------------------------------------------------
286
+ # HTTP error exchange summary — surfaces 4xx/5xx exchanges as flagged errors
287
+ # in `agent-trace replay` output instead of leaving them as undifferentiated
288
+ # raw rows indistinguishable from a normal 200 (a captured error response
289
+ # like Azure's content_filter 400 sits in the fixture with nothing in the
290
+ # CLI distinguishing it today).
291
+ # ---------------------------------------------------------------------------
292
+
293
+
294
+ def _print_http_error_exchanges(exchanges: list[dict[str, object]]) -> None:
295
+ from agent_trace import _inspect as ins
296
+
297
+ flags = ins.flag_4xx_5xx_exchanges(exchanges)
298
+ if not flags:
299
+ return
300
+ print()
301
+ print(f"HTTP error exchanges ({len(flags)}):")
302
+ for flag in flags:
303
+ print(f" {flag['method']:<6} {flag['url']:<50} HTTP {flag['status']}")
304
+
305
+
306
+ # ---------------------------------------------------------------------------
307
+ # Checkpoint durability diagnostic — correlates checkpoint:put/put_writes
308
+ # spans (recorded by TracingCheckpointSaver,
309
+ # agent_trace.integrations.langgraph_checkpoint) against the rest of the
310
+ # trace, surfacing a terminal durable|partial|abandoned status per the shape
311
+ # independently proposed in the #5672 thread (cancellation_requested,
312
+ # writes_enqueued_count, writes_flushed_count, checkpoint_status) instead of
313
+ # requiring a developer to manually diff two span sets by hand to answer
314
+ # "did what the user saw actually get durably persisted before the run
315
+ # ended".
316
+ # ---------------------------------------------------------------------------
317
+
318
+ _CHECKPOINT_WRITE_SPAN_NAMES = frozenset(
319
+ {
320
+ "checkpoint:put",
321
+ "checkpoint:aput",
322
+ "checkpoint:put_writes",
323
+ "checkpoint:aput_writes",
324
+ }
325
+ )
326
+
327
+
328
+ def _checkpoint_durability_summary(
329
+ spans: list[dict[str, object]],
330
+ ) -> dict[str, object] | None:
331
+ """Return the durability summary for *spans*, or None if no
332
+ checkpoint-write spans were recorded in this trace at all (the
333
+ TracingCheckpointSaver wrapper wasn't wired in — nothing to report,
334
+ distinct from "wired in but zero writes flushed")."""
335
+ write_spans = [s for s in spans if s.get("name") in _CHECKPOINT_WRITE_SPAN_NAMES]
336
+ if not write_spans:
337
+ return None
338
+
339
+ enqueued = len(write_spans)
340
+ flushed = 0
341
+ for span in write_spans:
342
+ attrs = span.get("attributes") or {}
343
+ if isinstance(attrs, dict) and attrs.get("checkpoint.completed") is True:
344
+ flushed += 1
345
+
346
+ cancellation_requested = any(s.get("status") == "CANCELLED" for s in spans)
347
+
348
+ if flushed == enqueued and not cancellation_requested:
349
+ status = "durable"
350
+ elif flushed == 0:
351
+ status = "abandoned"
352
+ else:
353
+ status = "partial"
354
+
355
+ return {
356
+ "cancellation_requested": cancellation_requested,
357
+ "writes_enqueued_count": enqueued,
358
+ "writes_flushed_count": flushed,
359
+ "checkpoint_status": status,
360
+ }
361
+
362
+
363
+ def _print_checkpoint_durability(spans: list[dict[str, object]]) -> None:
364
+ summary = _checkpoint_durability_summary(spans)
365
+ if summary is None:
366
+ return
367
+ print()
368
+ print("Checkpoint durability:")
369
+ print(f" checkpoint_status: {summary['checkpoint_status']}")
370
+ print(f" writes_enqueued_count: {summary['writes_enqueued_count']}")
371
+ print(f" writes_flushed_count: {summary['writes_flushed_count']}")
372
+ print(f" cancellation_requested: {summary['cancellation_requested']}")
373
+ if summary["checkpoint_status"] != "durable":
374
+ print(
375
+ " (Not every checkpoint write span closed checkpoint.completed=True "
376
+ "before the run ended — state observed during the run may not "
377
+ "match what was actually persisted. See issue #5672.)"
378
+ )
379
+
380
+
381
+ # ---------------------------------------------------------------------------
382
+ # Zero-tasks-scheduled diagnostic — flags a checkpoint:update_state span
383
+ # (recorded by traced_update_state()/traced_aupdate_state(),
384
+ # agent_trace.integrations.langgraph_checkpoint) whose post-write task
385
+ # schedule came back empty, the exact silent-no-op-resume shape behind issue
386
+ # #4217 (a missing/incorrect as_node on an external state write).
387
+ # ---------------------------------------------------------------------------
388
+
389
+
390
+ def _zero_task_update_rows(spans: list[dict[str, object]]) -> list[dict[str, object]]:
391
+ rows: list[dict[str, object]] = []
392
+ for span in spans:
393
+ if span.get("name") != "checkpoint:update_state":
394
+ continue
395
+ attrs = span.get("attributes") or {}
396
+ if not isinstance(attrs, dict):
397
+ continue
398
+ if attrs.get("checkpoint.zero_tasks_scheduled") is True:
399
+ rows.append(
400
+ {
401
+ "as_node": attrs.get("checkpoint.as_node", "<not provided>"),
402
+ "as_node_provided": attrs.get("checkpoint.as_node_provided", False),
403
+ }
404
+ )
405
+ return rows
406
+
407
+
408
+ def _print_zero_task_updates(spans: list[dict[str, object]]) -> None:
409
+ rows = _zero_task_update_rows(spans)
410
+ if not rows:
411
+ return
412
+ print()
413
+ print("Zero tasks scheduled after a state update (likely misattributed write):")
414
+ for row in rows:
415
+ provided = "yes" if row["as_node_provided"] else "no"
416
+ print(f" as_node={row['as_node']} (explicitly provided: {provided})")
417
+ print(
418
+ " (An update_state() call was followed by an empty scheduled-task "
419
+ "list — the graph will not advance from this state. Usually a "
420
+ "missing/incorrect as_node. See issue #4217.)"
421
+ )
422
+
423
+
424
+ # ---------------------------------------------------------------------------
425
+ # Exception-text surfacing — pulls the exception.message off any ERROR-status
426
+ # span (already captured by Span.record_exception) into a print-ready view,
427
+ # for both `agent-trace show --errors-only` and the plain-text summary block
428
+ # both `show`/`replay` print after their main output.
429
+ # ---------------------------------------------------------------------------
430
+
431
+
432
+ def _span_exception_message(span: dict[str, object]) -> str | None:
433
+ """Return the exception.message text for *span*, or None if it has no
434
+ recorded exception event."""
435
+ events = span.get("events")
436
+ if not isinstance(events, list):
437
+ return None
438
+ for event in events:
439
+ if not isinstance(event, dict) or event.get("name") != "exception":
440
+ continue
441
+ attrs = event.get("attributes") or {}
442
+ if isinstance(attrs, dict) and "exception.message" in attrs:
443
+ return str(attrs["exception.message"])
444
+ return None
445
+
446
+
447
+ def _span_exception_http_detail(span: dict[str, object]) -> str | None:
448
+ """Return "HTTP <status>: <body preview>" for *span*'s recorded
449
+ exception event when it carried an HTTP error response body (see
450
+ Span.record_exception's exception.http_response_body/
451
+ exception.http_status_code attributes, core/span.py) — the actual
452
+ provider/proxy error text (#4940), not just the generic one-line
453
+ str(exc) message which is all `exception.message` alone gives you for
454
+ e.g. requests.exceptions.HTTPError."""
455
+ events = span.get("events")
456
+ if not isinstance(events, list):
457
+ return None
458
+ for event in events:
459
+ if not isinstance(event, dict) or event.get("name") != "exception":
460
+ continue
461
+ attrs = event.get("attributes") or {}
462
+ if not isinstance(attrs, dict):
463
+ continue
464
+ body = attrs.get("exception.http_response_body")
465
+ if not body:
466
+ continue
467
+ status = attrs.get("exception.http_status_code")
468
+ prefix = f"HTTP {status}: " if status is not None else "HTTP: "
469
+ return f"{prefix}{body}"
470
+ return None
471
+
472
+
473
+ def _error_spans(spans: list[dict[str, object]]) -> list[dict[str, object]]:
474
+ return [s for s in spans if s.get("status") == "ERROR"]
475
+
476
+
477
+ def _print_errors_only(spans: list[dict[str, object]]) -> None:
478
+ """`agent-trace show --errors-only` — filter to ERROR-status spans and
479
+ print each with its captured exception text inline, instead of dumping
480
+ the full trace.json and requiring a manual grep for
481
+ "exception.stacktrace"."""
482
+ errors = _error_spans(spans)
483
+ print(f"Error spans: {len(errors)} of {len(spans)} total")
484
+ if not errors:
485
+ return
486
+ print()
487
+ for span in errors:
488
+ print(f"[ERR] {span.get('name', '?')}")
489
+ message = _span_exception_message(span)
490
+ if message:
491
+ print(f" {message}")
492
+ http_detail = _span_exception_http_detail(span)
493
+ if http_detail:
494
+ print(f" {http_detail}")
495
+ _print_error_classification(spans)
496
+
497
+
498
+ # ---------------------------------------------------------------------------
499
+ # Retry-storm detection — flags a single node:* span with more than one
500
+ # llm:* child span, the shape produced by application code that silently
501
+ # re-invokes the LLM in a loop when a response is empty/malformed (#2920):
502
+ # an invisible cause of "sometimes fast, sometimes very slow" with no error
503
+ # and no user-facing signal.
504
+ # ---------------------------------------------------------------------------
505
+
506
+
507
+ def _retry_storm_rows(spans: list[dict[str, object]]) -> list[dict[str, object]]:
508
+ children_by_parent: dict[str | None, list[dict[str, object]]] = {}
509
+ for span in spans:
510
+ parent_id = span.get("parent_id")
511
+ parent_key = parent_id if isinstance(parent_id, str) else None
512
+ children_by_parent.setdefault(parent_key, []).append(span)
513
+
514
+ rows: list[dict[str, object]] = []
515
+ for span in spans:
516
+ name = span.get("name")
517
+ if not isinstance(name, str) or not name.startswith("node:"):
518
+ continue
519
+ span_id = span.get("span_id")
520
+ children = (
521
+ children_by_parent.get(span_id, []) if isinstance(span_id, str) else []
522
+ )
523
+ llm_children = [
524
+ c for c in children if str(c.get("name", "")).startswith("llm:")
525
+ ]
526
+ if len(llm_children) > 1:
527
+ rows.append({"node": name, "llm_child_count": len(llm_children)})
528
+ return rows
529
+
530
+
531
+ def _print_retry_storms(spans: list[dict[str, object]]) -> None:
532
+ rows = _retry_storm_rows(spans)
533
+ if not rows:
534
+ return
535
+ print()
536
+ print("Repeated LLM calls under one node span (possible retry storm):")
537
+ for row in rows:
538
+ print(f" {row['node']:<30} {row['llm_child_count']} llm:* child spans")
539
+ print(
540
+ " (A node re-invoking the LLM multiple times per call produces no "
541
+ "error and no user-facing signal, but explains 'sometimes fast, "
542
+ "sometimes very slow' behavior. See issue #2920.)"
543
+ )
544
+
545
+
546
+ # ---------------------------------------------------------------------------
547
+ # Orphaned/misattributed span detection — flags a span whose callback-derived
548
+ # parent_id looks suspicious: a root span (parent_id is None) that is not the
549
+ # trace's earliest root and whose start_time falls inside another span's
550
+ # active [start, end) window. This is the exact visible symptom of
551
+ # langgraph#3975 (ChatOpenAI calls flattened to the trace root instead of
552
+ # nested under their originating node when a compiled graph is piped into a
553
+ # raw lambda) — a reconciliation pass against the trace's own chronological
554
+ # span ordering (the callback-independent signal LangGraphTracer's
555
+ # parent_run_id-registry lookup has no visibility into) surfaces it
556
+ # automatically instead of requiring a developer to eyeball trace.json.
557
+ # ---------------------------------------------------------------------------
558
+
559
+
560
+ def _numeric_start_time(span: dict[str, object]) -> float | None:
561
+ start = span.get("start_time")
562
+ return float(start) if isinstance(start, (int, float)) else None
563
+
564
+
565
+ def _http_sequence_confirms(
566
+ suspect: dict[str, object],
567
+ likely_parent: dict[str, object],
568
+ exchanges: list[dict[str, object]],
569
+ ) -> bool | None:
570
+ """Cross-check a *suspect*/*likely_parent* guess — built purely from
571
+ wall-clock span `start_time`/`end_time` — against the independent,
572
+ always-correct total ordering already captured in fixture.db's
573
+ `sequence_num` column (see `_replay/fixture.py`'s module docstring: "a
574
+ monotonically increasing sequence_num" assigned once per HTTP round-trip
575
+ at record time, immune to the callback-registry misses that produce
576
+ misattributed spans in the first place). This is the reconciliation
577
+ pass the "unaccounted-for root span" heuristic above needs: two spans'
578
+ wall-clock timestamps can only be trusted so far, but the HTTP capture
579
+ layer's sequence_num ordering is a second, structurally independent
580
+ signal for the same underlying chronology.
581
+
582
+ Returns True when at least one HTTP exchange was recorded during
583
+ *likely_parent*'s activity strictly before *suspect* started (i.e. the
584
+ HTTP layer independently confirms *likely_parent* was already doing
585
+ work before *suspect* began — corroborating the timestamp-based guess),
586
+ False when every such exchange has a `sequence_num` *greater than* an
587
+ exchange recorded during *suspect*'s own window (the HTTP layer
588
+ contradicts the guess), or None when there isn't enough HTTP-layer data
589
+ on either side to judge (note *likely_parent*'s window necessarily
590
+ contains *suspect*'s, since that's how it was chosen — so only the
591
+ portion of *likely_parent*'s window strictly before *suspect* started is
592
+ used, otherwise every exchange inside *suspect*'s own window would also
593
+ trivially count as "inside likely_parent's window" too).
594
+ """
595
+
596
+ def _sequence_nums_in(start: float, end_bound: float) -> list[int]:
597
+ seqs: list[int] = []
598
+ for exchange in exchanges:
599
+ recorded_at = exchange.get("recorded_at")
600
+ seq = exchange.get("sequence_num")
601
+ if not isinstance(recorded_at, (int, float)) or not isinstance(
602
+ seq, int
603
+ ):
604
+ continue
605
+ if start <= recorded_at < end_bound:
606
+ seqs.append(seq)
607
+ return seqs
608
+
609
+ parent_start = _numeric_start_time(likely_parent)
610
+ suspect_start = _numeric_start_time(suspect)
611
+ if parent_start is None or suspect_start is None:
612
+ return None
613
+
614
+ suspect_end = suspect.get("end_time")
615
+ suspect_end_bound = (
616
+ suspect_end if isinstance(suspect_end, (int, float)) else float("inf")
617
+ )
618
+
619
+ parent_exclusive_seqs = _sequence_nums_in(parent_start, suspect_start)
620
+ suspect_seqs = _sequence_nums_in(suspect_start, suspect_end_bound)
621
+ if not parent_exclusive_seqs or not suspect_seqs:
622
+ return None
623
+ return max(parent_exclusive_seqs) < min(suspect_seqs)
624
+
625
+
626
+ def _misattributed_span_rows(
627
+ spans: list[dict[str, object]],
628
+ exchanges: list[dict[str, object]] | None = None,
629
+ ) -> list[dict[str, object]]:
630
+ rootlike = [s for s in spans if s.get("parent_id") is None]
631
+ timed_roots = [
632
+ (s, t) for s in rootlike if (t := _numeric_start_time(s)) is not None
633
+ ]
634
+ roots = [s for s, _t in sorted(timed_roots, key=lambda pair: pair[1])]
635
+ if len(roots) <= 1:
636
+ return []
637
+
638
+ suspicious_root_ids = {id(s) for s in roots[1:]}
639
+ suspicious_roots = roots[1:]
640
+
641
+ # Spans that could plausibly be the "real" parent: anything open (no
642
+ # end_time, or start <= suspect.start < end) at the moment the
643
+ # suspicious root started. Includes the trace's earliest/genuine root —
644
+ # the langgraph#3975 shape is exactly "this call should have nested
645
+ # under the run's actual root/node span but didn't". Excludes other
646
+ # suspicious roots (one flattened call is never the "real" parent of
647
+ # another).
648
+ candidates = [s for s in spans if id(s) not in suspicious_root_ids]
649
+
650
+ rows: list[dict[str, object]] = []
651
+ for suspect in suspicious_roots:
652
+ start = _numeric_start_time(suspect)
653
+ if start is None:
654
+ continue
655
+ best: dict[str, object] | None = None
656
+ best_start: float | None = None
657
+ for candidate in candidates:
658
+ if candidate is suspect:
659
+ continue
660
+ c_start = _numeric_start_time(candidate)
661
+ if c_start is None:
662
+ continue
663
+ c_end = candidate.get("end_time")
664
+ if isinstance(c_end, (int, float)):
665
+ still_open = c_start <= start < c_end
666
+ else:
667
+ # No end_time (still open) or a malformed non-numeric value —
668
+ # treat both as "open" rather than excluding the candidate.
669
+ still_open = c_start <= start
670
+ if still_open and (best_start is None or c_start > best_start):
671
+ best = candidate
672
+ best_start = c_start
673
+ if best is not None:
674
+ row: dict[str, object] = {
675
+ "span": suspect.get("name"),
676
+ "likely_parent": best.get("name"),
677
+ }
678
+ if exchanges is not None:
679
+ row["http_sequence_confirmed"] = _http_sequence_confirms(
680
+ suspect, best, exchanges
681
+ )
682
+ rows.append(row)
683
+ return rows
684
+
685
+
686
+ def _print_misattributed_spans(
687
+ spans: list[dict[str, object]],
688
+ exchanges: list[dict[str, object]] | None = None,
689
+ ) -> None:
690
+ rows = _misattributed_span_rows(spans, exchanges)
691
+ if not rows:
692
+ return
693
+ print()
694
+ print(
695
+ "Possibly misattributed spans (unexpected root, chronologically "
696
+ "overlaps another span):"
697
+ )
698
+ for row in rows:
699
+ suffix = ""
700
+ confirmed = row.get("http_sequence_confirmed")
701
+ if confirmed is True:
702
+ suffix = " [confirmed via HTTP sequence_num ordering]"
703
+ elif confirmed is False:
704
+ suffix = " [HTTP sequence_num ordering does NOT confirm this guess]"
705
+ print(
706
+ f" {row['span']:<30} likely belongs under "
707
+ f"{row['likely_parent']}{suffix}"
708
+ )
709
+ print(
710
+ " (A span with no parent that started while another span was still "
711
+ "open may have been flattened to the trace root instead of nested "
712
+ "under its originating node — the shape behind langgraph#3975. "
713
+ "Reconciled against fixture.db's sequence_num-ordered HTTP capture "
714
+ "where available, per the timestamp-independent cross-check above.)"
715
+ )
716
+
717
+
718
+ # ---------------------------------------------------------------------------
719
+ # Subcommand: show
720
+ # ---------------------------------------------------------------------------
721
+
722
+
723
+ def cmd_show(args: argparse.Namespace) -> None:
724
+ """Print the trace.json for a run, pretty-printed."""
725
+ run_id: str = args.run_id
726
+ _require_run_dir(run_id)
727
+
728
+ trace_path = _trace_json_path(run_id)
729
+ if not trace_path.exists():
730
+ sys.exit(
731
+ f"error: no trace.json found for run {run_id!r}\n"
732
+ f"Expected: {trace_path}\n"
733
+ "The run directory exists but no trace was written. "
734
+ "Did the trace context exit normally?"
735
+ )
736
+
737
+ try:
738
+ data = json.loads(trace_path.read_text(encoding="utf-8"))
739
+ except json.JSONDecodeError as exc:
740
+ sys.exit(f"error: trace.json is not valid JSON: {exc}")
741
+
742
+ spans = data.get("spans", [])
743
+
744
+ if getattr(args, "errors_only", False):
745
+ _print_errors_only(spans)
746
+ return
747
+
748
+ # Try rich for colored output; fall back to plain json.dumps
749
+ try:
750
+ from rich.console import Console
751
+ from rich.syntax import Syntax
752
+
753
+ console = Console()
754
+ syntax = Syntax(
755
+ json.dumps(data, indent=2),
756
+ "json",
757
+ theme="monokai",
758
+ line_numbers=False,
759
+ )
760
+ console.print(syntax)
761
+ except ImportError:
762
+ print(json.dumps(data, indent=2))
763
+
764
+ print()
765
+ print(f"File: {trace_path}")
766
+ print(f"Spans: {len(spans)}")
767
+
768
+ # Load fixture.db's exchanges (if this run was recorded with
769
+ # record=True) purely so _print_misattributed_spans can reconcile its
770
+ # wall-clock-timestamp guess against the independent, always-correct
771
+ # sequence_num ordering the HTTP capture layer already has.
772
+ exchanges: list[dict[str, object]] | None = None
773
+ fixture_db = _fixture_path(run_id)
774
+ if fixture_db.exists():
775
+ try:
776
+ from agent_trace._replay.fixture import Fixture
777
+
778
+ with Fixture(fixture_db) as f:
779
+ exchanges = f.all_exchanges()
780
+ except Exception:
781
+ exchanges = None
782
+
783
+ _print_error_classification(spans)
784
+ _print_duplicate_node_spans(spans)
785
+ _print_retry_storms(spans)
786
+ _print_misattributed_spans(spans, exchanges)
787
+ _print_checkpoint_durability(spans)
788
+ _print_zero_task_updates(spans)
789
+
790
+
791
+ # ---------------------------------------------------------------------------
792
+ # Subcommand: replay
793
+ # ---------------------------------------------------------------------------
794
+
795
+
796
+ def _print_replay_span_diagnostics(
797
+ spans: list[dict[str, object]],
798
+ exchanges: list[dict[str, object]] | None = None,
799
+ ) -> None:
800
+ """The full diagnostic block `agent-trace replay` prints after the span
801
+ tree — split out from cmd_replay() purely to keep that function's
802
+ statement count manageable, not for reuse elsewhere."""
803
+ _print_error_classification(spans)
804
+ _print_duplicate_node_spans(spans)
805
+ _print_retry_storms(spans)
806
+ _print_misattributed_spans(spans, exchanges)
807
+ _print_checkpoint_durability(spans)
808
+ _print_zero_task_updates(spans)
809
+
810
+
811
+ def cmd_replay(args: argparse.Namespace) -> None:
812
+ """Replay a recorded run and print the resulting span tree."""
813
+ run_id: str = args.run_id
814
+ _require_run_dir(run_id)
815
+
816
+ fixture = _fixture_path(run_id)
817
+ if not fixture.exists():
818
+ sys.exit(
819
+ f"error: no fixture.db found for run {run_id!r}\n"
820
+ f"Expected: {fixture}\n"
821
+ "Did you record this run with record=True?\n"
822
+ "Without record=True only trace.json is written, not fixture.db."
823
+ )
824
+
825
+ trace_json_path = _trace_json_path(run_id)
826
+
827
+ print(f"Replaying run: {run_id}")
828
+ print(f"Fixture: {fixture}")
829
+
830
+ # Load the original trace to find span names
831
+ original_spans: list[dict[str, object]] = []
832
+ if trace_json_path.exists():
833
+ try:
834
+ data = json.loads(trace_json_path.read_text(encoding="utf-8"))
835
+ original_spans = data.get("spans", [])
836
+ except Exception: # noqa: S110
837
+ pass
838
+
839
+ print(f"Original span count: {len(original_spans)}")
840
+
841
+ # Count exchanges via Fixture (honours WAL mode + schema)
842
+ from agent_trace._replay.fixture import Fixture
843
+
844
+ try:
845
+ with Fixture(fixture) as f:
846
+ exchange_count = f.exchange_count()
847
+ all_exchanges = f.all_exchanges()
848
+ except Exception:
849
+ exchange_count = 0
850
+ all_exchanges = []
851
+
852
+ print(f"Recorded exchanges: {exchange_count}")
853
+ _print_streaming_timing(all_exchanges)
854
+ _print_http_error_exchanges(all_exchanges)
855
+ print()
856
+
857
+ # Enter replay mode
858
+ from agent_trace import replay as _replay
859
+
860
+ with _replay(run_id, trace_dir=_trace_dir()) as ctx:
861
+ print(f"Replay active — {ctx.fixture.exchange_count()} exchange(s) available")
862
+ print("(No HTTP requests were made — fixture is ready for agent code)")
863
+ print()
864
+
865
+ # Load and print the original trace
866
+ if trace_json_path.exists():
867
+ try:
868
+ from agent_trace.core.trace import Trace
869
+ from agent_trace.exporters.stdout import StdoutExporter
870
+
871
+ trace_data = json.loads(trace_json_path.read_text(encoding="utf-8"))
872
+ trace_obj = Trace.from_dict(trace_data)
873
+ print("--- Original span tree (from trace.json) ---")
874
+ StdoutExporter().export(trace_obj)
875
+ _print_replay_span_diagnostics(trace_data.get("spans", []), all_exchanges)
876
+ except Exception as exc:
877
+ print(f"Could not render span tree: {exc}")
878
+ else:
879
+ print(f"(No trace.json — run 'agent-trace show {run_id}' after recording)")
880
+
881
+
882
+ # ---------------------------------------------------------------------------
883
+ # Subcommand: inspect
884
+ # ---------------------------------------------------------------------------
885
+ #
886
+ # The one-stop diagnosis command: decodes fixture.db request/response bodies
887
+ # and runs every pattern check in agent_trace._inspect against them,
888
+ # printing a flagged summary instead of requiring a developer to hand-write
889
+ # a comparison script against Fixture.all_exchanges() (the exact manual step
890
+ # this command replaces, closing the gap reported in upstream issue #531).
891
+ # ---------------------------------------------------------------------------
892
+
893
+
894
+ def _load_run_exchanges_and_spans(
895
+ run_id: str,
896
+ ) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
897
+ """Return (exchanges, spans) for *run_id* — empty lists for whichever
898
+ file (fixture.db / trace.json) doesn't exist rather than raising, since
899
+ a run may have been recorded with record=False (spans only) or may not
900
+ have a trace.json yet."""
901
+ from agent_trace._replay.fixture import Fixture
902
+
903
+ exchanges: list[dict[str, Any]] = []
904
+ fixture_path = _fixture_path(run_id)
905
+ if fixture_path.exists():
906
+ with Fixture(fixture_path) as f:
907
+ exchanges = f.all_exchanges()
908
+
909
+ spans: list[dict[str, Any]] = []
910
+ trace_path = _trace_json_path(run_id)
911
+ if trace_path.exists():
912
+ try:
913
+ data = json.loads(trace_path.read_text(encoding="utf-8"))
914
+ spans = data.get("spans", [])
915
+ except json.JSONDecodeError:
916
+ pass
917
+
918
+ return exchanges, spans
919
+
920
+
921
+ def _print_flags(title: str, flags: list[dict[str, Any]]) -> None:
922
+ if not flags:
923
+ return
924
+ print()
925
+ print(f"{title} ({len(flags)}):")
926
+ for flag in flags:
927
+ print(f" - {flag.get('detail', flag)}")
928
+
929
+
930
+ def cmd_inspect(args: argparse.Namespace) -> None:
931
+ """Auto-flag/search raw request-response bodies for known malformed
932
+ shapes, plus a set of cross-span diagnostics — a CLI command so a
933
+ developer doesn't have to hand-write a comparison script against
934
+ Fixture.all_exchanges()."""
935
+ from agent_trace import _inspect as ins
936
+
937
+ run_id: str = args.run_id
938
+ _require_run_dir(run_id)
939
+ exchanges, spans = _load_run_exchanges_and_spans(run_id)
940
+
941
+ print(f"Inspecting run: {run_id}")
942
+ print(f"Exchanges: {len(exchanges)} Spans: {len(spans)}")
943
+
944
+ results = ins.run_all_exchange_checks(exchanges)
945
+ for check_name, flags in results.items():
946
+ _print_flags(check_name, flags)
947
+
948
+ _print_flags(
949
+ "known_error_pattern",
950
+ ins.match_known_error_patterns(spans),
951
+ )
952
+ _print_flags(
953
+ "reserved_kwarg_collision",
954
+ ins.check_reserved_kwarg_collision(spans),
955
+ )
956
+ _print_flags(
957
+ "near_duplicate_sibling_content",
958
+ ins.find_near_duplicate_sibling_content(spans),
959
+ )
960
+ _print_flags(
961
+ "phantom_tool_call",
962
+ ins.check_phantom_tool_call(spans, exchanges),
963
+ )
964
+ _print_flags(
965
+ "system_prompt_dropped",
966
+ ins.check_system_prompt_dropped(spans),
967
+ )
968
+
969
+ if args.registered_tools:
970
+ registered = set(args.registered_tools.split(","))
971
+ _print_flags(
972
+ "tool_call_name_fuzzy_match",
973
+ ins.check_tool_call_name_fuzzy_match(exchanges, registered),
974
+ )
975
+ _print_flags(
976
+ "tool_call_name_dotted_compound",
977
+ ins.check_tool_call_name_dotted_compound(exchanges, registered),
978
+ )
979
+ _print_flags(
980
+ "action_name_not_registered",
981
+ ins.check_action_name_not_registered(exchanges, registered),
982
+ )
983
+ _print_flags(
984
+ "tool_call_name_not_registered",
985
+ ins.check_tool_call_name_not_registered(exchanges, registered),
986
+ )
987
+
988
+ if args.configured_host:
989
+ _print_flags(
990
+ "endpoint_host_mismatch",
991
+ ins.check_endpoint_host_mismatch(exchanges, args.configured_host),
992
+ )
993
+
994
+ if args.check_kwarg:
995
+ _print_flags(
996
+ "missing_extra_kwarg",
997
+ ins.check_missing_extra_kwarg(exchanges, args.check_kwarg),
998
+ )
999
+
1000
+ if args.diff_field:
1001
+ _print_flags(
1002
+ "field_present_on_wire_absent_downstream",
1003
+ ins.field_present_on_wire_absent_downstream(
1004
+ exchanges, spans, args.diff_field
1005
+ ),
1006
+ )
1007
+
1008
+ if args.diff_get_post_field:
1009
+ _print_flags(
1010
+ "get_post_field_mismatch",
1011
+ ins.check_get_post_field_mismatch(
1012
+ exchanges,
1013
+ args.diff_get_post_field,
1014
+ get_id_field=args.diff_get_post_id_field,
1015
+ post_id_field=args.diff_get_post_post_id_field,
1016
+ ),
1017
+ )
1018
+
1019
+ if not results and not spans:
1020
+ print()
1021
+ print("No anomalies flagged (or nothing recorded for this run).")
1022
+
1023
+
1024
+ # ---------------------------------------------------------------------------
1025
+ # Subcommand: diff
1026
+ # ---------------------------------------------------------------------------
1027
+ #
1028
+ # `agent-trace diff <run_id_a> <run_id_b>` — the CLI never previously
1029
+ # exposed raw exchange bodies at all, let alone a comparison between two
1030
+ # runs. cmd_show only pretty-prints trace.json span metadata; cmd_replay
1031
+ # only prints exchange counts and the span tree.
1032
+ # ---------------------------------------------------------------------------
1033
+
1034
+
1035
+ def _exchanges_by_url(
1036
+ exchanges: list[dict[str, Any]],
1037
+ ) -> dict[str, list[dict[str, Any]]]:
1038
+ by_url: dict[str, list[dict[str, Any]]] = {}
1039
+ for exchange in exchanges:
1040
+ by_url.setdefault(str(exchange.get("url")), []).append(exchange)
1041
+ return by_url
1042
+
1043
+
1044
+ def _diff_text(label_a: str, text_a: str, label_b: str, text_b: str) -> list[str]:
1045
+ return list(
1046
+ difflib.unified_diff(
1047
+ text_a.splitlines(keepends=True),
1048
+ text_b.splitlines(keepends=True),
1049
+ fromfile=label_a,
1050
+ tofile=label_b,
1051
+ lineterm="",
1052
+ )
1053
+ )
1054
+
1055
+
1056
+ def _print_exchange_diff(
1057
+ exchanges_a: list[dict[str, Any]],
1058
+ exchanges_b: list[dict[str, Any]],
1059
+ run_id_a: str,
1060
+ run_id_b: str,
1061
+ ) -> None:
1062
+ """Print the field-level diff of two runs' exchanges, matched by URL."""
1063
+ by_url_a = _exchanges_by_url(exchanges_a)
1064
+ by_url_b = _exchanges_by_url(exchanges_b)
1065
+
1066
+ all_urls = sorted(set(by_url_a) | set(by_url_b))
1067
+ if not all_urls:
1068
+ print("No exchanges recorded in either run.")
1069
+ return
1070
+
1071
+ any_diff = False
1072
+ for url in all_urls:
1073
+ rows_a = by_url_a.get(url, [])
1074
+ rows_b = by_url_b.get(url, [])
1075
+ if not rows_a:
1076
+ print(f"\n{url}: only present in {run_id_b} ({len(rows_b)} exchange(s))")
1077
+ any_diff = True
1078
+ continue
1079
+ if not rows_b:
1080
+ print(f"\n{url}: only present in {run_id_a} ({len(rows_a)} exchange(s))")
1081
+ any_diff = True
1082
+ continue
1083
+
1084
+ for i in range(max(len(rows_a), len(rows_b))):
1085
+ row_a = rows_a[i] if i < len(rows_a) else None
1086
+ row_b = rows_b[i] if i < len(rows_b) else None
1087
+ if row_a is None or row_b is None:
1088
+ print(f"\n{url} [{i}]: exchange count differs between runs")
1089
+ any_diff = True
1090
+ continue
1091
+ for field in ("request_body", "response_body"):
1092
+ text_a = str(row_a.get(field, ""))
1093
+ text_b = str(row_b.get(field, ""))
1094
+ if text_a == text_b:
1095
+ continue
1096
+ any_diff = True
1097
+ print(f"\n{url} [{i}] {field}:")
1098
+ for line in _diff_text(f"{run_id_a}", text_a, f"{run_id_b}", text_b):
1099
+ print(f" {line}")
1100
+
1101
+ if not any_diff:
1102
+ print("\nNo differences found — every matched exchange is byte-identical.")
1103
+
1104
+
1105
+ def cmd_diff(args: argparse.Namespace) -> None:
1106
+ """Print a structured diff of two recorded runs' exchanges, matched by
1107
+ URL, highlighting field-level differences between request/response
1108
+ bodies."""
1109
+ from agent_trace import _inspect as ins
1110
+
1111
+ run_id_a: str = args.run_id_a
1112
+ run_id_b: str = args.run_id_b
1113
+ _require_run_dir(run_id_a)
1114
+ _require_run_dir(run_id_b)
1115
+
1116
+ exchanges_a, spans_a = _load_run_exchanges_and_spans(run_id_a)
1117
+ exchanges_b, spans_b = _load_run_exchanges_and_spans(run_id_b)
1118
+
1119
+ print(f"Diffing {run_id_a} ({len(exchanges_a)} exchanges) vs "
1120
+ f"{run_id_b} ({len(exchanges_b)} exchanges)")
1121
+
1122
+ _print_exchange_diff(exchanges_a, exchanges_b, run_id_a, run_id_b)
1123
+
1124
+ # Restart-vs-resume: does run_b's root chain span for a shared
1125
+ # LangGraph thread_id continue from run_a's last recorded
1126
+ # langgraph_step, or did the graph start over from scratch (#161)?
1127
+ _print_flags(
1128
+ "restart_vs_resume",
1129
+ ins.check_restart_vs_resume(spans_a, spans_b),
1130
+ )
1131
+
1132
+
1133
+ # ---------------------------------------------------------------------------
1134
+ # Subcommand: run
1135
+ # ---------------------------------------------------------------------------
1136
+ #
1137
+ # Wraps an arbitrary child process (e.g. `langgraph dev`) with recording
1138
+ # pre-enabled process-wide, via the AGENT_TRACE_AUTO_RECORD env var
1139
+ # (Tracer.start_auto_record() / agent_trace._activate_auto_record_from_env())
1140
+ # — the supported mechanism for capturing a framework-managed dev server
1141
+ # process whose entrypoint the developer doesn't own, and doesn't want to
1142
+ # hand-instrument with a `with tracer.start_trace(...)` block.
1143
+ # ---------------------------------------------------------------------------
1144
+
1145
+
1146
+ def _strip_leading_separator(command: list[str]) -> list[str]:
1147
+ """`agent-trace run -- langgraph dev` — argparse.REMAINDER captures the
1148
+ `--` separator verbatim as the first token of the remainder. Strip it
1149
+ (when present) so the child process is exec'd with the real command
1150
+ only. A bare `agent-trace run langgraph dev` (no `--`) works too — there
1151
+ is simply nothing to strip in that case."""
1152
+ if command and command[0] == "--":
1153
+ return command[1:]
1154
+ return command
1155
+
1156
+
1157
+ def cmd_run(args: argparse.Namespace) -> None:
1158
+ """Exec a child process with recording pre-enabled process-wide.
1159
+
1160
+ Sets ``AGENT_TRACE_AUTO_RECORD=1`` (plus ``AGENT_TRACE_RUN_ID``/
1161
+ ``AGENT_TRACE_AUTO_RECORD_NAME``/``AGENT_TRACE_TRACE_DIR``) in the
1162
+ child's environment before launching it, so the *first* `import
1163
+ agent_trace` anywhere inside that process — including one owned
1164
+ entirely by a third-party CLI like `langgraph dev` that the developer
1165
+ never touches — activates process-wide recording with zero code
1166
+ changes required in the developer's own graph/agent code. See
1167
+ ``agent_trace.Tracer.start_auto_record`` and
1168
+ ``agent_trace._activate_auto_record_from_env`` for the activation path
1169
+ this env var feeds into.
1170
+
1171
+ Exits with the child process's own exit code.
1172
+ """
1173
+ command = _strip_leading_separator(args.child_command)
1174
+ if not command:
1175
+ sys.exit(
1176
+ "error: no command given.\n"
1177
+ "Usage: agent-trace run -- <command> [args...]\n"
1178
+ "Example: agent-trace run -- langgraph dev"
1179
+ )
1180
+
1181
+ run_id = args.run_id or f"run_{uuid.uuid4().hex[:12]}"
1182
+ trace_dir = _trace_dir()
1183
+
1184
+ env = os.environ.copy()
1185
+ env["AGENT_TRACE_AUTO_RECORD"] = "1"
1186
+ env["AGENT_TRACE_RUN_ID"] = run_id
1187
+ env["AGENT_TRACE_AUTO_RECORD_NAME"] = args.name
1188
+ env["AGENT_TRACE_TRACE_DIR"] = str(trace_dir)
1189
+
1190
+ print("agent-trace: recording enabled (AGENT_TRACE_AUTO_RECORD=1)")
1191
+ print(f"agent-trace: run_id: {run_id}")
1192
+ print(f"agent-trace: trace_dir: {trace_dir}")
1193
+ print(f"agent-trace: command: {' '.join(command)}")
1194
+ print()
1195
+
1196
+ try:
1197
+ result = subprocess.run(command, env=env, check=False) # noqa: S603
1198
+ except FileNotFoundError as exc:
1199
+ sys.exit(f"error: could not exec {command[0]!r}: {exc}")
1200
+ except KeyboardInterrupt:
1201
+ sys.exit(130)
1202
+
1203
+ print()
1204
+ print(f"agent-trace: child process exited with code {result.returncode}")
1205
+ print(f"agent-trace: inspect this run with: agent-trace show {run_id}")
1206
+ sys.exit(result.returncode)
1207
+
1208
+
1209
+ # ---------------------------------------------------------------------------
1210
+ # Entry point
1211
+ # ---------------------------------------------------------------------------
1212
+
1213
+
1214
+ def main() -> None:
1215
+ parser = argparse.ArgumentParser(
1216
+ prog="agent-trace",
1217
+ description="agent-trace — AI agent observability with record/replay",
1218
+ formatter_class=argparse.RawDescriptionHelpFormatter,
1219
+ epilog=(
1220
+ "Examples:\n"
1221
+ " agent-trace list\n"
1222
+ " agent-trace show run_abc123def456\n"
1223
+ " agent-trace show run_abc123def456 --errors-only\n"
1224
+ " agent-trace replay run_abc123def456\n"
1225
+ " agent-trace inspect run_abc123def456\n"
1226
+ " agent-trace diff run_a run_b\n"
1227
+ " agent-trace run -- langgraph dev\n"
1228
+ " agent-trace version\n"
1229
+ ),
1230
+ )
1231
+
1232
+ sub = parser.add_subparsers(dest="command", metavar="COMMAND")
1233
+ sub.required = True
1234
+
1235
+ # version
1236
+ sub.add_parser("version", help="Print version and exit")
1237
+
1238
+ # list
1239
+ sub.add_parser("list", help="List all recorded runs")
1240
+
1241
+ # show
1242
+ show_p = sub.add_parser("show", help="Pretty-print trace.json for a run")
1243
+ show_p.add_argument("run_id", help="Run ID (e.g. run_abc123def456)")
1244
+ show_p.add_argument(
1245
+ "--errors-only",
1246
+ dest="errors_only",
1247
+ action="store_true",
1248
+ help="Only print ERROR-status spans with their captured exception text",
1249
+ )
1250
+
1251
+ # replay
1252
+ replay_p = sub.add_parser(
1253
+ "replay",
1254
+ help="Enter replay mode for a run and print the span tree",
1255
+ )
1256
+ replay_p.add_argument("run_id", help="Run ID (e.g. run_abc123def456)")
1257
+
1258
+ # inspect
1259
+ inspect_p = sub.add_parser(
1260
+ "inspect",
1261
+ help="Auto-flag/search raw request-response bodies for known malformed shapes",
1262
+ )
1263
+ inspect_p.add_argument("run_id", help="Run ID (e.g. run_abc123def456)")
1264
+ inspect_p.add_argument(
1265
+ "--registered-tools",
1266
+ dest="registered_tools",
1267
+ default=None,
1268
+ help="Comma-separated list of registered tool names, enables tool-call "
1269
+ "name fuzzy-match/dotted-compound/ReAct-action-name checks",
1270
+ )
1271
+ inspect_p.add_argument(
1272
+ "--configured-host",
1273
+ dest="configured_host",
1274
+ default=None,
1275
+ help="Framework's configured LLM endpoint host, enables the "
1276
+ "endpoint-host-mismatch check",
1277
+ )
1278
+ inspect_p.add_argument(
1279
+ "--check-kwarg",
1280
+ dest="check_kwarg",
1281
+ default=None,
1282
+ help="Dotted kwarg path (e.g. extra_body.chat_template_kwargs.thinking) "
1283
+ "expected to be present on the wire; flags requests where it's absent",
1284
+ )
1285
+ inspect_p.add_argument(
1286
+ "--diff-field",
1287
+ dest="diff_field",
1288
+ default=None,
1289
+ help="Response field to check for wire-present-but-downstream-"
1290
+ "absent — a top-level key (e.g. usage) or a dotted/nested path "
1291
+ "with numeric list-index segments (e.g. "
1292
+ "choices.0.message.reasoning_content, for provider fields nested "
1293
+ "inside the response body like DeepSeek's reasoning_content)",
1294
+ )
1295
+ inspect_p.add_argument(
1296
+ "--diff-get-post-field",
1297
+ dest="diff_get_post_field",
1298
+ default=None,
1299
+ help="Dotted field path (e.g. instructions) to compare between an "
1300
+ "earlier GET response and a later, causally-related POST request "
1301
+ "body referencing the same resource id (#2620); flags stale-value "
1302
+ "mismatches such as a GPTAssistantAgent POST /runs still sending "
1303
+ "instructions that no longer match what GET /assistants/{id} "
1304
+ "returns",
1305
+ )
1306
+ inspect_p.add_argument(
1307
+ "--diff-get-post-id-field",
1308
+ dest="diff_get_post_id_field",
1309
+ default="id",
1310
+ help="Field name the GET response uses for the resource id "
1311
+ "(default: id)",
1312
+ )
1313
+ inspect_p.add_argument(
1314
+ "--diff-get-post-post-id-field",
1315
+ dest="diff_get_post_post_id_field",
1316
+ default=None,
1317
+ help="Field name the POST request body uses to reference the same "
1318
+ "resource id, if different from --diff-get-post-id-field (e.g. "
1319
+ "assistant_id)",
1320
+ )
1321
+
1322
+ # diff
1323
+ diff_p = sub.add_parser(
1324
+ "diff",
1325
+ help="Diff two recorded runs' exchanges (matched by URL)",
1326
+ )
1327
+ diff_p.add_argument("run_id_a", help="First run ID")
1328
+ diff_p.add_argument("run_id_b", help="Second run ID")
1329
+
1330
+ # run
1331
+ run_p = sub.add_parser(
1332
+ "run",
1333
+ help=(
1334
+ "Exec a child process with recording pre-enabled process-wide "
1335
+ "(e.g. agent-trace run -- langgraph dev)"
1336
+ ),
1337
+ )
1338
+ run_p.add_argument(
1339
+ "--run-id",
1340
+ dest="run_id",
1341
+ default=None,
1342
+ help="Explicit run ID (default: random, printed on start)",
1343
+ )
1344
+ run_p.add_argument(
1345
+ "--name",
1346
+ dest="name",
1347
+ default="auto-record",
1348
+ help="Trace name recorded in trace.json metadata (default: auto-record)",
1349
+ )
1350
+ run_p.add_argument(
1351
+ "child_command",
1352
+ nargs=argparse.REMAINDER,
1353
+ help="Command to exec, e.g. -- langgraph dev",
1354
+ )
1355
+
1356
+ args = parser.parse_args()
1357
+
1358
+ dispatch = {
1359
+ "version": cmd_version,
1360
+ "list": cmd_list,
1361
+ "show": cmd_show,
1362
+ "replay": cmd_replay,
1363
+ "inspect": cmd_inspect,
1364
+ "diff": cmd_diff,
1365
+ "run": cmd_run,
1366
+ }
1367
+
1368
+ handler = dispatch.get(args.command)
1369
+ if handler is None:
1370
+ parser.print_help()
1371
+ sys.exit(1)
1372
+
1373
+ handler(args)
1374
+
1375
+
1376
+ if __name__ == "__main__":
1377
+ main()