agent-observability-trace-cli 0.1.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. agent_observability_trace_cli-0.1.2.dist-info/METADATA +336 -0
  2. agent_observability_trace_cli-0.1.2.dist-info/RECORD +51 -0
  3. agent_observability_trace_cli-0.1.2.dist-info/WHEEL +4 -0
  4. agent_observability_trace_cli-0.1.2.dist-info/entry_points.txt +2 -0
  5. agent_observability_trace_cli-0.1.2.dist-info/licenses/LICENSE +198 -0
  6. agent_trace/__init__.py +1182 -0
  7. agent_trace/_cli.py +1377 -0
  8. agent_trace/_inspect.py +2020 -0
  9. agent_trace/_replay/__init__.py +1 -0
  10. agent_trace/_replay/engine.py +268 -0
  11. agent_trace/_replay/fixture.py +868 -0
  12. agent_trace/core/__init__.py +1 -0
  13. agent_trace/core/clock.py +91 -0
  14. agent_trace/core/exceptions.py +51 -0
  15. agent_trace/core/span.py +271 -0
  16. agent_trace/core/trace.py +92 -0
  17. agent_trace/exporters/__init__.py +1 -0
  18. agent_trace/exporters/file.py +80 -0
  19. agent_trace/exporters/otlp.py +199 -0
  20. agent_trace/exporters/remote_fixture.py +307 -0
  21. agent_trace/exporters/stdout.py +258 -0
  22. agent_trace/integrations/__init__.py +69 -0
  23. agent_trace/integrations/agno.py +489 -0
  24. agent_trace/integrations/autogen.py +581 -0
  25. agent_trace/integrations/crewai.py +486 -0
  26. agent_trace/integrations/google_genai.py +731 -0
  27. agent_trace/integrations/haystack.py +254 -0
  28. agent_trace/integrations/langchain_core.py +361 -0
  29. agent_trace/integrations/langgraph.py +2093 -0
  30. agent_trace/integrations/langgraph_checkpoint.py +763 -0
  31. agent_trace/integrations/langgraph_state_diff.py +345 -0
  32. agent_trace/integrations/langgraph_stream_debug.py +202 -0
  33. agent_trace/integrations/llama_index.py +472 -0
  34. agent_trace/integrations/mcp.py +281 -0
  35. agent_trace/integrations/openai_agents.py +811 -0
  36. agent_trace/integrations/pydantic_ai.py +504 -0
  37. agent_trace/integrations/streaming.py +218 -0
  38. agent_trace/interceptor/__init__.py +64 -0
  39. agent_trace/interceptor/aiohttp_hook.py +152 -0
  40. agent_trace/interceptor/botocore_hook.py +223 -0
  41. agent_trace/interceptor/grpc_hook.py +651 -0
  42. agent_trace/interceptor/httpx_hook.py +866 -0
  43. agent_trace/interceptor/logging_hook.py +137 -0
  44. agent_trace/interceptor/requests_patch.py +184 -0
  45. agent_trace/interceptor/sse.py +176 -0
  46. agent_trace/interceptor/stdio_hook.py +245 -0
  47. agent_trace/interceptor/warnings_hook.py +132 -0
  48. agent_trace/interceptor/websocket_hook.py +323 -0
  49. agent_trace/plugins/__init__.py +35 -0
  50. agent_trace/plugins/base.py +111 -0
  51. agent_trace/py.typed +0 -0
@@ -0,0 +1,2093 @@
1
+ """
2
+ LangGraph integration — callback handler and graph-aware span enrichment.
3
+
4
+ Instruments a LangGraph StateGraph to emit spans for each node execution,
5
+ edge traversal, and LLM call within the graph.
6
+
7
+ Usage:
8
+ from agent_trace.integrations.langgraph import LangGraphTracer
9
+ from agent_trace import tracer
10
+
11
+ with tracer.start_trace("my_graph", record=True) as trace:
12
+ result = graph.invoke(
13
+ input_state,
14
+ config={"callbacks": [LangGraphTracer(tracer=tracer, trace=trace)]}
15
+ )
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ import asyncio
21
+ import contextvars
22
+ import functools
23
+ import hashlib
24
+ import inspect
25
+ import json
26
+ import logging
27
+ import threading
28
+ import uuid
29
+ from collections.abc import AsyncGenerator, AsyncIterable, Callable, Generator, Iterable
30
+ from typing import TYPE_CHECKING, Any
31
+
32
+ from agent_trace.core.clock import get_time
33
+ from agent_trace.core.span import Span, SpanStatus
34
+ from agent_trace.interceptor.httpx_hook import pop_correlation_id, push_correlation_id
35
+
36
+ if TYPE_CHECKING:
37
+ from agent_trace import Trace, Tracer
38
+
39
+ __all__ = [
40
+ "LangGraphTracer",
41
+ "derive_trace_id",
42
+ "find_tool_params_shaped_like_state",
43
+ "instrument_graph_factory",
44
+ "traced_astream",
45
+ "traced_stream",
46
+ ]
47
+
48
+ logger = logging.getLogger(__name__)
49
+
50
+ _INSTALL_HINT = (
51
+ "LangGraphTracer requires langchain-core and langgraph.\n"
52
+ "Install them with:\n\n"
53
+ " pip install langchain-core langgraph\n"
54
+ )
55
+
56
+ # Span attributes only accept str/int/float/bool (see agent_trace.core.span).
57
+ # Anything structured (BaseMessage, dicts full of BaseMessages, dataclasses,
58
+ # etc.) has to be flattened to a bounded string before it can be stored.
59
+ _MAX_ATTR_LEN = 8_000
60
+
61
+ # _deep_serialize() bounds — deliberately small. This is span-attribute
62
+ # summary data, not a full-fidelity dump, and the bound also protects against
63
+ # pathological/self-referential objects (e.g. an unconfigured
64
+ # unittest.mock.MagicMock, whose attribute access and method calls always
65
+ # yield a brand-new child Mock) that would otherwise recurse indefinitely.
66
+ _MAX_SERIALIZE_DEPTH = 6
67
+ _MAX_COLLECTION_ITEMS = 200
68
+
69
+ # Cap on the number of llm_stream_delta SpanEvents recorded per LLM span —
70
+ # a long stream (thousands of tokens) would otherwise grow a single span
71
+ # unboundedly. llm.stream_token_count keeps counting past this cap; only
72
+ # new SpanEvents stop being appended.
73
+ _MAX_STREAM_EVENTS_PER_SPAN = 500
74
+
75
+
76
+ def _safe_str(value: Any) -> str:
77
+ """str(value), degrading to a placeholder instead of raising."""
78
+ try:
79
+ return str(value)
80
+ except Exception:
81
+ return "<unserializable>"
82
+
83
+
84
+ def _serialize_mapping(
85
+ value: dict[Any, Any], *, _depth: int, _seen: frozenset[int]
86
+ ) -> dict[str, Any]:
87
+ """_deep_serialize() branch for dict-like values, capped at
88
+ _MAX_COLLECTION_ITEMS keys."""
89
+ out: dict[str, Any] = {}
90
+ items = list(value.items())
91
+ for k, v in items[:_MAX_COLLECTION_ITEMS]:
92
+ out[str(k)] = _deep_serialize(v, _depth=_depth + 1, _seen=_seen)
93
+ if len(items) > _MAX_COLLECTION_ITEMS:
94
+ out["..."] = f"<{len(items) - _MAX_COLLECTION_ITEMS} more items truncated>"
95
+ return out
96
+
97
+
98
+ def _serialize_sequence(
99
+ value: list[Any] | tuple[Any, ...], *, _depth: int, _seen: frozenset[int]
100
+ ) -> list[Any]:
101
+ """_deep_serialize() branch for list/tuple values, capped at
102
+ _MAX_COLLECTION_ITEMS items."""
103
+ items = list(value)
104
+ out_list = [
105
+ _deep_serialize(item, _depth=_depth + 1, _seen=_seen)
106
+ for item in items[:_MAX_COLLECTION_ITEMS]
107
+ ]
108
+ if len(items) > _MAX_COLLECTION_ITEMS:
109
+ out_list.append(f"<{len(items) - _MAX_COLLECTION_ITEMS} more items truncated>")
110
+ return out_list
111
+
112
+
113
+ def _serialize_container(
114
+ value: dict[Any, Any] | list[Any] | tuple[Any, ...],
115
+ *,
116
+ _depth: int,
117
+ _seen: frozenset[int],
118
+ ) -> Any:
119
+ """Dispatch a dict/list/tuple to its dedicated serializer.
120
+
121
+ Split out purely to keep _deep_serialize's own branch/return count low.
122
+ """
123
+ if isinstance(value, dict):
124
+ return _serialize_mapping(value, _depth=_depth, _seen=_seen)
125
+ return _serialize_sequence(value, _depth=_depth, _seen=_seen)
126
+
127
+
128
+ def _dump_via_attr(value: Any, attr_name: str) -> Any:
129
+ """Call value.<attr_name>() if present and callable, else return None.
130
+
131
+ Used for both the pydantic-v2 ``model_dump()`` shape and the pydantic-v1
132
+ / dict-like ``dict()`` shape — a failure or absence of the method is not
133
+ an error, just "this strategy doesn't apply to this object".
134
+ """
135
+ method = getattr(value, attr_name, None)
136
+ if not callable(method):
137
+ return None
138
+ try:
139
+ return method()
140
+ except Exception:
141
+ return None
142
+
143
+
144
+ def _deep_serialize(
145
+ value: Any,
146
+ *,
147
+ _depth: int = 0,
148
+ _seen: frozenset[int] = frozenset(),
149
+ ) -> Any:
150
+ """Recursively convert *value* into JSON-primitive-only data.
151
+
152
+ Tries, in order, per non-primitive object: a pydantic-v2-style
153
+ ``model_dump()`` (what ``langchain_core.messages.BaseMessage`` exposes),
154
+ a ``dict()`` method (pydantic v1 / other dict-likes), and finally
155
+ ``str()``. Bounded on both depth and collection size (rather than
156
+ delegating recursion to ``json.dumps``'s own ``default`` callback) so a
157
+ pathological or infinitely-self-generating object cannot spin this into
158
+ a multi-thousand-frame stack walk — every recursive branch strictly
159
+ increments ``_depth`` and stops at ``_MAX_SERIALIZE_DEPTH`` regardless of
160
+ what the object's own methods return.
161
+ """
162
+ if value is None or isinstance(value, (str, int, float, bool)):
163
+ return value
164
+
165
+ obj_id = id(value)
166
+ if obj_id in _seen:
167
+ return "<circular-reference>"
168
+ if _depth >= _MAX_SERIALIZE_DEPTH:
169
+ return _safe_str(value)
170
+
171
+ seen_here = _seen | {obj_id}
172
+
173
+ if isinstance(value, (dict, list, tuple)):
174
+ return _serialize_container(value, _depth=_depth, _seen=seen_here)
175
+
176
+ for attr_name in ("model_dump", "dict"):
177
+ dumped = _dump_via_attr(value, attr_name)
178
+ if dumped is not None:
179
+ return _deep_serialize(dumped, _depth=_depth + 1, _seen=seen_here)
180
+
181
+ return _safe_str(value)
182
+
183
+
184
+ def _to_attr_string(value: Any, *, max_len: int = _MAX_ATTR_LEN) -> str:
185
+ """Defensively serialize *value* into a bounded, span-safe string.
186
+
187
+ Used for anything structured (message lists, chain inputs/outputs,
188
+ metadata dicts, response_metadata, ...) that can't be stored as a raw
189
+ span attribute directly.
190
+ """
191
+ try:
192
+ text = json.dumps(_deep_serialize(value), ensure_ascii=False, default=str)
193
+ except Exception:
194
+ try:
195
+ text = str(value)
196
+ except Exception:
197
+ text = "<unserializable>"
198
+ if len(text) > max_len:
199
+ text = text[:max_len] + "...<truncated>"
200
+ return text
201
+
202
+
203
+ def _stringify(value: Any, *, max_len: int = _MAX_ATTR_LEN) -> str:
204
+ """Turn *value* into a span-safe string.
205
+
206
+ Plain strings are truncated as-is (so raw tool I/O isn't wrapped in JSON
207
+ quotes); anything else goes through :func:`_to_attr_string`.
208
+ """
209
+ if isinstance(value, str):
210
+ text = value
211
+ else:
212
+ return _to_attr_string(value, max_len=max_len)
213
+ if len(text) > max_len:
214
+ text = text[:max_len] + "...<truncated>"
215
+ return text
216
+
217
+
218
+ # ---------------------------------------------------------------------------
219
+ # trace_id derivation from LangGraph thread_id/checkpoint identity
220
+ # ---------------------------------------------------------------------------
221
+ #
222
+ # Tracer.start_trace() always mints a random trace_id (uuid.uuid4().hex) and
223
+ # LangGraphTracer never reads config["configurable"]["thread_id"] or any
224
+ # checkpoint identity. Two worker processes independently recording "the
225
+ # same" logical operation (e.g. an original long-running tool call and its
226
+ # checkpoint-swept re-dispatch on a managed platform — see issue #7417)
227
+ # therefore produce two unrelated, un-linkable traces today: there is no way
228
+ # to recognize or diff them as the same logical run after the fact.
229
+ #
230
+ # derive_trace_id() lets a caller opt into a *deterministic* trace_id (still
231
+ # a 128-bit hex string, so it stays valid for the OTLP exporter) computed
232
+ # from LangGraph's own thread_id (+ an optional checkpoint id), instead of a
233
+ # fresh random UUID:
234
+ #
235
+ # thread_id = config["configurable"]["thread_id"]
236
+ # with tracer.start_trace("my_graph", record=True,
237
+ # trace_id=derive_trace_id(thread_id)) as trace:
238
+ # graph.invoke(inputs, config=config)
239
+ #
240
+ # Two processes deriving from the same thread_id (+ checkpoint id, when one
241
+ # resumes from a specific checkpoint) always produce the identical trace_id,
242
+ # letting tooling recognize/diff them as the same logical run — exactly the
243
+ # "tooling to diff runs" use case Trace's own docstring already describes
244
+ # trace_id as existing for.
245
+
246
+
247
+ def derive_trace_id(thread_id: str, checkpoint_id: str | None = None) -> str:
248
+ """Deterministically derive a 128-bit-hex trace_id from a LangGraph
249
+ ``thread_id`` (and, optionally, a checkpoint id).
250
+
251
+ Same (thread_id, checkpoint_id) always produces the same trace_id,
252
+ regardless of which process or machine computes it — the mechanism
253
+ needed to correlate an original run and its checkpoint-swept
254
+ re-dispatch (issue #7417) as one logical run after the fact.
255
+ """
256
+ material = thread_id if checkpoint_id is None else f"{thread_id}:{checkpoint_id}"
257
+ return hashlib.sha256(material.encode("utf-8")).hexdigest()[:32]
258
+
259
+
260
+ # ---------------------------------------------------------------------------
261
+ # Pre-invocation (graph-construction-phase) instrumentation entry point
262
+ # ---------------------------------------------------------------------------
263
+ #
264
+ # LangGraphTracer only ever attaches at graph.invoke()/graph.stream() time —
265
+ # a `config["callbacks"]=[LangGraphTracer(...)]` argument passed to a call
266
+ # that must already exist. `langgraph dev`/LangGraph Platform inverts that:
267
+ # it imports the developer's own `make_graph()`-style factory function once
268
+ # at server startup and calls it *before* any invocation exists to attach a
269
+ # callback to. A bug in that construction phase itself (MCP client setup,
270
+ # tool loading, config parsing — issue #4798) is therefore unreachable by
271
+ # LangGraphTracer no matter how it's wired into the eventual invoke() call.
272
+ #
273
+ # instrument_graph_factory() closes that gap: it wraps the factory function
274
+ # itself, so recording/tracing is active for the duration of the call that
275
+ # *builds* the graph, not only the calls that later run it.
276
+
277
+
278
+ def instrument_graph_factory(
279
+ tracer: Tracer,
280
+ factory: Callable[..., Any] | None = None,
281
+ *,
282
+ name: str = "graph-construction",
283
+ ) -> Any:
284
+ """Give a LangGraph Platform ``make_graph()``-style entry point a
285
+ construction-phase instrumentation hook, so tracing/recording is active
286
+ while the graph is being *built* — MCP client setup, tool loading,
287
+ config parsing — not only while it is later invoked.
288
+
289
+ Usable as a decorator directly on the factory::
290
+
291
+ from agent_trace import tracer
292
+ from agent_trace.integrations.langgraph import instrument_graph_factory
293
+
294
+ @instrument_graph_factory(tracer)
295
+ def make_graph(config: dict) -> CompiledStateGraph:
296
+ mcp_client = MultiServerMCPClient(...) # now captured
297
+ tools = mcp_client.get_tools()
298
+ return build_graph(tools)
299
+
300
+ ...or wrapping an existing factory inline::
301
+
302
+ graph = instrument_graph_factory(tracer, make_graph)(config)
303
+
304
+ Behavior depends on whether a trace is already active in the calling
305
+ context:
306
+
307
+ - If :attr:`Tracer.active_trace` is already set (e.g.
308
+ :meth:`Tracer.start_auto_record` activated via
309
+ ``AGENT_TRACE_AUTO_RECORD``, or an enclosing
310
+ ``start_trace(record=True)`` block), the factory call is wrapped in
311
+ its own child ``graph-construction`` span, nested under whatever is
312
+ already active, so construction-phase work is visible as a
313
+ distinct span in the trace.
314
+ - If no trace is active at all, this activates a scoped
315
+ ``start_trace(name, record=True)`` for the duration of the factory
316
+ call only — so at minimum, HTTP calls made during construction
317
+ (e.g. an MCP ``streamable_http`` client's tool-listing request) are
318
+ captured, closing the alternative of *zero* visibility into
319
+ construction-phase bugs.
320
+
321
+ Works for both sync and async factories (``async def make_graph(...)``
322
+ is detected via :func:`inspect.iscoroutinefunction` and awaited
323
+ correctly).
324
+ """
325
+
326
+ def _decorate(fn: Callable[..., Any]) -> Callable[..., Any]:
327
+ if inspect.iscoroutinefunction(fn):
328
+
329
+ @functools.wraps(fn)
330
+ async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
331
+ if tracer.active_trace is not None:
332
+ with tracer.span(name):
333
+ return await fn(*args, **kwargs)
334
+ with tracer.start_trace(name, record=True):
335
+ return await fn(*args, **kwargs)
336
+
337
+ return async_wrapper
338
+
339
+ @functools.wraps(fn)
340
+ def wrapper(*args: Any, **kwargs: Any) -> Any:
341
+ if tracer.active_trace is not None:
342
+ with tracer.span(name):
343
+ return fn(*args, **kwargs)
344
+ with tracer.start_trace(name, record=True):
345
+ return fn(*args, **kwargs)
346
+
347
+ return wrapper
348
+
349
+ if factory is not None:
350
+ return _decorate(factory)
351
+ return _decorate
352
+
353
+
354
+ # ---------------------------------------------------------------------------
355
+ # Runtime/context capture
356
+ # ---------------------------------------------------------------------------
357
+ #
358
+ # LangGraph injects a per-run ``Runtime``/context object into node functions
359
+ # via a private config key (``CONFIG_KEY_RUNTIME`` = "__pregel_runtime")
360
+ # that is never exposed to the public ``BaseCallbackHandler`` interface —
361
+ # on_chain_start only ever receives (serialized, inputs, run_id, ...), with
362
+ # no way to see the runtime object at all. The only way to capture it is to
363
+ # read it out of the RunnableConfig at the point LangGraph itself resolves
364
+ # it, which happens inside ``RunnableCallable.invoke``/``ainvoke`` — a
365
+ # private module. This is stashed into a ContextVar immediately before
366
+ # LangGraph calls into the callback manager, so LangGraphTracer.on_chain_start
367
+ # (which fires synchronously inside that same call) can read it back out.
368
+
369
+ _current_runtime: contextvars.ContextVar[Any | None] = contextvars.ContextVar(
370
+ "agent_trace_langgraph_runtime", default=None
371
+ )
372
+
373
+ _runtime_patch_lock = threading.Lock()
374
+ _runtime_patch_installed = False
375
+
376
+
377
+ def _install_runtime_capture_patch() -> None:
378
+ """Best-effort monkeypatch that makes the LangGraph Runtime/context object
379
+ observable to LangGraphTracer.
380
+
381
+ This touches a private LangGraph module
382
+ (``langgraph._internal._runnable``/``langgraph._internal._constants``)
383
+ that may change shape across versions without notice. Every step here is
384
+ wrapped so a mismatch degrades to "no runtime captured" (the pre-existing
385
+ behavior) rather than breaking tracing or import of this module.
386
+ """
387
+ global _runtime_patch_installed # noqa: PLW0603
388
+ if _runtime_patch_installed:
389
+ return
390
+ with _runtime_patch_lock:
391
+ if _runtime_patch_installed:
392
+ return
393
+ try:
394
+ from langgraph._internal._constants import (
395
+ CONF,
396
+ CONFIG_KEY_RUNTIME,
397
+ )
398
+ from langgraph._internal._runnable import (
399
+ RunnableCallable,
400
+ )
401
+ except Exception:
402
+ logger.debug(
403
+ "agent-trace: LangGraph Runtime-capture patch unavailable "
404
+ "(internal module shape not as expected on this LangGraph "
405
+ "version); Runtime/context objects will not be captured on "
406
+ "spans.",
407
+ exc_info=True,
408
+ )
409
+ _runtime_patch_installed = True # don't retry every call
410
+ return
411
+
412
+ original_invoke = RunnableCallable.invoke
413
+ original_ainvoke = RunnableCallable.ainvoke
414
+
415
+ def _capture_runtime_from_config(config: Any) -> None:
416
+ """Best-effort: stash config's Runtime object into the ContextVar.
417
+
418
+ Swallows everything — a shape mismatch here must never break the
419
+ actual LangGraph invocation it's piggybacking on.
420
+ """
421
+ try:
422
+ if config is not None:
423
+ runtime = config.get(CONF, {}).get(CONFIG_KEY_RUNTIME)
424
+ if runtime is not None:
425
+ _current_runtime.set(runtime)
426
+ except Exception:
427
+ logger.debug(
428
+ "agent-trace: failed to read Runtime/context off config",
429
+ exc_info=True,
430
+ )
431
+
432
+ def _patched_invoke(
433
+ self: Any, input: Any, config: Any = None, **kwargs: Any
434
+ ) -> Any:
435
+ _capture_runtime_from_config(config)
436
+ return original_invoke(self, input, config, **kwargs)
437
+
438
+ async def _patched_ainvoke(
439
+ self: Any, input: Any, config: Any = None, **kwargs: Any
440
+ ) -> Any:
441
+ _capture_runtime_from_config(config)
442
+ return await original_ainvoke(self, input, config, **kwargs)
443
+
444
+ RunnableCallable.invoke = _patched_invoke # type: ignore[method-assign]
445
+ RunnableCallable.ainvoke = _patched_ainvoke # type: ignore[method-assign]
446
+ _runtime_patch_installed = True
447
+
448
+
449
+ # ---------------------------------------------------------------------------
450
+ # Exception classification — origin layer + known error signatures
451
+ # ---------------------------------------------------------------------------
452
+ #
453
+ # agent-trace's callback layer already captures exception.type/message onto
454
+ # error spans, but nothing classifies it: a developer has to read the raw
455
+ # trace.json and independently recognize a known LangGraph error code (e.g.
456
+ # ErrorCode.INVALID_CHAT_HISTORY) or figure out whether the failure came from
457
+ # the LLM provider's SDK or from application/framework code. This section
458
+ # tags every error span with a best-effort "error.origin" and, where the
459
+ # message matches a known pattern, an "error.known_pattern" attribute.
460
+
461
+ # Top-level package names recognized as an LLM-provider SDK. type(exc).__module__
462
+ # starts with one of these -> the exception originated inside the provider's
463
+ # client library (a 4xx/5xx wire error, a provider-side validation error),
464
+ # not inside the developer's own chain/application code.
465
+ _PROVIDER_MODULE_PREFIXES: frozenset[str] = frozenset(
466
+ {
467
+ "openai",
468
+ "anthropic",
469
+ "groq",
470
+ "google",
471
+ "genai",
472
+ "cohere",
473
+ "mistralai",
474
+ "boto3",
475
+ "botocore",
476
+ }
477
+ )
478
+
479
+ # Top-level package names recognized as framework/orchestration code, as
480
+ # distinct from the developer's own application code.
481
+ _CHAIN_MODULE_PREFIXES: frozenset[str] = frozenset(
482
+ {"langgraph", "langchain_core", "langchain", "langchain_community"}
483
+ )
484
+
485
+ # (message substring, label) pairs for exception messages that match a known,
486
+ # previously root-caused failure signature. Matched case-insensitively
487
+ # against str(exc). Order matters only in that the first match wins, but
488
+ # entries are written to be mutually exclusive in practice.
489
+ _KNOWN_ERROR_SIGNATURES: tuple[tuple[str, str], ...] = (
490
+ (
491
+ "invalid_chat_history",
492
+ "langgraph_invalid_chat_history",
493
+ ),
494
+ (
495
+ "selected invalid tool",
496
+ "middleware_invalid_tool_selection",
497
+ ),
498
+ )
499
+
500
+
501
+ def _classify_exception_origin(error: BaseException) -> str:
502
+ """Best-effort "which layer did this exception come from" tag.
503
+
504
+ Returns "provider" (an LLM-SDK-raised exception — e.g. an OpenAI/
505
+ Anthropic/Groq 4xx), "chain" (LangGraph/LangChain framework code), or
506
+ "application" (everything else — the developer's own node/tool code, the
507
+ most common case for a genuine bug).
508
+ """
509
+ module = type(error).__module__ or ""
510
+ top_level = module.split(".", 1)[0]
511
+ if top_level in _PROVIDER_MODULE_PREFIXES:
512
+ return "provider"
513
+ if top_level in _CHAIN_MODULE_PREFIXES:
514
+ return "chain"
515
+ return "application"
516
+
517
+
518
+ def _match_known_error_signature(message: str) -> str | None:
519
+ """Return a short label if *message* matches a known, previously
520
+ root-caused failure signature, else None."""
521
+ if not message:
522
+ return None
523
+ lowered = message.lower()
524
+ for needle, label in _KNOWN_ERROR_SIGNATURES:
525
+ if needle in lowered:
526
+ return label
527
+ return None
528
+
529
+
530
+ def _classify_and_tag_exception(span: Span, error: BaseException) -> None:
531
+ """Apply error.origin + error.known_pattern attributes to *span*.
532
+
533
+ Called after ``span.record_exception`` for any span closing ERROR, so
534
+ the classification is available directly on the span instead of
535
+ requiring a developer to read exception.message out of raw trace.json
536
+ and recognize the pattern themselves.
537
+ """
538
+ span.set_attribute("error.origin", _classify_exception_origin(error))
539
+ known_pattern = _match_known_error_signature(str(error))
540
+ if known_pattern:
541
+ span.set_attribute("error.known_pattern", known_pattern)
542
+
543
+
544
+ # ---------------------------------------------------------------------------
545
+ # LangGraph internal control-flow signals — not application errors
546
+ # ---------------------------------------------------------------------------
547
+ #
548
+ # LangGraph raises its own exceptions internally to implement control flow
549
+ # that has nothing to do with application failure:
550
+ # - ParentCommand: raised when a node returns Command(graph=Command.PARENT,
551
+ # ...) to implement a multi-agent handoff jump up to the parent graph.
552
+ # - GraphInterrupt: raised when a node calls interrupt() to pause a run
553
+ # for human-in-the-loop resumption.
554
+ # Both subclass langgraph.errors.GraphBubbleUp (confirmed against the
555
+ # installed langgraph package). Without special-casing these, on_chain_error
556
+ # marks the node span ERROR with exception.type=ParentCommand/GraphInterrupt,
557
+ # identical in shape to a genuine application exception — a developer has to
558
+ # manually filter these out before finding the real error in a trace.
559
+
560
+ _control_flow_exception_types: tuple[type[BaseException], ...] | None = None
561
+ _control_flow_types_lock = threading.Lock()
562
+
563
+
564
+ def _get_control_flow_exception_types() -> tuple[type[BaseException], ...]:
565
+ """Lazily resolve LangGraph's internal control-flow exception types.
566
+
567
+ Tries the shared ``GraphBubbleUp`` base first (covers both
568
+ ``ParentCommand`` and ``GraphInterrupt`` in one isinstance check on
569
+ LangGraph versions that have it). Falls back to importing the two known
570
+ concrete types individually for versions where no shared base exists.
571
+ Every import is wrapped so a version mismatch degrades to "nothing
572
+ special-cased" (the pre-existing behavior) rather than breaking tracing.
573
+ """
574
+ global _control_flow_exception_types # noqa: PLW0603
575
+ if _control_flow_exception_types is not None:
576
+ return _control_flow_exception_types
577
+ with _control_flow_types_lock:
578
+ if _control_flow_exception_types is not None:
579
+ return _control_flow_exception_types
580
+
581
+ found: list[type[BaseException]] = []
582
+ try:
583
+ from langgraph.errors import GraphBubbleUp
584
+
585
+ found.append(GraphBubbleUp)
586
+ except Exception:
587
+ for type_name in ("ParentCommand", "GraphInterrupt"):
588
+ try:
589
+ from langgraph import errors as _lg_errors
590
+
591
+ found.append(getattr(_lg_errors, type_name))
592
+ except Exception:
593
+ logger.debug(
594
+ "agent-trace: could not import langgraph.errors.%s "
595
+ "on this LangGraph version; it will not be "
596
+ "special-cased as a control-flow signal.",
597
+ type_name,
598
+ exc_info=True,
599
+ )
600
+ _control_flow_exception_types = tuple(found)
601
+ return _control_flow_exception_types
602
+
603
+
604
+ def _is_langgraph_control_flow_signal(error: BaseException) -> bool:
605
+ """True if *error* is LangGraph's own internal control-flow signal
606
+ (a Command/ParentCommand handoff jump or a GraphInterrupt pause) rather
607
+ than an application-level exception."""
608
+ types_ = _get_control_flow_exception_types()
609
+ return bool(types_) and isinstance(error, types_)
610
+
611
+
612
+ def _record_control_flow_signal(span: Span, error: BaseException) -> None:
613
+ """Close *span* OK with an informational attribute instead of ERROR.
614
+
615
+ Distinguishes a GraphInterrupt (run paused, not failed) from a
616
+ ParentCommand/other handoff jump via separate boolean attributes, since
617
+ the two mean different things to a developer reading the trace.
618
+ """
619
+ type_name = type(error).__name__
620
+ span.set_attribute("langgraph.control_flow_signal", type_name)
621
+ if type_name == "GraphInterrupt":
622
+ span.set_attribute("langgraph.interrupted", True)
623
+ else:
624
+ span.set_attribute("langgraph.handoff", True)
625
+ span.add_event(
626
+ "langgraph_control_flow",
627
+ attributes={
628
+ "control_flow.type": type_name,
629
+ "control_flow.message": _safe_str(error)[:_MAX_ATTR_LEN],
630
+ },
631
+ )
632
+
633
+
634
+ # ---------------------------------------------------------------------------
635
+ # Branch (conditional-edge) dispatch exception capture
636
+ # ---------------------------------------------------------------------------
637
+ #
638
+ # LangGraph builds a conditional edge's routing dispatch as a RunnableCallable
639
+ # constructed with trace=False (langgraph/graph/_branch.py, BranchSpec.run()):
640
+ # a deliberate choice by LangGraph itself so the dispatch step doesn't show up
641
+ # as its own chain span. RunnableCallable.invoke/ainvoke skip the callback
642
+ # manager entirely when self.trace is falsy (langgraph/_internal/_runnable.py)
643
+ # — no on_chain_start/on_chain_error ever fires for this component, so an
644
+ # exception raised inside it (e.g. a KeyError from BranchSpec._finish() when a
645
+ # router's return value doesn't match a registered destination) produces zero
646
+ # agent-trace spans or callback events today.
647
+ #
648
+ # The capture point has to be RunnableCallable.invoke/ainvoke themselves (the
649
+ # same class the Runtime-capture patch above already wraps), NOT
650
+ # BranchSpec._route/_aroute directly: BranchSpec.run() captures
651
+ # `func=self._route`/`afunc=self._aroute` as bound-method values baked into a
652
+ # RunnableCallable instance at *graph-compile time* (builder.compile()) —
653
+ # typically long before any LangGraphTracer is ever constructed. Patching
654
+ # BranchSpec._route/_aroute as class attributes only affects bound-method
655
+ # lookups that happen *after* the patch installs; a RunnableCallable compiled
656
+ # earlier already holds a direct reference to the pre-patch function and would
657
+ # never observe a later patch. RunnableCallable.invoke/ainvoke, by contrast,
658
+ # are resolved fresh via normal method lookup every time `.invoke()`/
659
+ # `.ainvoke()` is called on any instance — patching them here (regardless of
660
+ # when any given RunnableCallable was constructed) reliably intercepts every
661
+ # call, exactly like the Runtime-capture patch above already relies on.
662
+
663
+ _current_langgraph_tracer: contextvars.ContextVar[Any | None] = contextvars.ContextVar(
664
+ "agent_trace_langgraph_current_tracer", default=None
665
+ )
666
+
667
+ _branch_patch_lock = threading.Lock()
668
+ _branch_patch_installed = False
669
+
670
+
671
+ def _is_branch_dispatch_callable(runnable_callable: Any, branch_spec_cls: type) -> bool:
672
+ """True if *runnable_callable* wraps a BranchSpec._route/_aroute bound
673
+ method — i.e. this is LangGraph's conditional-edge dispatch step, not one
674
+ of the other unrelated trace=False RunnableCallables LangGraph also
675
+ constructs internally (channel writes, ToolNode, etc.)."""
676
+ for attr_name in ("func", "afunc"):
677
+ bound_method = getattr(runnable_callable, attr_name, None)
678
+ bound_owner = getattr(bound_method, "__self__", None)
679
+ if isinstance(bound_owner, branch_spec_cls):
680
+ return True
681
+ return False
682
+
683
+
684
+ def _branch_spec_instance(runnable_callable: Any) -> Any | None:
685
+ """Return the BranchSpec instance a dispatch RunnableCallable wraps, or
686
+ None. Shared by the router-name and registered-destinations lookups
687
+ below so there's exactly one place that reaches into
688
+ ``.func``/``.afunc.__self__``."""
689
+ bound_method = getattr(runnable_callable, "func", None) or getattr(
690
+ runnable_callable, "afunc", None
691
+ )
692
+ return getattr(bound_method, "__self__", None)
693
+
694
+
695
+ def _resolve_router_name(runnable_callable: Any) -> str:
696
+ """Best-effort: the user's own router function name (e.g.
697
+ ``should_continue``) for a Branch dispatch RunnableCallable —
698
+ ``BranchSpec.path`` is the Runnable wrapping whatever callable was
699
+ passed to ``add_conditional_edges``, typically a ``RunnableLambda``
700
+ whose ``.func``/``.afunc`` is the original function object. Falls back
701
+ to ``"<anonymous>"`` for a lambda or when the shape doesn't match
702
+ (private LangGraph/LangChain internals, may change across versions).
703
+ """
704
+ branch_self = _branch_spec_instance(runnable_callable)
705
+ path = getattr(branch_self, "path", None)
706
+ underlying = getattr(path, "func", None) or getattr(path, "afunc", None) or path
707
+ name = getattr(underlying, "__name__", None)
708
+ return str(name) if name else "<anonymous>"
709
+
710
+
711
+ def _registered_destinations(runnable_callable: Any) -> str:
712
+ """Comma-joined set of destinations this Branch could route to (its
713
+ ``path_map``/``ends``), or "" if unavailable."""
714
+ branch_self = _branch_spec_instance(runnable_callable)
715
+ ends = getattr(branch_self, "ends", None) or {}
716
+ return ",".join(str(v) for v in ends.values())
717
+
718
+
719
+ def _record_branch_dispatch(
720
+ runnable_callable: Any, *, error: BaseException | None
721
+ ) -> None:
722
+ """Open a standalone ``branch:dispatch`` span recording which
723
+ conditional-edge/router function made a routing decision — on both the
724
+ success and failure path — since LangGraph deliberately builds this
725
+ component with ``trace=False`` and no ``on_chain_start``/``on_chain_end``/
726
+ ``on_chain_error`` ever fires for it otherwise.
727
+
728
+ Without this, a tool-call span's downstream exception (e.g. a
729
+ ``ValidationError`` on a missing injected field) looks identical
730
+ regardless of whether the cause was a routing bug that dispatched to the
731
+ wrong node, or genuinely malformed model output — a developer has no
732
+ record of which router ran, or that it ran at all.
733
+
734
+ Best-effort: swallows every exception itself (an instrumentation bug
735
+ here must never break — or change the exception raised by — the real
736
+ LangGraph routing dispatch it's piggybacking on). No-ops entirely if no
737
+ LangGraphTracer instance is active in this context (e.g. the graph was
738
+ invoked without one).
739
+ """
740
+ handler = _current_langgraph_tracer.get()
741
+ if handler is None:
742
+ return
743
+ try:
744
+ span = handler._tracer.start_span("branch:dispatch")
745
+ span.set_attribute("langgraph.branch_dispatch", True)
746
+ router_name = _resolve_router_name(runnable_callable)
747
+ span.set_attribute("branch.router_name", router_name)
748
+ destinations = _registered_destinations(runnable_callable)
749
+ if destinations:
750
+ span.set_attribute("branch.registered_destinations", destinations)
751
+ if error is not None:
752
+ span.record_exception(error)
753
+ _classify_and_tag_exception(span, error)
754
+ span.end(SpanStatus.ERROR)
755
+ else:
756
+ span.end(SpanStatus.OK)
757
+ except Exception:
758
+ logger.debug(
759
+ "agent-trace: failed to record Branch dispatch",
760
+ exc_info=True,
761
+ )
762
+
763
+
764
+ def _install_branch_dispatch_capture_patch() -> None:
765
+ """Best-effort monkeypatch making LangGraph's trace=False Branch dispatch
766
+ (conditional-edge routing) observable to agent-trace — both which router
767
+ ran and, on failure, why it raised.
768
+
769
+ Touches private-ish LangGraph modules (``langgraph.graph._branch``,
770
+ ``langgraph._internal._runnable``) that may change shape across versions
771
+ without notice; every step here is wrapped so a mismatch degrades to
772
+ "dispatch not captured" (the pre-existing behavior) rather than breaking
773
+ tracing or import.
774
+ """
775
+ global _branch_patch_installed # noqa: PLW0603
776
+ if _branch_patch_installed:
777
+ return
778
+ with _branch_patch_lock:
779
+ if _branch_patch_installed:
780
+ return
781
+ try:
782
+ from langgraph._internal._runnable import RunnableCallable
783
+ from langgraph.graph._branch import BranchSpec
784
+ except Exception:
785
+ logger.debug(
786
+ "agent-trace: LangGraph Branch dispatch capture patch "
787
+ "unavailable (internal module shape not as expected on "
788
+ "this LangGraph version); conditional-edge routing "
789
+ "dispatch will not be captured.",
790
+ exc_info=True,
791
+ )
792
+ _branch_patch_installed = True # don't retry every call
793
+ return
794
+
795
+ original_invoke = RunnableCallable.invoke
796
+ original_ainvoke = RunnableCallable.ainvoke
797
+
798
+ def _patched_invoke(
799
+ self: Any, input: Any, config: Any = None, **kwargs: Any
800
+ ) -> Any:
801
+ is_branch = not self.trace and _is_branch_dispatch_callable(
802
+ self, BranchSpec
803
+ )
804
+ try:
805
+ result = original_invoke(self, input, config, **kwargs)
806
+ except BaseException as exc:
807
+ if is_branch:
808
+ _record_branch_dispatch(self, error=exc)
809
+ raise
810
+ if is_branch:
811
+ _record_branch_dispatch(self, error=None)
812
+ return result
813
+
814
+ async def _patched_ainvoke(
815
+ self: Any, input: Any, config: Any = None, **kwargs: Any
816
+ ) -> Any:
817
+ is_branch = not self.trace and _is_branch_dispatch_callable(
818
+ self, BranchSpec
819
+ )
820
+ try:
821
+ result = await original_ainvoke(self, input, config, **kwargs)
822
+ except BaseException as exc:
823
+ if is_branch:
824
+ _record_branch_dispatch(self, error=exc)
825
+ raise
826
+ if is_branch:
827
+ _record_branch_dispatch(self, error=None)
828
+ return result
829
+
830
+ RunnableCallable.invoke = _patched_invoke # type: ignore[method-assign]
831
+ RunnableCallable.ainvoke = _patched_ainvoke # type: ignore[method-assign]
832
+ _branch_patch_installed = True
833
+
834
+
835
+ # ---------------------------------------------------------------------------
836
+ # Tool-argument injection (InjectedState/InjectedStore) capture
837
+ # ---------------------------------------------------------------------------
838
+ #
839
+ # ToolNode._inject_tool_args() resolves Annotated[..., InjectedState]/
840
+ # InjectedStore/InjectedToolCallId tool arguments right before the tool is
841
+ # actually invoked — confirmed via direct inspection of the installed
842
+ # langgraph.prebuilt.tool_node module: it is called via normal instance
843
+ # method lookup (self._inject_tool_args(...)) from ToolNode's own dispatch
844
+ # code, so patching the class method (like RunnableCallable.invoke/ainvoke
845
+ # above) reliably intercepts every call regardless of when any given
846
+ # ToolNode instance was constructed. Nothing in agent_trace previously
847
+ # observed this step at all (confirmed via repo-wide grep: zero hits for
848
+ # InjectedState/InjectedStore/inject_tool_args before this patch) — a
849
+ # ValidationError raised later inside the tool call itself looks identical
850
+ # whether the cause was a routing bug that bypassed injection, a caller
851
+ # passing a malformed tool schema, or genuinely malformed model output.
852
+ #
853
+ # _inject_tool_args runs *before* the tool's own .invoke()/.ainvoke() call,
854
+ # which is what fires on_tool_start — so there is no open tool: span yet to
855
+ # attach to at the point this patch observes injection. Recorded as its own
856
+ # standalone span instead (the same pattern as branch:dispatch above), whose
857
+ # start_time necessarily precedes the corresponding tool: span's.
858
+
859
+ _tool_injection_patch_lock = threading.Lock()
860
+ _tool_injection_patch_installed = False
861
+
862
+
863
+ def _record_tool_arg_injection(tool_name: str, injected_keys: list[str]) -> None:
864
+ """Open a standalone ``tool_inject:<tool_name>`` span recording whether
865
+ InjectedState/InjectedStore/InjectedToolCallId resolution ran for
866
+ *tool_name*, and which argument names it resolved.
867
+
868
+ Deliberately named with a ``tool_inject:`` prefix rather than
869
+ ``tool:...`` — code/tests filtering spans by ``name.startswith("tool:")``
870
+ to find the *actual* tool-call span must not also match this one, since
871
+ this span always precedes (and is distinct from) the real ``tool:<name>``
872
+ span for the same call (injection runs before on_tool_start fires).
873
+
874
+ Best-effort: swallows every exception itself. No-ops entirely if no
875
+ LangGraphTracer instance is active in this context.
876
+ """
877
+ handler = _current_langgraph_tracer.get()
878
+ if handler is None:
879
+ return
880
+ try:
881
+ span = handler._tracer.start_span(f"tool_inject:{tool_name}")
882
+ span.set_attribute("tool.name", tool_name)
883
+ span.set_attribute("tool.injection_ran", bool(injected_keys))
884
+ if injected_keys:
885
+ span.set_attribute("tool.injected_arg_keys", ",".join(injected_keys))
886
+ span.end(SpanStatus.OK)
887
+ except Exception:
888
+ logger.debug(
889
+ "agent-trace: failed to record tool-argument injection for "
890
+ "tool %r",
891
+ tool_name,
892
+ exc_info=True,
893
+ )
894
+
895
+
896
+ def _install_tool_arg_injection_capture_patch() -> None:
897
+ """Best-effort monkeypatch making ``ToolNode``'s tool-argument injection
898
+ step observable to agent-trace.
899
+
900
+ Touches ``langgraph.prebuilt.tool_node.ToolNode._inject_tool_args``,
901
+ which may change shape across LangGraph versions without notice; every
902
+ step here is wrapped so a mismatch degrades to "injection not captured"
903
+ (the pre-existing behavior) rather than breaking tracing, import, or a
904
+ real tool call.
905
+ """
906
+ global _tool_injection_patch_installed # noqa: PLW0603
907
+ if _tool_injection_patch_installed:
908
+ return
909
+ with _tool_injection_patch_lock:
910
+ if _tool_injection_patch_installed:
911
+ return
912
+ try:
913
+ from langgraph.prebuilt.tool_node import ToolNode
914
+ except Exception:
915
+ logger.debug(
916
+ "agent-trace: LangGraph tool-argument injection capture "
917
+ "patch unavailable (ToolNode._inject_tool_args not found "
918
+ "on this LangGraph version); InjectedState/InjectedStore "
919
+ "resolution will not be captured.",
920
+ exc_info=True,
921
+ )
922
+ _tool_injection_patch_installed = True # don't retry every call
923
+ return
924
+
925
+ original_inject = ToolNode._inject_tool_args
926
+
927
+ def _patched_inject_tool_args(
928
+ self: Any, tool_call: Any, tool_runtime: Any, tool: Any = None
929
+ ) -> Any:
930
+ result = original_inject(self, tool_call, tool_runtime, tool)
931
+ try:
932
+ tool_name = tool_call.get("name", "") if tool_call else ""
933
+ injected = (self._injected_args or {}).get(tool_name)
934
+ injected_keys = (
935
+ sorted(str(k) for k in injected.all_injected_keys)
936
+ if injected is not None
937
+ else []
938
+ )
939
+ _record_tool_arg_injection(tool_name, injected_keys)
940
+ except Exception:
941
+ logger.debug(
942
+ "agent-trace: failed to observe tool-argument "
943
+ "injection outcome",
944
+ exc_info=True,
945
+ )
946
+ return result
947
+
948
+ ToolNode._inject_tool_args = _patched_inject_tool_args # type: ignore[method-assign]
949
+ _tool_injection_patch_installed = True
950
+
951
+
952
+ # ---------------------------------------------------------------------------
953
+ # on_llm_end helpers — pulled out to keep the callback itself flat/scannable
954
+ # ---------------------------------------------------------------------------
955
+
956
+
957
+ def _first_generation_and_message(response: Any) -> tuple[Any, Any]:
958
+ """Return (first_generation, first_generation.message) from a
959
+ langchain-core LLMResult/ChatResult, or (None, None) if absent."""
960
+ generations = getattr(response, "generations", None) or []
961
+ if not generations or not generations[0]:
962
+ return None, None
963
+ first_gen = generations[0][0]
964
+ first_message = getattr(first_gen, "message", None)
965
+ return first_gen, first_message
966
+
967
+
968
+ def _extract_token_usage(response: Any, first_message: Any) -> dict[str, Any]:
969
+ """token_usage from llm_output, falling back to the first message's
970
+ usage_metadata — modern langchain-core often attaches usage there
971
+ independently of llm_output, especially under streaming configurations.
972
+ """
973
+ usage = getattr(response, "llm_output", {}) or {}
974
+ token_usage = usage.get("token_usage") or usage.get("usage") or {}
975
+ if token_usage or first_message is None:
976
+ return token_usage
977
+ msg_usage = getattr(first_message, "usage_metadata", None)
978
+ if not msg_usage:
979
+ return {}
980
+ return {
981
+ "prompt_tokens": msg_usage.get("input_tokens", 0),
982
+ "completion_tokens": msg_usage.get("output_tokens", 0),
983
+ "total_tokens": msg_usage.get("total_tokens", 0),
984
+ }
985
+
986
+
987
+ def _extract_bound_tools(kwargs: dict[str, Any]) -> Any:
988
+ """Return the tools/functions schema bound to this specific LLM call
989
+ (i.e. what LangChain actually sent the provider as `tools`/
990
+ `functions`), or None if none were bound.
991
+
992
+ Read from the `invocation_params` dict LangChain's own callback
993
+ manager passes as a kwarg to on_chat_model_start/on_llm_start
994
+ (confirmed via direct inspection against the installed langchain-core:
995
+ `invocation_params={'tools': [...], ...}` when `.bind_tools(...)` was
996
+ used) — #5665's ask, so a nested LLM span can show not just *that* a
997
+ tool-nested call happened but *what* it was bound to (e.g. a
998
+ `with_structured_output(SimpleSchema)` schema leaking in as a bound
999
+ tool on an unexpected call)."""
1000
+ invocation_params = kwargs.get("invocation_params")
1001
+ if not isinstance(invocation_params, dict):
1002
+ return None
1003
+ return invocation_params.get("tools") or invocation_params.get("functions")
1004
+
1005
+
1006
+ def _extract_finish_reason(
1007
+ response_metadata: dict[str, Any] | None,
1008
+ generation_info: dict[str, Any] | None,
1009
+ ) -> str | None:
1010
+ """finish_reason (or stop_reason) from response_metadata, falling back to
1011
+ generation_info — e.g. Gemini's finish_reason=MALFORMED_FUNCTION_CALL,
1012
+ OpenAI's finish_reason/tool_calls presence signal."""
1013
+ for source in (response_metadata, generation_info):
1014
+ if not source:
1015
+ continue
1016
+ reason = source.get("finish_reason") or source.get("stop_reason")
1017
+ if reason:
1018
+ return str(reason)
1019
+ return None
1020
+
1021
+
1022
+ def _extract_content(first_message: Any, first_gen: Any) -> Any:
1023
+ """Actual generated content — not just usage counts.
1024
+
1025
+ Chat models: response.generations[0][0].message.content
1026
+ Legacy completions: response.generations[0][0].text
1027
+ """
1028
+ if first_message is not None:
1029
+ return getattr(first_message, "content", None)
1030
+ if first_gen is not None:
1031
+ return getattr(first_gen, "text", None)
1032
+ return None
1033
+
1034
+
1035
+ def _record_llm_end_data(span: Span, response: Any) -> None:
1036
+ """Extract and persist everything on_llm_end previously discarded:
1037
+ token usage (with the usage_metadata fallback), response_metadata/
1038
+ generation_info (finish_reason, tool-call presence), and the actual
1039
+ generated content."""
1040
+ first_gen, first_message = _first_generation_and_message(response)
1041
+
1042
+ token_usage = _extract_token_usage(response, first_message)
1043
+ if token_usage:
1044
+ span.set_attribute(
1045
+ "llm.usage.prompt_tokens", int(token_usage.get("prompt_tokens", 0))
1046
+ )
1047
+ span.set_attribute(
1048
+ "llm.usage.completion_tokens",
1049
+ int(token_usage.get("completion_tokens", 0)),
1050
+ )
1051
+ span.set_attribute(
1052
+ "llm.usage.total_tokens", int(token_usage.get("total_tokens", 0))
1053
+ )
1054
+
1055
+ response_metadata = (
1056
+ getattr(first_message, "response_metadata", None)
1057
+ if first_message is not None
1058
+ else None
1059
+ )
1060
+ generation_info = getattr(first_gen, "generation_info", None)
1061
+
1062
+ finish_reason = _extract_finish_reason(response_metadata, generation_info)
1063
+ if finish_reason:
1064
+ span.set_attribute("llm.finish_reason", finish_reason)
1065
+ if first_message is not None:
1066
+ span.set_attribute(
1067
+ "llm.has_tool_calls", bool(getattr(first_message, "tool_calls", None))
1068
+ )
1069
+ if response_metadata:
1070
+ span.set_attribute("llm.response_metadata", _to_attr_string(response_metadata))
1071
+ if generation_info:
1072
+ span.set_attribute("llm.generation_info", _to_attr_string(generation_info))
1073
+
1074
+ content = _extract_content(first_message, first_gen)
1075
+ if content:
1076
+ span.set_attribute("llm.content", _stringify(content))
1077
+
1078
+
1079
+ def _extract_tool_call_chunks(chunk: Any) -> Any:
1080
+ """Best-effort extraction of a streaming ChatGenerationChunk's
1081
+ ``message.tool_call_chunks`` (the partial/incremental tool-call-argument
1082
+ fragments LangChain attaches per streamed delta), or None if *chunk* is
1083
+ absent or isn't that shape (e.g. a plain GenerationChunk from a legacy,
1084
+ non-chat LLM)."""
1085
+ message = getattr(chunk, "message", None)
1086
+ if message is None:
1087
+ return None
1088
+ return getattr(message, "tool_call_chunks", None) or None
1089
+
1090
+
1091
+ def _get_declared_node_tags(graph: Any, node_name: str) -> list[str] | None:
1092
+ """Best-effort: read a compiled LangGraph graph's node-level *declared*
1093
+ tags — the tags a developer attached to the node's own action/runnable
1094
+ at graph-construction time (e.g. ``builder.add_node("n",
1095
+ my_fn.with_config(tags=["nostream"]))``) — as distinct from the purely
1096
+ LangGraph-internal *runtime* tags (e.g. ``"graph:step:2"``) that
1097
+ ``on_chain_start``'s own ``tags`` kwarg already carries for every node
1098
+ run, which never include a developer-declared tag like ``"nostream"``.
1099
+
1100
+ How this actually works (confirmed by direct inspection of the
1101
+ installed LangGraph, not assumed from docs): the current
1102
+ ``StateGraph.add_node()`` has **no** ``tags=`` keyword argument at all —
1103
+ a node's own declared tags only exist if the developer wrapped the
1104
+ node's action in ``.with_config(tags=[...])`` *before* passing it to
1105
+ ``add_node()``. LangGraph's own ``PregelNode.tags`` field (whose
1106
+ docstring says "Tags to attach to the node for tracing") is never
1107
+ actually populated by ``StateGraph`` for a regular node — it stays
1108
+ ``None`` regardless of what was declared. The tags survive only on the
1109
+ compiled node's own bound Runnable's ``.config`` dict, reachable at
1110
+ ``graph.nodes[node_name].bound.config.get("tags")``.
1111
+
1112
+ Wrapped entirely in try/except: this reaches into private-ish LangGraph
1113
+ attributes (``.nodes``, ``.bound``, ``.config``) that may not exist, or
1114
+ may be shaped differently, on other LangGraph versions — a mismatch
1115
+ degrades to "no declared tags captured" (returns None), never an
1116
+ exception raised into the caller's ``graph.invoke()``/``.stream()``
1117
+ call.
1118
+ """
1119
+ try:
1120
+ nodes = getattr(graph, "nodes", None)
1121
+ if nodes is None:
1122
+ return None
1123
+ node = nodes.get(node_name)
1124
+ if node is None:
1125
+ return None
1126
+ bound = getattr(node, "bound", None)
1127
+ config = getattr(bound, "config", None) or {}
1128
+ tags = config.get("tags")
1129
+ if not tags:
1130
+ return None
1131
+ return [str(t) for t in tags]
1132
+ except Exception:
1133
+ logger.debug(
1134
+ "agent-trace: failed to read declared node tags for node %r",
1135
+ node_name,
1136
+ exc_info=True,
1137
+ )
1138
+ return None
1139
+
1140
+
1141
+ # ---------------------------------------------------------------------------
1142
+ # Schema-level check: tool parameters shaped like framework state that
1143
+ # aren't marked injected
1144
+ # ---------------------------------------------------------------------------
1145
+ #
1146
+ # A bound tool's *model-facing* JSON schema (BaseTool.args, which LangChain
1147
+ # itself already computes with every Annotated[..., InjectedState]/
1148
+ # InjectedStore/InjectedToolCallId parameter excluded — confirmed by direct
1149
+ # inspection: a param annotated InjectedState never appears in tool.args)
1150
+ # tells us exactly which parameters the LLM will have to invent a value for
1151
+ # at call time. Comparing those names against the compiled graph's own state
1152
+ # field names (StateGraph.builder.schemas) encodes, as an automatable rule,
1153
+ # the diagnosis a LangGraph maintainer made purely by inspection in issue
1154
+ # #3266 ("the LLM is populating the state field and hallucinating the video
1155
+ # path"): a tool parameter that happens to share a name with a real state
1156
+ # field, but was never excluded from the model-facing schema via
1157
+ # InjectedState, is a parameter the model will fill in itself rather than
1158
+ # receive from real graph state.
1159
+
1160
+
1161
+ def _state_field_names(graph: Any) -> set[str]:
1162
+ """Best-effort: every field name declared across a compiled StateGraph's
1163
+ state schema(s) (``builder.schemas`` maps each schema class to a dict of
1164
+ channel-name -> Channel). Returns an empty set if the shape doesn't
1165
+ match (private-ish LangGraph attribute, may change across versions)."""
1166
+ try:
1167
+ schemas = getattr(getattr(graph, "builder", None), "schemas", None) or {}
1168
+ names: set[str] = set()
1169
+ for channel_map in schemas.values():
1170
+ names.update(str(k) for k in channel_map)
1171
+ return names
1172
+ except Exception:
1173
+ logger.debug(
1174
+ "agent-trace: failed to read state schema field names off graph",
1175
+ exc_info=True,
1176
+ )
1177
+ return set()
1178
+
1179
+
1180
+ def _tools_by_name_for_node(node: Any) -> dict[str, Any]:
1181
+ """Best-effort: the ``{tool_name: BaseTool}`` mapping a compiled graph
1182
+ node runs, if that node is (or wraps) a ``ToolNode`` — read off the
1183
+ node's own ``bound`` Runnable's ``_tools_by_name`` attribute. Returns {}
1184
+ for any node that isn't a ToolNode (the overwhelmingly common case —
1185
+ most nodes aren't) or if the shape doesn't match."""
1186
+ try:
1187
+ bound = getattr(node, "bound", None)
1188
+ tools_by_name = getattr(bound, "_tools_by_name", None)
1189
+ if not tools_by_name:
1190
+ return {}
1191
+ return dict(tools_by_name)
1192
+ except Exception:
1193
+ return {}
1194
+
1195
+
1196
+ def find_tool_params_shaped_like_state(graph: Any) -> list[dict[str, str]]:
1197
+ """Flag tool parameters that are model-facing (not excluded via
1198
+ InjectedState/InjectedStore/InjectedToolCallId) but share a name with
1199
+ one of the compiled graph's own state fields.
1200
+
1201
+ Returns a list of ``{"node": ..., "tool": ..., "param": ...}`` findings
1202
+ (empty if none, or if the graph/state-schema shape can't be introspected
1203
+ — e.g. *graph* isn't a compiled LangGraph ``CompiledStateGraph``).
1204
+ Read-only: never mutates *graph* or raises into the caller.
1205
+ """
1206
+ findings: list[dict[str, str]] = []
1207
+ try:
1208
+ state_fields = _state_field_names(graph)
1209
+ if not state_fields:
1210
+ return findings
1211
+ nodes = getattr(graph, "nodes", None) or {}
1212
+ for node_name, node in nodes.items():
1213
+ tools_by_name = _tools_by_name_for_node(node)
1214
+ for tool_name, tool_obj in tools_by_name.items():
1215
+ model_facing_args = getattr(tool_obj, "args", None) or {}
1216
+ for param_name in model_facing_args:
1217
+ if param_name in state_fields:
1218
+ findings.append(
1219
+ {
1220
+ "node": str(node_name),
1221
+ "tool": str(tool_name),
1222
+ "param": str(param_name),
1223
+ }
1224
+ )
1225
+ except Exception:
1226
+ logger.debug(
1227
+ "agent-trace: failed to check tool params against state schema",
1228
+ exc_info=True,
1229
+ )
1230
+ return findings
1231
+
1232
+
1233
+ def _require_langchain_core() -> Any:
1234
+ """Lazy import guard — raises a clear error if langchain_core is absent."""
1235
+ try:
1236
+ import langchain_core
1237
+
1238
+ return langchain_core
1239
+ except ImportError as exc:
1240
+ raise ImportError(_INSTALL_HINT) from exc
1241
+
1242
+
1243
+ def _base_callback_handler() -> Any:
1244
+ """Return langchain_core.callbacks.BaseCallbackHandler (lazy import)."""
1245
+ _require_langchain_core()
1246
+ from langchain_core.callbacks import BaseCallbackHandler
1247
+
1248
+ return BaseCallbackHandler
1249
+
1250
+
1251
+ # Module-level singleton: the concrete tracer class is built once (the first
1252
+ # time _get_tracer_class() is called) so that BaseCallbackHandler is a real
1253
+ # base at class-definition time rather than being spliced in at instantiation
1254
+ # via __bases__ mutation (which is unsafe under concurrency and forbidden in
1255
+ # Python 3.14+).
1256
+ _LangGraphTracerClass: type | None = None
1257
+ _tracer_class_lock: threading.Lock = threading.Lock()
1258
+
1259
+
1260
+ def _get_tracer_class() -> type:
1261
+ """Return (and lazily build) the concrete LangGraphTracer implementation.
1262
+
1263
+ The class is created exactly once, with ``BaseCallbackHandler`` as a
1264
+ genuine base class at definition time. Subsequent calls return the cached
1265
+ class without re-importing or re-defining anything.
1266
+ """
1267
+ global _LangGraphTracerClass # noqa: PLW0603
1268
+ if _LangGraphTracerClass is not None:
1269
+ return _LangGraphTracerClass
1270
+
1271
+ with _tracer_class_lock:
1272
+ # Double-checked locking: re-test inside the lock.
1273
+ if _LangGraphTracerClass is not None:
1274
+ return _LangGraphTracerClass
1275
+
1276
+ base_cls: type = _base_callback_handler()
1277
+ # Best-effort — makes the injected Runtime/context object observable
1278
+ # to on_chain_start below. No-ops safely if the private LangGraph
1279
+ # internals it depends on aren't present.
1280
+ _install_runtime_capture_patch()
1281
+ # Best-effort — makes Branch (conditional-edge) routing dispatch
1282
+ # (which router ran, and on failure why it raised) observable
1283
+ # despite LangGraph building that component with trace=False.
1284
+ # No-ops safely if the internals it depends on aren't present.
1285
+ _install_branch_dispatch_capture_patch()
1286
+ # Best-effort — makes ToolNode's InjectedState/InjectedStore
1287
+ # argument-injection step observable. No-ops safely if the
1288
+ # internals it depends on aren't present.
1289
+ _install_tool_arg_injection_capture_patch()
1290
+
1291
+ class _LangGraphTracerImpl(base_cls): # type: ignore[misc]
1292
+ """Concrete implementation — see LangGraphTracer for public docs."""
1293
+
1294
+ def __init__(
1295
+ self,
1296
+ tracer: Tracer,
1297
+ trace: Trace,
1298
+ *,
1299
+ graph: Any = None,
1300
+ long_span_threshold_secs: float | None = None,
1301
+ ) -> None:
1302
+ super().__init__()
1303
+ self._tracer: Tracer = tracer
1304
+ self._trace: Trace = trace
1305
+ # Optional: the compiled graph this tracer instruments.
1306
+ # When supplied, on_chain_start can additionally look up
1307
+ # each node's graph-construction-time *declared* tags (see
1308
+ # _get_declared_node_tags) — information the runtime `tags`
1309
+ # callback kwarg alone never carries. None (the default)
1310
+ # keeps the pre-existing behavior: no declared-tags lookup,
1311
+ # every other capability unaffected.
1312
+ self._graph: Any = graph
1313
+ # Optional: flags any span (node/llm/tool) whose measured
1314
+ # open duration exceeds this many seconds once it closes —
1315
+ # e.g. LangGraph Cloud's ~180s checkpoint-sweep re-dispatch
1316
+ # window (issue #7417). None (the default) disables the
1317
+ # check entirely.
1318
+ self._long_span_threshold_secs: float | None = (
1319
+ long_span_threshold_secs
1320
+ )
1321
+ # Thread-safe span registry: run_id (UUID str) -> open Span
1322
+ self._spans: dict[str, Span] = {}
1323
+ # Per-run streaming-token counters (on_llm_new_token), so
1324
+ # llm.stream_token_count can keep counting past the
1325
+ # per-span SpanEvent cap without holding the events
1326
+ # themselves. Cleared alongside the span on close.
1327
+ self._stream_token_counts: dict[str, int] = {}
1328
+ # Per-run correlation-id contextvar tokens (#6037): every
1329
+ # span _open_span creates pushes its own span_id as the
1330
+ # active httpx_hook correlation id for the duration it's
1331
+ # open, so any HTTP exchange made anywhere inside it —
1332
+ # including inside a supervisor topology's sub-agent node
1333
+ # — is automatically tagged with the originating span via
1334
+ # Fixture.exchanges_for_correlation_id(). Popped (in the
1335
+ # same context the push happened in) alongside the span on
1336
+ # close.
1337
+ self._correlation_tokens: dict[str, contextvars.Token[str | None]] = {}
1338
+ self._lock: threading.Lock = threading.Lock()
1339
+ # Best-effort: makes this instance discoverable to the
1340
+ # Branch-dispatch and tool-argument-injection patches (see
1341
+ # _record_branch_dispatch / _record_tool_arg_injection),
1342
+ # which have no other way to reach a LangGraphTracer/Tracer
1343
+ # since the callback manager never invokes them for a
1344
+ # trace=False component. Constructed in the same execution
1345
+ # context that will go on to call graph.invoke()/.ainvoke(),
1346
+ # so the ContextVar value is visible throughout that call.
1347
+ _current_langgraph_tracer.set(self)
1348
+ # Best-effort, one-time: flag tool parameters shaped like
1349
+ # graph state but not marked InjectedState/InjectedStore —
1350
+ # see find_tool_params_shaped_like_state(). Only runs when a
1351
+ # graph was supplied; writes findings onto the trace's own
1352
+ # metadata dict (not a span — this is a static property of
1353
+ # the graph's wiring, not a per-invocation event) so it
1354
+ # survives even if the graph never routes through the
1355
+ # flagged tool during this particular run.
1356
+ if graph is not None:
1357
+ self._check_tool_state_shaped_params(graph)
1358
+
1359
+ # ------------------------------------------------------------------
1360
+ # Internal helpers
1361
+ # ------------------------------------------------------------------
1362
+
1363
+ def _check_tool_state_shaped_params(self, graph: Any) -> None:
1364
+ """Run find_tool_params_shaped_like_state() against *graph*
1365
+ once, at construction time, and record any findings onto
1366
+ the active trace's metadata dict under
1367
+ "tool_state_shaped_params" (JSON-serialized). Best-effort:
1368
+ a failure here must never break constructing the tracer."""
1369
+ try:
1370
+ findings = find_tool_params_shaped_like_state(graph)
1371
+ if not findings:
1372
+ return
1373
+ active = getattr(self._tracer, "active_trace", None)
1374
+ if active is None:
1375
+ return
1376
+ active.metadata["tool_state_shaped_params"] = _to_attr_string(
1377
+ findings
1378
+ )
1379
+ except Exception:
1380
+ logger.debug(
1381
+ "agent-trace: failed to run "
1382
+ "find_tool_params_shaped_like_state()",
1383
+ exc_info=True,
1384
+ )
1385
+
1386
+ def _open_span(
1387
+ self,
1388
+ run_id: uuid.UUID | str,
1389
+ name: str,
1390
+ parent_run_id: uuid.UUID | str | None = None,
1391
+ ) -> Span:
1392
+ """Create a span and register it in the local registry."""
1393
+ run_key = str(run_id)
1394
+ parent_span_id: str | None = None
1395
+ if parent_run_id is not None:
1396
+ with self._lock:
1397
+ parent_span = self._spans.get(str(parent_run_id))
1398
+ parent_span_id = (
1399
+ parent_span.span_id if parent_span is not None else None
1400
+ )
1401
+
1402
+ span = self._tracer.start_span(name, parent_id=parent_span_id)
1403
+ if name.startswith("llm:") and parent_span_id is not None:
1404
+ with self._lock:
1405
+ parent_span = self._spans.get(str(parent_run_id))
1406
+ if parent_span is not None and parent_span.name.startswith("tool:"):
1407
+ # This LLM call is happening inside a tool's own
1408
+ # execution, not directly under a graph node —
1409
+ # flag it explicitly rather than leaving "this call
1410
+ # happened inside a tool" recoverable only from
1411
+ # tree indentation or a manual parent_id read
1412
+ # (#5665: the exact schema-leak-triggering shape).
1413
+ span.set_attribute("llm.nested_in_tool", True)
1414
+ # Tag every HTTP exchange made anywhere while this span is
1415
+ # open with its span_id (#6037) — recoverable afterwards
1416
+ # via Fixture.exchanges_for_correlation_id(span_id). Best
1417
+ # effort: pushing/popping a contextvars.Token must never be
1418
+ # allowed to break span creation itself.
1419
+ #
1420
+ # Known limitation, verified empirically against the
1421
+ # installed langgraph (1.2.8): this correctly attributes
1422
+ # HTTP calls for graphs invoked via the *synchronous*
1423
+ # entrypoint (graph.invoke(...)) — including a
1424
+ # create_supervisor topology's sequential sub-agent
1425
+ # handoffs, #6037's actual shape — because the callback
1426
+ # that pushes the token and the node body that makes the
1427
+ # HTTP call run in the same context/thread.
1428
+ #
1429
+ # It does NOT propagate when the graph is invoked via the
1430
+ # *asynchronous* entrypoint (graph.ainvoke(...)/
1431
+ # graph.astream(...)), regardless of whether the node
1432
+ # itself is sync or async, and regardless of whether
1433
+ # dispatch is sequential or concurrent (Send()-based
1434
+ # fan-out, #7129's shape): LangGraph's async runtime
1435
+ # dispatches node execution in a way that doesn't inherit a
1436
+ # contextvar pushed by this callback (empirically, even a
1437
+ # single sequential sync node loses the token under
1438
+ # ainvoke()) — current_correlation_id() reads None inside
1439
+ # the node body even though push_correlation_id() ran
1440
+ # moments earlier in on_chain_start/on_tool_start. Fixing
1441
+ # this would need a different mechanism entirely (e.g.
1442
+ # LangGraph itself threading a correlation id through its
1443
+ # own task/Send dispatch) — out of scope here. Documented,
1444
+ # not silently assumed to work universally; see
1445
+ # tests/integration/test_langgraph.py::
1446
+ # TestHttpExchangeCorrelationToOriginatingSpan for both the
1447
+ # working (sync invoke) and known-limited (async
1448
+ # ainvoke/Send) cases pinned as tests.
1449
+ token: contextvars.Token[str | None] | None = None
1450
+ try:
1451
+ token = push_correlation_id(span.span_id)
1452
+ except Exception:
1453
+ logger.debug(
1454
+ "agent-trace: failed to push correlation id for span %r",
1455
+ span.name,
1456
+ exc_info=True,
1457
+ )
1458
+
1459
+ with self._lock:
1460
+ self._spans[run_key] = span
1461
+ if token is not None:
1462
+ self._correlation_tokens[run_key] = token
1463
+ return span
1464
+
1465
+ def _flag_if_long_running(self, span: Span) -> None:
1466
+ """Best-effort: mark *span* with
1467
+ ``span.exceeded_long_running_threshold=true`` if its measured
1468
+ open duration crosses ``self._long_span_threshold_secs``
1469
+ (disabled entirely when that's None — the default).
1470
+
1471
+ Checked at close time rather than via a background timer —
1472
+ agent-trace has no in-flight monitoring thread for spans, so
1473
+ this surfaces the same information (a span that ran longer
1474
+ than a known-risky platform duration, e.g. LangGraph Cloud's
1475
+ ~180s checkpoint-sweep re-dispatch window, issue #7417) the
1476
+ one place this callback-driven architecture actually can:
1477
+ once the span closes and its true duration is known.
1478
+ """
1479
+ if self._long_span_threshold_secs is None:
1480
+ return
1481
+ try:
1482
+ elapsed = get_time() - span.start_time
1483
+ span.set_attribute(
1484
+ "span.duration_secs_at_close", round(elapsed, 3)
1485
+ )
1486
+ if elapsed >= self._long_span_threshold_secs:
1487
+ span.set_attribute(
1488
+ "span.exceeded_long_running_threshold", True
1489
+ )
1490
+ span.set_attribute(
1491
+ "span.long_running_threshold_secs",
1492
+ self._long_span_threshold_secs,
1493
+ )
1494
+ except Exception:
1495
+ logger.debug(
1496
+ "agent-trace: failed to evaluate long-running-span "
1497
+ "threshold for span %r",
1498
+ span.name,
1499
+ exc_info=True,
1500
+ )
1501
+
1502
+ def _pop_correlation_token(self, run_key: str) -> None:
1503
+ """Best-effort counterpart to the push in _open_span. A
1504
+ cross-context Token reset (e.g. a span opened on one
1505
+ asyncio task's context and closed from another after a
1506
+ cancellation/executor edge case) must degrade to "leave the
1507
+ correlation id set a little longer than intended", never
1508
+ raise and break span closure itself."""
1509
+ with self._lock:
1510
+ token = self._correlation_tokens.pop(run_key, None)
1511
+ if token is None:
1512
+ return
1513
+ try:
1514
+ pop_correlation_id(token)
1515
+ except Exception:
1516
+ logger.debug(
1517
+ "agent-trace: failed to pop correlation id for run %r",
1518
+ run_key,
1519
+ exc_info=True,
1520
+ )
1521
+
1522
+ def _close_span(
1523
+ self,
1524
+ run_id: uuid.UUID | str,
1525
+ status: SpanStatus = SpanStatus.OK,
1526
+ ) -> Span | None:
1527
+ """End the span for *run_id* and remove it from the registry."""
1528
+ run_key = str(run_id)
1529
+ with self._lock:
1530
+ span = self._spans.pop(run_key, None)
1531
+ self._stream_token_counts.pop(run_key, None)
1532
+ self._pop_correlation_token(run_key)
1533
+ if span is not None and span.end_time is None:
1534
+ self._flag_if_long_running(span)
1535
+ span.end(status)
1536
+ return span
1537
+
1538
+ def _close_span_with_exception(
1539
+ self,
1540
+ run_id: uuid.UUID | str,
1541
+ error: BaseException,
1542
+ ) -> None:
1543
+ """Pop the span and close it according to what *error* means.
1544
+
1545
+ Three distinct outcomes, checked in order:
1546
+ 1. LangGraph's own internal control-flow signal (a
1547
+ Command/ParentCommand handoff jump or a GraphInterrupt
1548
+ pause) -> span closes OK with an informational
1549
+ attribute, not ERROR — it isn't an application failure.
1550
+ 2. asyncio.CancelledError -> span closes CANCELLED, kept
1551
+ distinct from ERROR so a reader can tell "this failed"
1552
+ apart from "this was cut off mid-flight".
1553
+ 3. Anything else -> genuine error: record the exception,
1554
+ classify its origin/known pattern, close ERROR.
1555
+
1556
+ Consolidates the three error callbacks into a single lock
1557
+ acquisition + record + end sequence.
1558
+ """
1559
+ run_key = str(run_id)
1560
+ with self._lock:
1561
+ span = self._spans.pop(run_key, None)
1562
+ self._stream_token_counts.pop(run_key, None)
1563
+ self._pop_correlation_token(run_key)
1564
+ if span is None:
1565
+ return
1566
+
1567
+ if _is_langgraph_control_flow_signal(error):
1568
+ _record_control_flow_signal(span, error)
1569
+ if span.end_time is None:
1570
+ self._flag_if_long_running(span)
1571
+ span.end(SpanStatus.OK)
1572
+ return
1573
+
1574
+ if isinstance(error, asyncio.CancelledError):
1575
+ span.record_exception(error, status=SpanStatus.CANCELLED)
1576
+ if span.end_time is None:
1577
+ self._flag_if_long_running(span)
1578
+ span.end(SpanStatus.CANCELLED)
1579
+ return
1580
+
1581
+ span.record_exception(error)
1582
+ _classify_and_tag_exception(span, error)
1583
+ if span.end_time is None:
1584
+ self._flag_if_long_running(span)
1585
+ span.end(SpanStatus.ERROR)
1586
+
1587
+ # ------------------------------------------------------------------
1588
+ # Chain (graph node) callbacks
1589
+ # ------------------------------------------------------------------
1590
+
1591
+ def on_chain_start(
1592
+ self,
1593
+ serialized: dict[str, Any],
1594
+ inputs: dict[str, Any],
1595
+ *,
1596
+ run_id: uuid.UUID,
1597
+ parent_run_id: uuid.UUID | None = None,
1598
+ tags: list[str] | None = None,
1599
+ metadata: dict[str, Any] | None = None,
1600
+ **kwargs: Any,
1601
+ ) -> None:
1602
+ """Start a span when a graph node (chain) begins execution."""
1603
+ # LangGraph 1.x passes serialized=None; node name is in kwargs['name'].
1604
+ ser = serialized or {}
1605
+ node_name: str = (
1606
+ kwargs.get("name")
1607
+ or ser.get("name")
1608
+ or (ser.get("id") or [None])[-1]
1609
+ or "chain"
1610
+ )
1611
+ span = self._open_span(run_id, f"node:{node_name}", parent_run_id)
1612
+ span.set_attribute("langgraph.node", node_name)
1613
+ if tags:
1614
+ span.set_attribute("langgraph.tags", ",".join(tags))
1615
+ if self._graph is not None:
1616
+ declared_tags = _get_declared_node_tags(self._graph, node_name)
1617
+ if declared_tags:
1618
+ span.set_attribute(
1619
+ "langgraph.declared_tags", ",".join(declared_tags)
1620
+ )
1621
+ if metadata:
1622
+ span.set_attribute("chain.metadata", _to_attr_string(metadata))
1623
+ if inputs:
1624
+ span.set_attribute("chain.inputs", _to_attr_string(inputs))
1625
+ # Best-effort Runtime/context capture — see
1626
+ # _install_runtime_capture_patch(). Deliberately kept out of
1627
+ # the "langgraph." attribute namespace since the Runtime
1628
+ # object (store/writer/context) is not guaranteed to
1629
+ # serialize identically across separate invocations of the
1630
+ # same graph (e.g. record vs. replay).
1631
+ runtime = _current_runtime.get()
1632
+ if runtime is not None:
1633
+ span.set_attribute("chain.runtime", _to_attr_string(runtime))
1634
+
1635
+ def on_chain_end(
1636
+ self,
1637
+ outputs: dict[str, Any],
1638
+ *,
1639
+ run_id: uuid.UUID,
1640
+ parent_run_id: uuid.UUID | None = None,
1641
+ tags: list[str] | None = None,
1642
+ **kwargs: Any,
1643
+ ) -> None:
1644
+ """End the span when a graph node completes successfully."""
1645
+ run_key = str(run_id)
1646
+ with self._lock:
1647
+ span = self._spans.get(run_key)
1648
+ if span is not None and outputs:
1649
+ try:
1650
+ span.set_attribute("chain.outputs", _to_attr_string(outputs))
1651
+ except Exception:
1652
+ logger.debug(
1653
+ "agent-trace: failed to record chain outputs for run %r",
1654
+ str(run_id),
1655
+ exc_info=True,
1656
+ )
1657
+ self._close_span(run_id, SpanStatus.OK)
1658
+
1659
+ def on_chain_error(
1660
+ self,
1661
+ error: BaseException,
1662
+ *,
1663
+ run_id: uuid.UUID,
1664
+ parent_run_id: uuid.UUID | None = None,
1665
+ tags: list[str] | None = None,
1666
+ **kwargs: Any,
1667
+ ) -> None:
1668
+ """End the span with ERROR status when a graph node raises."""
1669
+ self._close_span_with_exception(run_id, error)
1670
+
1671
+ # ------------------------------------------------------------------
1672
+ # LLM callbacks
1673
+ # ------------------------------------------------------------------
1674
+
1675
+ def on_llm_start(
1676
+ self,
1677
+ serialized: dict[str, Any],
1678
+ prompts: list[str],
1679
+ *,
1680
+ run_id: uuid.UUID,
1681
+ parent_run_id: uuid.UUID | None = None,
1682
+ tags: list[str] | None = None,
1683
+ metadata: dict[str, Any] | None = None,
1684
+ **kwargs: Any,
1685
+ ) -> None:
1686
+ """Start a span for a legacy LLM call, recording the model name."""
1687
+ ser = serialized or {}
1688
+ model_name: str = (
1689
+ kwargs.get("name")
1690
+ or (ser.get("kwargs") or {}).get("model_name")
1691
+ or (ser.get("kwargs") or {}).get("model")
1692
+ or ser.get("name")
1693
+ or "llm"
1694
+ )
1695
+ span = self._open_span(run_id, f"llm:{model_name}", parent_run_id)
1696
+ span.set_attribute("llm.model", model_name)
1697
+ span.set_attribute("llm.prompt_count", len(prompts))
1698
+ if metadata:
1699
+ span.set_attribute("llm.metadata", _to_attr_string(metadata))
1700
+ bound_tools = _extract_bound_tools(kwargs)
1701
+ if bound_tools:
1702
+ span.set_attribute("llm.bound_tools", _to_attr_string(bound_tools))
1703
+
1704
+ def on_chat_model_start(
1705
+ self,
1706
+ serialized: dict[str, Any],
1707
+ messages: list[list[Any]],
1708
+ *,
1709
+ run_id: uuid.UUID,
1710
+ parent_run_id: uuid.UUID | None = None,
1711
+ tags: list[str] | None = None,
1712
+ metadata: dict[str, Any] | None = None,
1713
+ **kwargs: Any,
1714
+ ) -> None:
1715
+ """Start a span for a ChatModel call (e.g. ChatOpenAI, ChatAnthropic).
1716
+
1717
+ Modern LangChain chat models fire ``on_chat_model_start`` instead
1718
+ of ``on_llm_start``. Without this handler those spans are silently
1719
+ dropped.
1720
+ """
1721
+ ser = serialized or {}
1722
+ model_name: str = (
1723
+ kwargs.get("name")
1724
+ or (ser.get("kwargs") or {}).get("model_name")
1725
+ or (ser.get("kwargs") or {}).get("model")
1726
+ or ser.get("name")
1727
+ or "unknown"
1728
+ )
1729
+ span = self._open_span(run_id, f"llm:{model_name}", parent_run_id)
1730
+ span.set_attribute("llm.model", model_name)
1731
+ if messages:
1732
+ span.set_attribute("llm.messages", _to_attr_string(messages))
1733
+ if metadata:
1734
+ span.set_attribute("llm.metadata", _to_attr_string(metadata))
1735
+ bound_tools = _extract_bound_tools(kwargs)
1736
+ if bound_tools:
1737
+ span.set_attribute("llm.bound_tools", _to_attr_string(bound_tools))
1738
+
1739
+ def on_llm_new_token(
1740
+ self,
1741
+ token: str,
1742
+ *,
1743
+ chunk: Any = None,
1744
+ run_id: uuid.UUID,
1745
+ parent_run_id: uuid.UUID | None = None,
1746
+ tags: list[str] | None = None,
1747
+ **kwargs: Any,
1748
+ ) -> None:
1749
+ """Record a per-token/per-delta streaming chunk instead of
1750
+ discarding it.
1751
+
1752
+ This is the one real streaming hook the current
1753
+ ``langchain_core`` ``BaseCallbackHandler`` interface exposes
1754
+ (confirmed via direct inspection of the installed
1755
+ langchain-core): it fires for both legacy ``LLM.stream()``
1756
+ calls and modern chat-model streaming alike — LangChain
1757
+ routes both through this single callback, passing a
1758
+ ``GenerationChunk`` or ``ChatGenerationChunk`` via *chunk*
1759
+ depending on which. There is no separate
1760
+ ``on_chat_model_stream`` method on the base handler to
1761
+ implement.
1762
+
1763
+ Bounded: after _MAX_STREAM_EVENTS_PER_SPAN tokens, further
1764
+ deltas stop generating new SpanEvents (a long stream would
1765
+ otherwise grow the span unboundedly) but
1766
+ ``llm.stream_token_count`` keeps counting every token that
1767
+ arrived.
1768
+ """
1769
+ run_key = str(run_id)
1770
+ with self._lock:
1771
+ span = self._spans.get(run_key)
1772
+ count = self._stream_token_counts.get(run_key, 0) + 1
1773
+ self._stream_token_counts[run_key] = count
1774
+ if span is None:
1775
+ return
1776
+ try:
1777
+ span.set_attribute("llm.streamed", True)
1778
+ span.set_attribute("llm.stream_token_count", count)
1779
+ if count <= _MAX_STREAM_EVENTS_PER_SPAN:
1780
+ attrs: dict[str, Any] = {"stream.index": count - 1}
1781
+ if token:
1782
+ attrs["token"] = _stringify(token, max_len=2000)
1783
+ tool_call_chunks = _extract_tool_call_chunks(chunk)
1784
+ if tool_call_chunks:
1785
+ attrs["tool_call_chunks"] = _to_attr_string(
1786
+ tool_call_chunks, max_len=2000
1787
+ )
1788
+ span.add_event("llm_stream_delta", attributes=attrs)
1789
+ except Exception:
1790
+ logger.debug(
1791
+ "agent-trace: failed to record streaming token for "
1792
+ "run %r",
1793
+ str(run_id),
1794
+ exc_info=True,
1795
+ )
1796
+
1797
+ def on_llm_end(
1798
+ self,
1799
+ response: Any,
1800
+ *,
1801
+ run_id: uuid.UUID,
1802
+ parent_run_id: uuid.UUID | None = None,
1803
+ tags: list[str] | None = None,
1804
+ **kwargs: Any,
1805
+ ) -> None:
1806
+ """End the LLM span, attaching token usage and response content
1807
+ when available."""
1808
+ run_key = str(run_id)
1809
+ with self._lock:
1810
+ span = self._spans.get(run_key)
1811
+ if span is not None:
1812
+ try:
1813
+ _record_llm_end_data(span, response)
1814
+ except Exception:
1815
+ logger.debug(
1816
+ "agent-trace: failed to record LLM response data "
1817
+ "for run %r",
1818
+ str(run_id),
1819
+ exc_info=True,
1820
+ )
1821
+ self._close_span(run_id, SpanStatus.OK)
1822
+
1823
+ def on_llm_error(
1824
+ self,
1825
+ error: BaseException,
1826
+ *,
1827
+ run_id: uuid.UUID,
1828
+ parent_run_id: uuid.UUID | None = None,
1829
+ tags: list[str] | None = None,
1830
+ **kwargs: Any,
1831
+ ) -> None:
1832
+ """End the LLM span with ERROR status."""
1833
+ self._close_span_with_exception(run_id, error)
1834
+
1835
+ # ------------------------------------------------------------------
1836
+ # Tool callbacks
1837
+ # ------------------------------------------------------------------
1838
+
1839
+ def on_tool_start(
1840
+ self,
1841
+ serialized: dict[str, Any],
1842
+ input_str: str,
1843
+ *,
1844
+ run_id: uuid.UUID,
1845
+ parent_run_id: uuid.UUID | None = None,
1846
+ tags: list[str] | None = None,
1847
+ metadata: dict[str, Any] | None = None,
1848
+ **kwargs: Any,
1849
+ ) -> None:
1850
+ """Start a span when a tool begins execution."""
1851
+ ser = serialized or {}
1852
+ tool_name: str = kwargs.get("name") or ser.get("name") or "tool"
1853
+ span = self._open_span(run_id, f"tool:{tool_name}", parent_run_id)
1854
+ span.set_attribute("tool.name", tool_name)
1855
+ if input_str:
1856
+ span.set_attribute("tool.input", _stringify(input_str))
1857
+ if metadata:
1858
+ span.set_attribute("tool.metadata", _to_attr_string(metadata))
1859
+ # Threading/event-loop context — the exact structured data
1860
+ # needed to diagnose a sync tool dispatched into a
1861
+ # ThreadPoolExecutor with no event loop (RuntimeError: "There
1862
+ # is no current event loop in thread ...").
1863
+ span.set_attribute("tool.thread_name", threading.current_thread().name)
1864
+ try:
1865
+ asyncio.get_running_loop()
1866
+ span.set_attribute("tool.has_event_loop", True)
1867
+ except RuntimeError:
1868
+ span.set_attribute("tool.has_event_loop", False)
1869
+
1870
+ def on_tool_end(
1871
+ self,
1872
+ output: str,
1873
+ *,
1874
+ run_id: uuid.UUID,
1875
+ parent_run_id: uuid.UUID | None = None,
1876
+ tags: list[str] | None = None,
1877
+ **kwargs: Any,
1878
+ ) -> None:
1879
+ """End the tool span with OK status, recording output text."""
1880
+ run_key = str(run_id)
1881
+ with self._lock:
1882
+ span = self._spans.get(run_key)
1883
+ if span is not None and output is not None:
1884
+ try:
1885
+ span.set_attribute("tool.output", _stringify(output))
1886
+ except Exception:
1887
+ logger.debug(
1888
+ "agent-trace: failed to record tool output for run %r",
1889
+ str(run_id),
1890
+ exc_info=True,
1891
+ )
1892
+ self._close_span(run_id, SpanStatus.OK)
1893
+
1894
+ def on_tool_error(
1895
+ self,
1896
+ error: BaseException,
1897
+ *,
1898
+ run_id: uuid.UUID,
1899
+ parent_run_id: uuid.UUID | None = None,
1900
+ tags: list[str] | None = None,
1901
+ **kwargs: Any,
1902
+ ) -> None:
1903
+ """End the tool span with ERROR status."""
1904
+ self._close_span_with_exception(run_id, error)
1905
+
1906
+ _LangGraphTracerClass = _LangGraphTracerImpl
1907
+ return _LangGraphTracerClass
1908
+
1909
+
1910
+ # ---------------------------------------------------------------------------
1911
+ # traced_stream / traced_astream — stream-yield timestamp + content capture
1912
+ # ---------------------------------------------------------------------------
1913
+ #
1914
+ # LangGraphTracer only implements the standard LangChain callback pairs
1915
+ # (on_chain_start/end, on_llm_start/end, on_tool_start/end, ...), none of
1916
+ # which fire on "a value was actually yielded from the graph's own
1917
+ # .stream()/.astream() iterator to the caller's code". That boundary matters:
1918
+ # graph.invoke(state, stream_mode=...) fully drains the generator internally
1919
+ # before returning (Pregel.invoke() loops `for chunk in self.stream(...)`
1920
+ # and only returns once exhausted) while graph.stream(...) yields
1921
+ # progressively — two very different externally-observed delivery timings
1922
+ # that look identical in a trace with only callback-derived spans, since
1923
+ # every chain/llm/tool span closes at the same internal moment either way.
1924
+ #
1925
+ # traced_stream()/traced_astream() wrap *any* iterable/async-iterable
1926
+ # (typically the return value of graph.stream(...)/graph.astream(...), with
1927
+ # whatever stream_mode was requested) in a dedicated span, recording a
1928
+ # SpanEvent — timestamped on the same clock as every other span
1929
+ # (core.clock.get_time(), via Span.add_event) — at the exact moment each
1930
+ # item is yielded back to the calling code, plus a bounded, serialized copy
1931
+ # of the chunk's own content. This also directly captures stream_mode's
1932
+ # actual per-chunk output (messages/updates/values, whichever mode was
1933
+ # requested) onto the trace, not just its timing.
1934
+
1935
+
1936
+ def _record_stream_yield(span: Span, index: int, item: Any) -> None:
1937
+ """Append one bounded stream_yield SpanEvent, capped at
1938
+ _MAX_STREAM_EVENTS_PER_SPAN so an unbounded stream can't grow a single
1939
+ span's event list without limit."""
1940
+ if index >= _MAX_STREAM_EVENTS_PER_SPAN:
1941
+ return
1942
+ span.add_event(
1943
+ "stream_yield",
1944
+ attributes={
1945
+ "stream.index": index,
1946
+ "stream.chunk": _stringify(item, max_len=2000),
1947
+ },
1948
+ )
1949
+
1950
+
1951
+ def traced_stream(
1952
+ tracer: Tracer,
1953
+ stream: Iterable[Any],
1954
+ *,
1955
+ span_name: str = "graph:stream",
1956
+ ) -> Generator[Any, None, None]:
1957
+ """Wrap a LangGraph graph's ``.stream()`` iterator (or any other
1958
+ iterable) so each item's yield moment — and a bounded copy of its
1959
+ content — lands on the trace timeline.
1960
+
1961
+ Usage::
1962
+
1963
+ for chunk in traced_stream(tracer, graph.stream(state,
1964
+ stream_mode="messages")):
1965
+ ...
1966
+
1967
+ Opens one ``graph:stream`` span (customizable via *span_name*) that
1968
+ stays open for the lifetime of the iteration, closing OK once the
1969
+ source stream is exhausted, ERROR if the source stream itself raises
1970
+ (the exception is recorded onto the span then re-raised unchanged), or
1971
+ CANCELLED if the caller stops iterating early (e.g. a ``break``) and
1972
+ this generator is garbage-collected/closed before exhaustion.
1973
+ """
1974
+ span = tracer.start_span(span_name)
1975
+ index = 0
1976
+ status = SpanStatus.OK
1977
+ try:
1978
+ for item in stream:
1979
+ _record_stream_yield(span, index, item)
1980
+ index += 1
1981
+ yield item
1982
+ except GeneratorExit:
1983
+ status = SpanStatus.CANCELLED
1984
+ raise
1985
+ except Exception as exc:
1986
+ status = SpanStatus.ERROR
1987
+ span.record_exception(exc)
1988
+ raise
1989
+ finally:
1990
+ span.set_attribute("stream.chunk_count", index)
1991
+ if span.end_time is None:
1992
+ span.end(status)
1993
+
1994
+
1995
+ async def traced_astream(
1996
+ tracer: Tracer,
1997
+ stream: AsyncIterable[Any],
1998
+ *,
1999
+ span_name: str = "graph:astream",
2000
+ ) -> AsyncGenerator[Any, None]:
2001
+ """Async equivalent of :func:`traced_stream` — wraps
2002
+ ``graph.astream(...)`` (or any other async iterable) the same way."""
2003
+ span = tracer.start_span(span_name)
2004
+ index = 0
2005
+ status = SpanStatus.OK
2006
+ try:
2007
+ async for item in stream:
2008
+ _record_stream_yield(span, index, item)
2009
+ index += 1
2010
+ yield item
2011
+ except GeneratorExit:
2012
+ status = SpanStatus.CANCELLED
2013
+ raise
2014
+ except Exception as exc:
2015
+ status = SpanStatus.ERROR
2016
+ span.record_exception(exc)
2017
+ raise
2018
+ finally:
2019
+ span.set_attribute("stream.chunk_count", index)
2020
+ if span.end_time is None:
2021
+ span.end(status)
2022
+
2023
+
2024
+ class LangGraphTracer:
2025
+ """Langchain/LangGraph callback handler that emits agent-trace spans.
2026
+
2027
+ Implements the ``BaseCallbackHandler`` interface from ``langchain_core``
2028
+ so it can be passed directly in the ``config["callbacks"]`` list of any
2029
+ LangGraph graph invocation.
2030
+
2031
+ ``langchain_core`` is imported lazily — importing this module succeeds even
2032
+ when ``langchain_core`` is not installed. The import (and the class
2033
+ definition that inherits from ``BaseCallbackHandler``) happens once, the
2034
+ first time a ``LangGraphTracer`` instance is created.
2035
+
2036
+ Parameters
2037
+ ----------
2038
+ tracer:
2039
+ The active :class:`~agent_trace.Tracer` instance.
2040
+ trace:
2041
+ The :class:`~agent_trace.Trace` that spans will be registered on.
2042
+ graph:
2043
+ Optional: the compiled LangGraph graph this tracer instruments.
2044
+ When supplied, node spans additionally carry a
2045
+ ``langgraph.declared_tags`` attribute — each node's
2046
+ graph-construction-time declared tags (e.g. from
2047
+ ``.with_config(tags=["nostream"])``), which the runtime callback
2048
+ ``tags`` kwarg alone never exposes. Omit (the default, None) to keep
2049
+ the pre-existing behavior with no declared-tags lookup.
2050
+ long_span_threshold_secs:
2051
+ Optional: once a span (node/llm/tool) closes having been open for at
2052
+ least this many seconds, it is flagged with
2053
+ ``span.exceeded_long_running_threshold=true`` and
2054
+ ``span.duration_secs_at_close``. Useful for platform-level behaviors
2055
+ keyed off wall-clock duration — e.g. LangGraph Cloud's ~180s
2056
+ checkpoint-sweep re-dispatch window (issue #7417). Omit (the
2057
+ default, None) to disable the check entirely.
2058
+ """
2059
+
2060
+ def __new__(
2061
+ cls,
2062
+ tracer: Tracer,
2063
+ trace: Trace,
2064
+ *,
2065
+ graph: Any = None,
2066
+ long_span_threshold_secs: float | None = None,
2067
+ ) -> LangGraphTracer:
2068
+ # Construct the concrete impl directly so Python's normal type.__call__
2069
+ # runs _LangGraphTracerImpl.__init__ automatically. We cannot use
2070
+ # impl_cls.__new__(impl_cls) + manual __init__ because the returned
2071
+ # object would not be an instance of LangGraphTracer, causing Python
2072
+ # to skip __init__ entirely — leaving _tracer/_trace/_spans unset.
2073
+ impl_cls = _get_tracer_class()
2074
+ return impl_cls( # type: ignore[no-any-return]
2075
+ tracer,
2076
+ trace,
2077
+ graph=graph,
2078
+ long_span_threshold_secs=long_span_threshold_secs,
2079
+ )
2080
+
2081
+ def __init__(
2082
+ self,
2083
+ tracer: Tracer,
2084
+ trace: Trace,
2085
+ *,
2086
+ graph: Any = None,
2087
+ long_span_threshold_secs: float | None = None,
2088
+ ) -> None:
2089
+ # __init__ is called on the instance whose __class__ is already the
2090
+ # concrete impl class (set by __new__). Delegate to its __init__.
2091
+ # This path is only reached if someone subclasses LangGraphTracer
2092
+ # directly; normal construction goes through the impl class __init__.
2093
+ pass # pragma: no cover