agent-observability-trace-cli 0.1.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_observability_trace_cli-0.1.2.dist-info/METADATA +336 -0
- agent_observability_trace_cli-0.1.2.dist-info/RECORD +51 -0
- agent_observability_trace_cli-0.1.2.dist-info/WHEEL +4 -0
- agent_observability_trace_cli-0.1.2.dist-info/entry_points.txt +2 -0
- agent_observability_trace_cli-0.1.2.dist-info/licenses/LICENSE +198 -0
- agent_trace/__init__.py +1182 -0
- agent_trace/_cli.py +1377 -0
- agent_trace/_inspect.py +2020 -0
- agent_trace/_replay/__init__.py +1 -0
- agent_trace/_replay/engine.py +268 -0
- agent_trace/_replay/fixture.py +868 -0
- agent_trace/core/__init__.py +1 -0
- agent_trace/core/clock.py +91 -0
- agent_trace/core/exceptions.py +51 -0
- agent_trace/core/span.py +271 -0
- agent_trace/core/trace.py +92 -0
- agent_trace/exporters/__init__.py +1 -0
- agent_trace/exporters/file.py +80 -0
- agent_trace/exporters/otlp.py +199 -0
- agent_trace/exporters/remote_fixture.py +307 -0
- agent_trace/exporters/stdout.py +258 -0
- agent_trace/integrations/__init__.py +69 -0
- agent_trace/integrations/agno.py +489 -0
- agent_trace/integrations/autogen.py +581 -0
- agent_trace/integrations/crewai.py +486 -0
- agent_trace/integrations/google_genai.py +731 -0
- agent_trace/integrations/haystack.py +254 -0
- agent_trace/integrations/langchain_core.py +361 -0
- agent_trace/integrations/langgraph.py +2093 -0
- agent_trace/integrations/langgraph_checkpoint.py +763 -0
- agent_trace/integrations/langgraph_state_diff.py +345 -0
- agent_trace/integrations/langgraph_stream_debug.py +202 -0
- agent_trace/integrations/llama_index.py +472 -0
- agent_trace/integrations/mcp.py +281 -0
- agent_trace/integrations/openai_agents.py +811 -0
- agent_trace/integrations/pydantic_ai.py +504 -0
- agent_trace/integrations/streaming.py +218 -0
- agent_trace/interceptor/__init__.py +64 -0
- agent_trace/interceptor/aiohttp_hook.py +152 -0
- agent_trace/interceptor/botocore_hook.py +223 -0
- agent_trace/interceptor/grpc_hook.py +651 -0
- agent_trace/interceptor/httpx_hook.py +866 -0
- agent_trace/interceptor/logging_hook.py +137 -0
- agent_trace/interceptor/requests_patch.py +184 -0
- agent_trace/interceptor/sse.py +176 -0
- agent_trace/interceptor/stdio_hook.py +245 -0
- agent_trace/interceptor/warnings_hook.py +132 -0
- agent_trace/interceptor/websocket_hook.py +323 -0
- agent_trace/plugins/__init__.py +35 -0
- agent_trace/plugins/base.py +111 -0
- agent_trace/py.typed +0 -0
agent_trace/__init__.py
ADDED
|
@@ -0,0 +1,1182 @@
|
|
|
1
|
+
"""
|
|
2
|
+
agent-trace — AI agent observability with deterministic record/replay.
|
|
3
|
+
|
|
4
|
+
Quick start:
|
|
5
|
+
from agent_trace import tracer
|
|
6
|
+
|
|
7
|
+
@tracer.instrument(record=True)
|
|
8
|
+
def my_agent(query: str) -> str:
|
|
9
|
+
...
|
|
10
|
+
|
|
11
|
+
result = my_agent("debug this")
|
|
12
|
+
# Trace saved to ~/.agent-trace/runs/run_<id>/
|
|
13
|
+
|
|
14
|
+
Replay offline:
|
|
15
|
+
from agent_trace import replay
|
|
16
|
+
|
|
17
|
+
with replay("run_<id>") as ctx:
|
|
18
|
+
result = my_agent(ctx.get_metadata("input"))
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import atexit
|
|
24
|
+
import functools
|
|
25
|
+
import inspect
|
|
26
|
+
import json
|
|
27
|
+
import logging
|
|
28
|
+
import os
|
|
29
|
+
import uuid
|
|
30
|
+
from collections.abc import Callable, Generator
|
|
31
|
+
from contextlib import contextmanager, nullcontext
|
|
32
|
+
from contextvars import ContextVar, Token
|
|
33
|
+
from pathlib import Path
|
|
34
|
+
from typing import Any, TypeVar
|
|
35
|
+
|
|
36
|
+
# Re-export canonical model types from their authoritative modules
|
|
37
|
+
from agent_trace._replay.fixture import Fixture
|
|
38
|
+
from agent_trace.core.exceptions import NetworkGuardError
|
|
39
|
+
from agent_trace.core.span import Span, SpanStatus
|
|
40
|
+
from agent_trace.core.trace import Trace
|
|
41
|
+
from agent_trace.plugins.base import Plugin, PluginBase, SpanPlugin, TracePlugin
|
|
42
|
+
|
|
43
|
+
__version__ = "0.1.0"
|
|
44
|
+
|
|
45
|
+
logger = logging.getLogger(__name__)
|
|
46
|
+
|
|
47
|
+
__all__ = [
|
|
48
|
+
"Fixture",
|
|
49
|
+
"NetworkGuardError",
|
|
50
|
+
"Plugin",
|
|
51
|
+
"PluginBase",
|
|
52
|
+
"ReplayContext",
|
|
53
|
+
"Span",
|
|
54
|
+
"SpanPlugin",
|
|
55
|
+
"SpanStatus",
|
|
56
|
+
"Trace",
|
|
57
|
+
"TracePlugin",
|
|
58
|
+
"Tracer",
|
|
59
|
+
"replay",
|
|
60
|
+
"tracer",
|
|
61
|
+
]
|
|
62
|
+
|
|
63
|
+
F = TypeVar("F", bound=Callable[..., Any])
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class _BoundBotocoreSend:
|
|
67
|
+
"""Adapts an (instance, unbound-function) pair to a ``.send(request)`` object.
|
|
68
|
+
|
|
69
|
+
``botocore.httpsession.URLLib3Session.send`` is patched at the class
|
|
70
|
+
level (like ``requests.Session.get_adapter``), so the original
|
|
71
|
+
implementation is an unbound function that needs an explicit
|
|
72
|
+
``session_self`` to call. ``RecordingSession``/``ReplaySession`` expect
|
|
73
|
+
``inner`` to be an object exposing ``.send(request)`` (matching
|
|
74
|
+
``RecordingTransport``/``RecordingAdapter``'s ``inner`` parameter), so
|
|
75
|
+
this tiny shim re-binds the saved original to the real session instance
|
|
76
|
+
that's actually configured with the caller's proxy/SSL/timeout settings.
|
|
77
|
+
"""
|
|
78
|
+
|
|
79
|
+
__slots__ = ("_orig", "_session")
|
|
80
|
+
|
|
81
|
+
def __init__(self, session: Any, orig: Callable[..., Any]) -> None:
|
|
82
|
+
self._session = session
|
|
83
|
+
self._orig = orig
|
|
84
|
+
|
|
85
|
+
def send(self, request: Any) -> Any:
|
|
86
|
+
return self._orig(self._session, request)
|
|
87
|
+
|
|
88
|
+
def close(self) -> None:
|
|
89
|
+
close = getattr(self._session, "close", None)
|
|
90
|
+
if close is not None:
|
|
91
|
+
close()
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
# ---------------------------------------------------------------------------
|
|
95
|
+
# Tracer
|
|
96
|
+
# ---------------------------------------------------------------------------
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class Tracer:
|
|
100
|
+
"""Central orchestrator for trace collection.
|
|
101
|
+
|
|
102
|
+
Create one global instance (``tracer = Tracer()``) and use it across your
|
|
103
|
+
application. All methods are thread-safe and async-safe: each coroutine
|
|
104
|
+
or thread has its own active trace via ContextVar.
|
|
105
|
+
"""
|
|
106
|
+
|
|
107
|
+
def __init__(self, trace_dir: Path | None = None) -> None:
|
|
108
|
+
# Honours AGENT_TRACE_TRACE_DIR (same env var agent_trace._cli's
|
|
109
|
+
# _trace_dir() reads) so that a tracer constructed anonymously in a
|
|
110
|
+
# process started via `agent-trace run` — most importantly the
|
|
111
|
+
# global `tracer` singleton, which AGENT_TRACE_AUTO_RECORD activates
|
|
112
|
+
# on below — writes to the same location the CLI will later look in
|
|
113
|
+
# by default, without requiring the caller to thread trace_dir
|
|
114
|
+
# through by hand.
|
|
115
|
+
env_trace_dir = os.environ.get("AGENT_TRACE_TRACE_DIR")
|
|
116
|
+
default_trace_dir = Path.home() / ".agent-trace" / "runs"
|
|
117
|
+
self._trace_dir: Path = trace_dir or (
|
|
118
|
+
Path(env_trace_dir) if env_trace_dir else default_trace_dir
|
|
119
|
+
)
|
|
120
|
+
# ContextVar gives each async task / thread its own active trace.
|
|
121
|
+
# This replaces the previous threading.Lock + single attribute approach
|
|
122
|
+
# which was not safe for concurrent asyncio agents.
|
|
123
|
+
self._active_trace_var: ContextVar[Trace | None] = ContextVar(
|
|
124
|
+
"agent_trace_active_trace", default=None
|
|
125
|
+
)
|
|
126
|
+
# Transport-patch nesting counter and saved originals are initialised
|
|
127
|
+
# here so their types are declared once and getattr-with-default is
|
|
128
|
+
# never needed.
|
|
129
|
+
self._transport_depth: int = 0
|
|
130
|
+
# Stored as a (sync, async) tuple when patched; None otherwise.
|
|
131
|
+
self._original_httpx_transport_for_url: tuple[Any, Any] | None = None
|
|
132
|
+
self._original_requests_get_adapter: Any = None
|
|
133
|
+
# The fixture that HTTP calls made *in the current context* should be
|
|
134
|
+
# recorded into. ContextVar (not a plain attribute) so that each
|
|
135
|
+
# asyncio Task / thread gets its own independent value: two
|
|
136
|
+
# concurrently active start_trace(record=True) contexts (e.g. two
|
|
137
|
+
# in-flight requests in a server process) each see only their own
|
|
138
|
+
# fixture here, never each other's. The class-level httpx/requests
|
|
139
|
+
# patches installed by _patch_httpx/_patch_requests read this at
|
|
140
|
+
# request-dispatch time rather than closing over a single fixture
|
|
141
|
+
# captured whenever the patch happened to first be installed.
|
|
142
|
+
self._active_fixture_var: ContextVar[Fixture | None] = ContextVar(
|
|
143
|
+
"agent_trace_active_fixture", default=None
|
|
144
|
+
)
|
|
145
|
+
# Stored as a (insecure, secure) tuple when patched; None otherwise.
|
|
146
|
+
# aio variants are stored separately since grpc.aio is a lazy import
|
|
147
|
+
# (importing it eagerly would force asyncio C-extension loading for
|
|
148
|
+
# every agent-trace user, even ones who never touch gRPC).
|
|
149
|
+
self._original_grpc_channel_fns: tuple[Any, Any] | None = None
|
|
150
|
+
self._original_grpc_aio_channel_fns: tuple[Any, Any] | None = None
|
|
151
|
+
self._original_aiohttp_request: Any = None
|
|
152
|
+
self._original_botocore_session_send: Any = None
|
|
153
|
+
self._original_websockets_connect: Any = None
|
|
154
|
+
# Registered plugins — called on span and trace lifecycle events.
|
|
155
|
+
self._plugins: list[Plugin] = []
|
|
156
|
+
# Set by start_auto_record()/AGENT_TRACE_AUTO_RECORD when this
|
|
157
|
+
# tracer is recording process-wide, outside any `with
|
|
158
|
+
# start_trace(...)` block the caller owns — see start_auto_record()
|
|
159
|
+
# docstring. None when no auto-record session is active.
|
|
160
|
+
self._auto_record_state: dict[str, Any] | None = None
|
|
161
|
+
|
|
162
|
+
# ------------------------------------------------------------------
|
|
163
|
+
# Trace lifecycle
|
|
164
|
+
# ------------------------------------------------------------------
|
|
165
|
+
|
|
166
|
+
@contextmanager
|
|
167
|
+
def start_trace(
|
|
168
|
+
self,
|
|
169
|
+
name: str,
|
|
170
|
+
record: bool = False,
|
|
171
|
+
run_id: str | None = None,
|
|
172
|
+
trace_id: str | None = None,
|
|
173
|
+
remote_backend: Any = None,
|
|
174
|
+
) -> Generator[Trace, None, None]:
|
|
175
|
+
"""Start a trace, yield it, then save trace.json on exit.
|
|
176
|
+
|
|
177
|
+
Nested calls are supported — the inner trace saves/restores the outer
|
|
178
|
+
one in ``_active_trace_var``. If *record* is True, all outbound HTTP
|
|
179
|
+
calls during the context are captured into a SQLite fixture at
|
|
180
|
+
``run_dir/fixture.db``.
|
|
181
|
+
|
|
182
|
+
*trace_id*, when supplied, overrides the default random
|
|
183
|
+
``uuid.uuid4().hex``. Pass a value derived from a stable external
|
|
184
|
+
identity — e.g. a LangGraph run's ``thread_id``/checkpoint id via
|
|
185
|
+
``agent_trace.integrations.langgraph.derive_trace_id()`` — so that
|
|
186
|
+
two worker processes independently recording "the same" logical
|
|
187
|
+
operation (e.g. an original long-running tool call and its
|
|
188
|
+
checkpoint-swept re-dispatch on a managed platform) produce
|
|
189
|
+
traces sharing one ``trace_id`` and can be recognized/diffed as the
|
|
190
|
+
same logical run after the fact, instead of two unrelated,
|
|
191
|
+
un-linkable random UUIDs.
|
|
192
|
+
|
|
193
|
+
*remote_backend*, when supplied (a
|
|
194
|
+
``agent_trace.exporters.remote_fixture.RemoteFixtureBackend``,
|
|
195
|
+
requires *record* to also be True), durably uploads each HTTP
|
|
196
|
+
exchange to remote storage as it's recorded, and syncs the final
|
|
197
|
+
``trace.json``/``fixture.db`` on exit — so a worker killed or swept
|
|
198
|
+
mid-run on a managed platform (issue #7417) still has its recording
|
|
199
|
+
recoverable from remote storage, instead of only the worker's own
|
|
200
|
+
ephemeral, developer-inaccessible local filesystem.
|
|
201
|
+
"""
|
|
202
|
+
effective_run_id = run_id or f"run_{uuid.uuid4().hex[:12]}"
|
|
203
|
+
base = self._trace_dir.resolve()
|
|
204
|
+
run_dir = (base / effective_run_id).resolve()
|
|
205
|
+
try:
|
|
206
|
+
run_dir.relative_to(base)
|
|
207
|
+
except ValueError:
|
|
208
|
+
raise ValueError(
|
|
209
|
+
f"Invalid run_id {effective_run_id!r}: path traversal detected"
|
|
210
|
+
) from None
|
|
211
|
+
run_dir.mkdir(parents=True, exist_ok=True)
|
|
212
|
+
|
|
213
|
+
# trace_id must be 128-bit hex for OTLP; run_id is the human-readable
|
|
214
|
+
# directory name ("run_abc123"). Always generate them independently
|
|
215
|
+
# unless the caller supplied a deterministic trace_id explicitly.
|
|
216
|
+
trace = Trace(trace_id=trace_id or uuid.uuid4().hex, run_id=effective_run_id)
|
|
217
|
+
trace.metadata["name"] = name
|
|
218
|
+
token: Token[Trace | None] = self._active_trace_var.set(trace)
|
|
219
|
+
|
|
220
|
+
self._call_plugin("on_trace_start", trace)
|
|
221
|
+
|
|
222
|
+
on_exchange_recorded = self._remote_exchange_callback(
|
|
223
|
+
record, remote_backend, effective_run_id
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
# Use Fixture as a context manager when recording; nullcontext() when
|
|
227
|
+
# not, so fixture lifecycle and transport patching are always balanced.
|
|
228
|
+
fixture_ctx: Any = (
|
|
229
|
+
Fixture(
|
|
230
|
+
run_dir / "fixture.db",
|
|
231
|
+
trace_id=trace.trace_id,
|
|
232
|
+
on_exchange_recorded=on_exchange_recorded,
|
|
233
|
+
)
|
|
234
|
+
if record
|
|
235
|
+
else nullcontext()
|
|
236
|
+
)
|
|
237
|
+
try:
|
|
238
|
+
with fixture_ctx as fixture:
|
|
239
|
+
fixture_token: Token[Fixture | None] | None = None
|
|
240
|
+
if fixture is not None:
|
|
241
|
+
# Set the ContextVar *before* installing the patch so
|
|
242
|
+
# that any HTTP call made anywhere in this context (or a
|
|
243
|
+
# nested one) during the patch's lifetime resolves back
|
|
244
|
+
# to this trace's fixture, even under overlapping
|
|
245
|
+
# concurrent recordings.
|
|
246
|
+
fixture_token = self._active_fixture_var.set(fixture)
|
|
247
|
+
self._install_recording_transport()
|
|
248
|
+
try:
|
|
249
|
+
yield trace
|
|
250
|
+
except Exception as exc:
|
|
251
|
+
for span in trace.spans:
|
|
252
|
+
if span.end_time is None:
|
|
253
|
+
span.record_exception(exc)
|
|
254
|
+
span.end(SpanStatus.ERROR)
|
|
255
|
+
raise
|
|
256
|
+
finally:
|
|
257
|
+
if fixture is not None:
|
|
258
|
+
self._uninstall_recording_transport()
|
|
259
|
+
if fixture_token is not None:
|
|
260
|
+
self._active_fixture_var.reset(fixture_token)
|
|
261
|
+
finally:
|
|
262
|
+
trace_json_path = run_dir / "trace.json"
|
|
263
|
+
try:
|
|
264
|
+
trace_json_path.write_text(
|
|
265
|
+
json.dumps(trace.to_dict(), indent=2), encoding="utf-8"
|
|
266
|
+
)
|
|
267
|
+
except OSError as _write_err:
|
|
268
|
+
logger.warning(
|
|
269
|
+
"agent-trace: could not write trace.json to %s: %s",
|
|
270
|
+
trace_json_path,
|
|
271
|
+
_write_err,
|
|
272
|
+
)
|
|
273
|
+
self._sync_run_to_remote(remote_backend, run_dir, effective_run_id)
|
|
274
|
+
self._call_plugin("on_trace_end", trace)
|
|
275
|
+
self._active_trace_var.reset(token)
|
|
276
|
+
|
|
277
|
+
@staticmethod
|
|
278
|
+
def _remote_exchange_callback(
|
|
279
|
+
record: bool, remote_backend: Any, run_id: str
|
|
280
|
+
) -> Any:
|
|
281
|
+
"""Return an on_exchange_recorded callback wired to *remote_backend*
|
|
282
|
+
(durably uploading each exchange as it's recorded — see
|
|
283
|
+
agent_trace.exporters.remote_fixture), or None when recording isn't
|
|
284
|
+
active or no remote backend was supplied. Split out of start_trace()
|
|
285
|
+
purely to keep that method's own branch/statement count low."""
|
|
286
|
+
if not record or remote_backend is None:
|
|
287
|
+
return None
|
|
288
|
+
from agent_trace.exporters.remote_fixture import remote_sync_callback
|
|
289
|
+
|
|
290
|
+
return remote_sync_callback(remote_backend, run_id)
|
|
291
|
+
|
|
292
|
+
@staticmethod
|
|
293
|
+
def _sync_run_to_remote(remote_backend: Any, run_dir: Path, run_id: str) -> None:
|
|
294
|
+
"""Best-effort final sync of trace.json/fixture.db to *remote_backend*
|
|
295
|
+
on start_trace() exit. Split out purely to keep start_trace()'s own
|
|
296
|
+
branch/statement count low."""
|
|
297
|
+
if remote_backend is None:
|
|
298
|
+
return
|
|
299
|
+
try:
|
|
300
|
+
from agent_trace.exporters.remote_fixture import sync_run_to_remote
|
|
301
|
+
|
|
302
|
+
sync_run_to_remote(run_dir, remote_backend, run_id)
|
|
303
|
+
except Exception:
|
|
304
|
+
logger.warning(
|
|
305
|
+
"agent-trace: failed to sync run %s to remote backend",
|
|
306
|
+
run_id,
|
|
307
|
+
exc_info=True,
|
|
308
|
+
)
|
|
309
|
+
|
|
310
|
+
# ------------------------------------------------------------------
|
|
311
|
+
# Auto-record — process-wide activation with no enclosing `with` block
|
|
312
|
+
# ------------------------------------------------------------------
|
|
313
|
+
#
|
|
314
|
+
# start_trace()/instrument() both require the caller to own the
|
|
315
|
+
# top-level invocation so they have somewhere to put a `with
|
|
316
|
+
# tracer.start_trace(record=True):` block. That assumption breaks for
|
|
317
|
+
# any framework-managed server process the developer doesn't control
|
|
318
|
+
# the entrypoint of — e.g. `langgraph dev`/LangGraph Studio, which
|
|
319
|
+
# imports the developer's `make_graph()` once at server startup and
|
|
320
|
+
# then owns every subsequent invocation's lifecycle itself. The methods
|
|
321
|
+
# below (and the AGENT_TRACE_AUTO_RECORD env var read at import time,
|
|
322
|
+
# below the class) are the supported mechanism for that case: recording
|
|
323
|
+
# activates for the remaining lifetime of the process instead of one
|
|
324
|
+
# caller-scoped block.
|
|
325
|
+
|
|
326
|
+
def start_auto_record(
|
|
327
|
+
self,
|
|
328
|
+
name: str = "auto-record",
|
|
329
|
+
run_id: str | None = None,
|
|
330
|
+
) -> Path:
|
|
331
|
+
"""Activate process-wide recording with no enclosing `with` block.
|
|
332
|
+
|
|
333
|
+
Unlike :meth:`start_trace`, this has no natural "end" the caller is
|
|
334
|
+
expected to reach — it's meant for a process whose top-level
|
|
335
|
+
invocation the caller does not own (e.g. a `langgraph dev`/
|
|
336
|
+
LangGraph Studio server process). Everything that happens for the
|
|
337
|
+
remainder of the process — every HTTP exchange, every span opened
|
|
338
|
+
via :meth:`start_span`/:meth:`span` — is captured into one Trace
|
|
339
|
+
and one Fixture, exactly as :meth:`start_trace` does, except the
|
|
340
|
+
capture window is "until the process exits or
|
|
341
|
+
:meth:`stop_auto_record` is called" instead of "for the duration of
|
|
342
|
+
one `with` block".
|
|
343
|
+
|
|
344
|
+
An ``atexit`` hook is registered so ``trace.json``/``fixture.db``
|
|
345
|
+
are flushed on ordinary interpreter shutdown even if the caller
|
|
346
|
+
never calls :meth:`stop_auto_record` explicitly (the common case —
|
|
347
|
+
the process is killed by its supervisor, not shut down by the
|
|
348
|
+
developer's own code).
|
|
349
|
+
|
|
350
|
+
Idempotent: calling this while an auto-record session is already
|
|
351
|
+
active logs a warning and returns the existing session's run
|
|
352
|
+
directory unchanged, rather than leaking a second Fixture/patch
|
|
353
|
+
layer.
|
|
354
|
+
|
|
355
|
+
Returns the run directory (``trace_dir/<run_id>``) recording is
|
|
356
|
+
being written to.
|
|
357
|
+
|
|
358
|
+
Coarser-grained than :meth:`start_trace` by design: because there's
|
|
359
|
+
no well-defined caller-owned "end", a long-lived process
|
|
360
|
+
accumulates every exchange/span for its entire remaining lifetime
|
|
361
|
+
into a single trace/fixture pair, not one logical run per
|
|
362
|
+
invocation. Prefer :meth:`start_trace` whenever the caller genuinely
|
|
363
|
+
owns the top-level invocation.
|
|
364
|
+
"""
|
|
365
|
+
if self._auto_record_state is not None:
|
|
366
|
+
logger.warning(
|
|
367
|
+
"agent-trace: start_auto_record() called while an auto-record "
|
|
368
|
+
"session is already active (run_dir=%s) — ignoring.",
|
|
369
|
+
self._auto_record_state["run_dir"],
|
|
370
|
+
)
|
|
371
|
+
return self._auto_record_state["run_dir"] # type: ignore[no-any-return]
|
|
372
|
+
|
|
373
|
+
effective_run_id = run_id or f"run_{uuid.uuid4().hex[:12]}"
|
|
374
|
+
base = self._trace_dir.resolve()
|
|
375
|
+
run_dir = (base / effective_run_id).resolve()
|
|
376
|
+
try:
|
|
377
|
+
run_dir.relative_to(base)
|
|
378
|
+
except ValueError:
|
|
379
|
+
raise ValueError(
|
|
380
|
+
f"Invalid run_id {effective_run_id!r}: path traversal detected"
|
|
381
|
+
) from None
|
|
382
|
+
run_dir.mkdir(parents=True, exist_ok=True)
|
|
383
|
+
|
|
384
|
+
trace = Trace(trace_id=uuid.uuid4().hex, run_id=effective_run_id)
|
|
385
|
+
trace.metadata["name"] = name
|
|
386
|
+
trace.metadata["auto_record"] = True
|
|
387
|
+
trace_token = self._active_trace_var.set(trace)
|
|
388
|
+
|
|
389
|
+
fixture = Fixture(run_dir / "fixture.db", trace_id=trace.trace_id)
|
|
390
|
+
fixture_token = self._active_fixture_var.set(fixture)
|
|
391
|
+
self._install_recording_transport()
|
|
392
|
+
|
|
393
|
+
atexit_callback = self.stop_auto_record
|
|
394
|
+
atexit.register(atexit_callback)
|
|
395
|
+
|
|
396
|
+
self._auto_record_state = {
|
|
397
|
+
"run_dir": run_dir,
|
|
398
|
+
"trace": trace,
|
|
399
|
+
"trace_token": trace_token,
|
|
400
|
+
"fixture": fixture,
|
|
401
|
+
"fixture_token": fixture_token,
|
|
402
|
+
"atexit_callback": atexit_callback,
|
|
403
|
+
}
|
|
404
|
+
self._call_plugin("on_trace_start", trace)
|
|
405
|
+
logger.info("agent-trace: auto-record active — writing to %s", run_dir)
|
|
406
|
+
return run_dir
|
|
407
|
+
|
|
408
|
+
def stop_auto_record(self) -> None:
|
|
409
|
+
"""Stop a :meth:`start_auto_record` session, flushing
|
|
410
|
+
``trace.json``/closing ``fixture.db``. A no-op when no auto-record
|
|
411
|
+
session is active (safe to call from an ``atexit`` hook even after
|
|
412
|
+
an explicit call already ran)."""
|
|
413
|
+
state = self._auto_record_state
|
|
414
|
+
if state is None:
|
|
415
|
+
return
|
|
416
|
+
self._auto_record_state = None
|
|
417
|
+
|
|
418
|
+
try:
|
|
419
|
+
self._uninstall_recording_transport()
|
|
420
|
+
finally:
|
|
421
|
+
fixture: Fixture = state["fixture"]
|
|
422
|
+
try:
|
|
423
|
+
fixture.close()
|
|
424
|
+
except Exception:
|
|
425
|
+
logger.warning(
|
|
426
|
+
"agent-trace: error closing auto-record fixture", exc_info=True
|
|
427
|
+
)
|
|
428
|
+
try:
|
|
429
|
+
self._active_fixture_var.reset(state["fixture_token"])
|
|
430
|
+
except ValueError:
|
|
431
|
+
# reset() requires the same Context the token's set() ran
|
|
432
|
+
# in; an atexit callback can run in a different Context
|
|
433
|
+
# than the original start_auto_record() call. Best-effort —
|
|
434
|
+
# the ContextVar's process-lifetime value no longer matters
|
|
435
|
+
# once the process is shutting down.
|
|
436
|
+
pass
|
|
437
|
+
|
|
438
|
+
trace: Trace = state["trace"]
|
|
439
|
+
run_dir: Path = state["run_dir"]
|
|
440
|
+
try:
|
|
441
|
+
(run_dir / "trace.json").write_text(
|
|
442
|
+
json.dumps(trace.to_dict(), indent=2), encoding="utf-8"
|
|
443
|
+
)
|
|
444
|
+
except OSError as _write_err:
|
|
445
|
+
logger.warning(
|
|
446
|
+
"agent-trace: could not write trace.json to %s: %s",
|
|
447
|
+
run_dir / "trace.json",
|
|
448
|
+
_write_err,
|
|
449
|
+
)
|
|
450
|
+
self._call_plugin("on_trace_end", trace)
|
|
451
|
+
|
|
452
|
+
try:
|
|
453
|
+
self._active_trace_var.reset(state["trace_token"])
|
|
454
|
+
except ValueError:
|
|
455
|
+
pass
|
|
456
|
+
|
|
457
|
+
# atexit.unregister() is documented to never raise, even when the
|
|
458
|
+
# callback was never registered or was already removed.
|
|
459
|
+
atexit.unregister(state["atexit_callback"])
|
|
460
|
+
|
|
461
|
+
# ------------------------------------------------------------------
|
|
462
|
+
# Decorator
|
|
463
|
+
# ------------------------------------------------------------------
|
|
464
|
+
|
|
465
|
+
def instrument(
|
|
466
|
+
self,
|
|
467
|
+
record: bool = False,
|
|
468
|
+
name: str | None = None,
|
|
469
|
+
) -> Callable[[F], F]:
|
|
470
|
+
"""Decorator that wraps a function in :meth:`start_trace`.
|
|
471
|
+
|
|
472
|
+
Works for both sync and async functions::
|
|
473
|
+
|
|
474
|
+
@tracer.instrument(record=True)
|
|
475
|
+
async def my_agent(query: str) -> str:
|
|
476
|
+
...
|
|
477
|
+
"""
|
|
478
|
+
|
|
479
|
+
def decorator(fn: F) -> F:
|
|
480
|
+
trace_name = name or fn.__name__
|
|
481
|
+
|
|
482
|
+
if inspect.iscoroutinefunction(fn):
|
|
483
|
+
|
|
484
|
+
@functools.wraps(fn)
|
|
485
|
+
async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
|
|
486
|
+
with self.start_trace(trace_name, record=record):
|
|
487
|
+
return await fn(*args, **kwargs)
|
|
488
|
+
|
|
489
|
+
return async_wrapper # type: ignore[return-value]
|
|
490
|
+
|
|
491
|
+
@functools.wraps(fn)
|
|
492
|
+
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
|
493
|
+
with self.start_trace(trace_name, record=record):
|
|
494
|
+
return fn(*args, **kwargs)
|
|
495
|
+
|
|
496
|
+
return wrapper # type: ignore[return-value]
|
|
497
|
+
|
|
498
|
+
return decorator
|
|
499
|
+
|
|
500
|
+
# ------------------------------------------------------------------
|
|
501
|
+
# Span management
|
|
502
|
+
# ------------------------------------------------------------------
|
|
503
|
+
|
|
504
|
+
@contextmanager
|
|
505
|
+
def span(
|
|
506
|
+
self,
|
|
507
|
+
name: str,
|
|
508
|
+
parent_id: str | None = None,
|
|
509
|
+
) -> Generator[Span, None, None]:
|
|
510
|
+
"""Context manager that creates a span and auto-calls :meth:`Span.end`.
|
|
511
|
+
|
|
512
|
+
On success the span is closed with ``SpanStatus.OK``; on exception it
|
|
513
|
+
is closed with ``SpanStatus.ERROR`` and the exception is re-raised.
|
|
514
|
+
"""
|
|
515
|
+
s = self.start_span(name, parent_id=parent_id)
|
|
516
|
+
try:
|
|
517
|
+
yield s
|
|
518
|
+
except Exception as exc:
|
|
519
|
+
s.record_exception(exc)
|
|
520
|
+
if s.end_time is None:
|
|
521
|
+
s.end(SpanStatus.ERROR)
|
|
522
|
+
raise
|
|
523
|
+
else:
|
|
524
|
+
if s.end_time is None:
|
|
525
|
+
s.end(SpanStatus.OK)
|
|
526
|
+
|
|
527
|
+
def start_span(
|
|
528
|
+
self,
|
|
529
|
+
name: str,
|
|
530
|
+
parent_id: str | None = None,
|
|
531
|
+
) -> Span:
|
|
532
|
+
"""Create and register a :class:`Span` on the active trace.
|
|
533
|
+
|
|
534
|
+
If there is no active trace this is a no-op that returns a detached
|
|
535
|
+
span (it will not appear in any serialised output).
|
|
536
|
+
|
|
537
|
+
Registered plugins receive ``on_span_start`` immediately and
|
|
538
|
+
``on_span_end`` when ``span.end()`` is called.
|
|
539
|
+
"""
|
|
540
|
+
active = self._active_trace_var.get()
|
|
541
|
+
span_id = uuid.uuid4().hex[:16]
|
|
542
|
+
trace_id = active.trace_id if active is not None else uuid.uuid4().hex
|
|
543
|
+
s = Span(name=name, span_id=span_id, trace_id=trace_id, parent_id=parent_id)
|
|
544
|
+
if active is not None:
|
|
545
|
+
active.add_span(s)
|
|
546
|
+
self._call_plugin("on_span_start", s)
|
|
547
|
+
tracer_ref = self
|
|
548
|
+
original_end = s.end
|
|
549
|
+
|
|
550
|
+
def _plugin_end(status: SpanStatus = SpanStatus.OK) -> None:
|
|
551
|
+
original_end(status)
|
|
552
|
+
tracer_ref._call_plugin("on_span_end", s)
|
|
553
|
+
|
|
554
|
+
s.end = _plugin_end # type: ignore[method-assign]
|
|
555
|
+
return s
|
|
556
|
+
|
|
557
|
+
# ------------------------------------------------------------------
|
|
558
|
+
# Active trace accessor
|
|
559
|
+
# ------------------------------------------------------------------
|
|
560
|
+
|
|
561
|
+
@property
|
|
562
|
+
def active_trace(self) -> Trace | None:
|
|
563
|
+
"""The currently active :class:`Trace`, or None outside a trace context."""
|
|
564
|
+
return self._active_trace_var.get()
|
|
565
|
+
|
|
566
|
+
# ------------------------------------------------------------------
|
|
567
|
+
# Plugin API
|
|
568
|
+
# ------------------------------------------------------------------
|
|
569
|
+
|
|
570
|
+
def add_plugin(self, plugin: Plugin) -> None:
|
|
571
|
+
"""Register a plugin to receive span and trace lifecycle callbacks.
|
|
572
|
+
|
|
573
|
+
Plugins are called synchronously in registration order. Exceptions
|
|
574
|
+
inside a plugin hook are caught and logged so a buggy plugin cannot
|
|
575
|
+
silently break the caller.
|
|
576
|
+
|
|
577
|
+
Example::
|
|
578
|
+
|
|
579
|
+
from agent_trace.plugins import PluginBase
|
|
580
|
+
|
|
581
|
+
class LogPlugin(PluginBase):
|
|
582
|
+
def on_span_end(self, span):
|
|
583
|
+
print(span.name, span.duration_ms)
|
|
584
|
+
|
|
585
|
+
tracer.add_plugin(LogPlugin())
|
|
586
|
+
"""
|
|
587
|
+
if plugin not in self._plugins:
|
|
588
|
+
self._plugins.append(plugin)
|
|
589
|
+
|
|
590
|
+
def remove_plugin(self, plugin: Plugin) -> None:
|
|
591
|
+
"""Unregister a previously added plugin."""
|
|
592
|
+
try:
|
|
593
|
+
self._plugins.remove(plugin)
|
|
594
|
+
except ValueError:
|
|
595
|
+
pass
|
|
596
|
+
|
|
597
|
+
def _call_plugin(self, method: str, arg: Any) -> None:
|
|
598
|
+
"""Call *method* on every registered plugin, swallowing exceptions."""
|
|
599
|
+
for plugin in self._plugins:
|
|
600
|
+
fn = getattr(plugin, method, None)
|
|
601
|
+
if fn is not None:
|
|
602
|
+
try:
|
|
603
|
+
fn(arg)
|
|
604
|
+
except Exception:
|
|
605
|
+
logger.warning(
|
|
606
|
+
"agent-trace: plugin %r raised in %s — skipping",
|
|
607
|
+
plugin,
|
|
608
|
+
method,
|
|
609
|
+
exc_info=True,
|
|
610
|
+
)
|
|
611
|
+
|
|
612
|
+
# ------------------------------------------------------------------
|
|
613
|
+
# Recording transport patching
|
|
614
|
+
# ------------------------------------------------------------------
|
|
615
|
+
|
|
616
|
+
def _install_recording_transport(self) -> None:
|
|
617
|
+
"""Monkey-patch httpx and requests to record HTTP traffic.
|
|
618
|
+
|
|
619
|
+
Uses a nesting counter so that overlapping/nested
|
|
620
|
+
start_trace(record=True) calls install the class-level patch exactly
|
|
621
|
+
once and only remove it once the *last* active recording exits.
|
|
622
|
+
Only the outermost call installs; inner/overlapping calls are no-ops
|
|
623
|
+
on the patch itself.
|
|
624
|
+
|
|
625
|
+
The installed patches do not close over a single fixture — each one
|
|
626
|
+
resolves ``self._active_fixture_var.get()`` fresh at request-dispatch
|
|
627
|
+
time, so this is safe even when two recordings are simultaneously
|
|
628
|
+
active (see the ContextVar comment on ``_active_fixture_var``).
|
|
629
|
+
"""
|
|
630
|
+
self._transport_depth += 1
|
|
631
|
+
if self._transport_depth > 1:
|
|
632
|
+
return
|
|
633
|
+
self._patch_httpx()
|
|
634
|
+
self._patch_requests()
|
|
635
|
+
self._patch_grpc()
|
|
636
|
+
self._patch_aiohttp()
|
|
637
|
+
self._patch_botocore()
|
|
638
|
+
self._patch_websockets()
|
|
639
|
+
|
|
640
|
+
def _uninstall_recording_transport(self) -> None:
|
|
641
|
+
"""Restore the original patched methods.
|
|
642
|
+
|
|
643
|
+
Only the outermost trace uninstalls; inner traces decrement the counter.
|
|
644
|
+
"""
|
|
645
|
+
self._transport_depth = max(0, self._transport_depth - 1)
|
|
646
|
+
if self._transport_depth > 0:
|
|
647
|
+
return
|
|
648
|
+
self._unpatch_httpx()
|
|
649
|
+
self._unpatch_requests()
|
|
650
|
+
self._unpatch_grpc()
|
|
651
|
+
self._unpatch_aiohttp()
|
|
652
|
+
self._unpatch_botocore()
|
|
653
|
+
self._unpatch_websockets()
|
|
654
|
+
|
|
655
|
+
def _patch_httpx(self) -> None:
|
|
656
|
+
try:
|
|
657
|
+
import httpx
|
|
658
|
+
|
|
659
|
+
from agent_trace.interceptor.httpx_hook import (
|
|
660
|
+
AsyncRecordingTransport,
|
|
661
|
+
RecordingTransport,
|
|
662
|
+
)
|
|
663
|
+
|
|
664
|
+
# Patch at request-dispatch time (`_transport_for_url`, called by
|
|
665
|
+
# httpx internally on every single request/redirect hop) rather
|
|
666
|
+
# than at Client.__init__ time. This fixes two problems with the
|
|
667
|
+
# old __init__-time patch:
|
|
668
|
+
#
|
|
669
|
+
# 1. A client constructed *before* recording activates (e.g. an
|
|
670
|
+
# LLM client built once at module-import time, as
|
|
671
|
+
# `langgraph dev`/`make_graph()` entry points typically do)
|
|
672
|
+
# permanently kept its original transport under the old
|
|
673
|
+
# design, so recording could never see its traffic no matter
|
|
674
|
+
# when `start_trace(record=True)` was later entered.
|
|
675
|
+
# `_transport_for_url` is looked up fresh on every send(), so
|
|
676
|
+
# pre-existing clients are captured too.
|
|
677
|
+
# 2. A client constructed with an explicit `transport=` (e.g.
|
|
678
|
+
# langchain-openai's TCP-keepalive transport, or any SDK that
|
|
679
|
+
# pre-populates the `transport` kwarg) defeated the old
|
|
680
|
+
# `kwargs.setdefault("transport", ...)` silently — setdefault
|
|
681
|
+
# never fires when the key is already present. Here we always
|
|
682
|
+
# wrap whatever transport httpx would have used (default or
|
|
683
|
+
# caller-supplied, including per-URL `mounts=` transports)
|
|
684
|
+
# as `inner`, so nothing bypasses recording.
|
|
685
|
+
active_fixture_var = self._active_fixture_var
|
|
686
|
+
orig_sync = httpx.Client._transport_for_url
|
|
687
|
+
orig_async = httpx.AsyncClient._transport_for_url
|
|
688
|
+
|
|
689
|
+
def _patched_sync(client_self: Any, url: Any) -> Any:
|
|
690
|
+
base_transport = orig_sync(client_self, url)
|
|
691
|
+
fixture = active_fixture_var.get()
|
|
692
|
+
if fixture is None or isinstance(base_transport, RecordingTransport):
|
|
693
|
+
return base_transport
|
|
694
|
+
return RecordingTransport(fixture, inner=base_transport)
|
|
695
|
+
|
|
696
|
+
def _patched_async(client_self: Any, url: Any) -> Any:
|
|
697
|
+
base_transport = orig_async(client_self, url)
|
|
698
|
+
fixture = active_fixture_var.get()
|
|
699
|
+
if fixture is None or isinstance(
|
|
700
|
+
base_transport, AsyncRecordingTransport
|
|
701
|
+
):
|
|
702
|
+
return base_transport
|
|
703
|
+
return AsyncRecordingTransport(fixture, inner=base_transport)
|
|
704
|
+
|
|
705
|
+
self._original_httpx_transport_for_url = (orig_sync, orig_async)
|
|
706
|
+
setattr(httpx.Client, "_transport_for_url", _patched_sync)
|
|
707
|
+
setattr(httpx.AsyncClient, "_transport_for_url", _patched_async)
|
|
708
|
+
except ImportError:
|
|
709
|
+
pass
|
|
710
|
+
|
|
711
|
+
def _patch_requests(self) -> None:
|
|
712
|
+
try:
|
|
713
|
+
import requests
|
|
714
|
+
|
|
715
|
+
from agent_trace.interceptor.requests_patch import RecordingAdapter
|
|
716
|
+
|
|
717
|
+
# requests.Session.get_adapter(url) is already resolved fresh on
|
|
718
|
+
# every request (not at Session-construction time), so — unlike
|
|
719
|
+
# the old httpx.Client.__init__ patch — this already covers
|
|
720
|
+
# pre-existing Sessions and caller-mounted custom adapters
|
|
721
|
+
# correctly. The only change here is resolving the fixture from
|
|
722
|
+
# the ContextVar at call time instead of a closed-over value, so
|
|
723
|
+
# concurrent recordings route correctly too.
|
|
724
|
+
active_fixture_var = self._active_fixture_var
|
|
725
|
+
orig = requests.Session.get_adapter
|
|
726
|
+
|
|
727
|
+
def _patched(session_self: Any, url: str, **kwargs: Any) -> Any:
|
|
728
|
+
# Call the original dispatch so custom/mounted adapters are
|
|
729
|
+
# respected, then wrap the returned adapter to record the
|
|
730
|
+
# exchange for whichever trace is active in this context.
|
|
731
|
+
inner = orig(session_self, url, **kwargs)
|
|
732
|
+
fixture = active_fixture_var.get()
|
|
733
|
+
if fixture is None or isinstance(inner, RecordingAdapter):
|
|
734
|
+
return inner
|
|
735
|
+
return RecordingAdapter(fixture, inner=inner)
|
|
736
|
+
|
|
737
|
+
self._original_requests_get_adapter = orig
|
|
738
|
+
setattr(requests.Session, "get_adapter", _patched)
|
|
739
|
+
except ImportError:
|
|
740
|
+
pass
|
|
741
|
+
|
|
742
|
+
def _unpatch_httpx(self) -> None:
|
|
743
|
+
orig = self._original_httpx_transport_for_url
|
|
744
|
+
if orig is None:
|
|
745
|
+
return
|
|
746
|
+
try:
|
|
747
|
+
import httpx
|
|
748
|
+
|
|
749
|
+
orig_sync, orig_async = orig
|
|
750
|
+
setattr(httpx.Client, "_transport_for_url", orig_sync)
|
|
751
|
+
setattr(httpx.AsyncClient, "_transport_for_url", orig_async)
|
|
752
|
+
except ImportError:
|
|
753
|
+
pass
|
|
754
|
+
self._original_httpx_transport_for_url = None
|
|
755
|
+
|
|
756
|
+
def _unpatch_requests(self) -> None:
|
|
757
|
+
orig = self._original_requests_get_adapter
|
|
758
|
+
if orig is None:
|
|
759
|
+
return
|
|
760
|
+
try:
|
|
761
|
+
import requests
|
|
762
|
+
|
|
763
|
+
setattr(requests.Session, "get_adapter", orig)
|
|
764
|
+
except ImportError:
|
|
765
|
+
pass
|
|
766
|
+
self._original_requests_get_adapter = None
|
|
767
|
+
|
|
768
|
+
def _patch_grpc(self) -> None:
|
|
769
|
+
"""Monkey-patch grpc's channel factories to record every RPC.
|
|
770
|
+
|
|
771
|
+
grpc.insecure_channel/secure_channel are plain module-level
|
|
772
|
+
functions (not a shared base-class method the way httpx.Client is),
|
|
773
|
+
so we patch the module attribute itself. See
|
|
774
|
+
agent_trace/interceptor/grpc_hook.py's module docstring for why this
|
|
775
|
+
is the correct interception point for google-api-core-backed SDKs.
|
|
776
|
+
|
|
777
|
+
Like _patch_httpx/_patch_requests, the fixture is resolved from
|
|
778
|
+
self._active_fixture_var at call time (not closed over at patch-
|
|
779
|
+
install time) so concurrently active start_trace(record=True)
|
|
780
|
+
contexts each record into their own fixture.
|
|
781
|
+
"""
|
|
782
|
+
active_fixture_var = self._active_fixture_var
|
|
783
|
+
try:
|
|
784
|
+
import grpc
|
|
785
|
+
|
|
786
|
+
from agent_trace.interceptor.grpc_hook import GRPCRecordingInterceptor
|
|
787
|
+
|
|
788
|
+
orig_insecure = grpc.insecure_channel
|
|
789
|
+
orig_secure = grpc.secure_channel
|
|
790
|
+
|
|
791
|
+
def _patched_insecure(
|
|
792
|
+
target: str, options: Any = None, compression: Any = None
|
|
793
|
+
) -> Any:
|
|
794
|
+
channel = orig_insecure(
|
|
795
|
+
target, options=options, compression=compression
|
|
796
|
+
)
|
|
797
|
+
fixture = active_fixture_var.get()
|
|
798
|
+
if fixture is None:
|
|
799
|
+
return channel
|
|
800
|
+
return grpc.intercept_channel(
|
|
801
|
+
channel, GRPCRecordingInterceptor(fixture, target)
|
|
802
|
+
)
|
|
803
|
+
|
|
804
|
+
def _patched_secure(
|
|
805
|
+
target: str,
|
|
806
|
+
credentials: Any,
|
|
807
|
+
options: Any = None,
|
|
808
|
+
compression: Any = None,
|
|
809
|
+
) -> Any:
|
|
810
|
+
channel = orig_secure(
|
|
811
|
+
target, credentials, options=options, compression=compression
|
|
812
|
+
)
|
|
813
|
+
fixture = active_fixture_var.get()
|
|
814
|
+
if fixture is None:
|
|
815
|
+
return channel
|
|
816
|
+
return grpc.intercept_channel(
|
|
817
|
+
channel, GRPCRecordingInterceptor(fixture, target)
|
|
818
|
+
)
|
|
819
|
+
|
|
820
|
+
self._original_grpc_channel_fns = (orig_insecure, orig_secure)
|
|
821
|
+
grpc.insecure_channel = _patched_insecure
|
|
822
|
+
grpc.secure_channel = _patched_secure
|
|
823
|
+
except ImportError:
|
|
824
|
+
pass
|
|
825
|
+
|
|
826
|
+
try:
|
|
827
|
+
from grpc import aio
|
|
828
|
+
|
|
829
|
+
from agent_trace.interceptor.grpc_hook import AsyncGRPCRecordingInterceptor
|
|
830
|
+
|
|
831
|
+
orig_aio_insecure = aio.insecure_channel
|
|
832
|
+
orig_aio_secure = aio.secure_channel
|
|
833
|
+
|
|
834
|
+
def _patched_aio_insecure(target: str, **kwargs: Any) -> Any:
|
|
835
|
+
fixture = active_fixture_var.get()
|
|
836
|
+
if fixture is None:
|
|
837
|
+
return orig_aio_insecure(target, **kwargs)
|
|
838
|
+
interceptors = list(kwargs.pop("interceptors", None) or [])
|
|
839
|
+
interceptors.append(AsyncGRPCRecordingInterceptor(fixture, target))
|
|
840
|
+
return orig_aio_insecure(target, interceptors=interceptors, **kwargs)
|
|
841
|
+
|
|
842
|
+
def _patched_aio_secure(
|
|
843
|
+
target: str, credentials: Any, **kwargs: Any
|
|
844
|
+
) -> Any:
|
|
845
|
+
fixture = active_fixture_var.get()
|
|
846
|
+
if fixture is None:
|
|
847
|
+
return orig_aio_secure(target, credentials, **kwargs)
|
|
848
|
+
interceptors = list(kwargs.pop("interceptors", None) or [])
|
|
849
|
+
interceptors.append(AsyncGRPCRecordingInterceptor(fixture, target))
|
|
850
|
+
return orig_aio_secure(
|
|
851
|
+
target, credentials, interceptors=interceptors, **kwargs
|
|
852
|
+
)
|
|
853
|
+
|
|
854
|
+
self._original_grpc_aio_channel_fns = (orig_aio_insecure, orig_aio_secure)
|
|
855
|
+
aio.insecure_channel = _patched_aio_insecure
|
|
856
|
+
aio.secure_channel = _patched_aio_secure
|
|
857
|
+
except ImportError:
|
|
858
|
+
pass
|
|
859
|
+
|
|
860
|
+
def _unpatch_grpc(self) -> None:
|
|
861
|
+
orig = self._original_grpc_channel_fns
|
|
862
|
+
if orig is not None:
|
|
863
|
+
try:
|
|
864
|
+
import grpc
|
|
865
|
+
|
|
866
|
+
orig_insecure, orig_secure = orig
|
|
867
|
+
grpc.insecure_channel = orig_insecure
|
|
868
|
+
grpc.secure_channel = orig_secure
|
|
869
|
+
except ImportError:
|
|
870
|
+
pass
|
|
871
|
+
self._original_grpc_channel_fns = None
|
|
872
|
+
|
|
873
|
+
orig_aio = self._original_grpc_aio_channel_fns
|
|
874
|
+
if orig_aio is not None:
|
|
875
|
+
try:
|
|
876
|
+
from grpc import aio
|
|
877
|
+
|
|
878
|
+
orig_aio_insecure, orig_aio_secure = orig_aio
|
|
879
|
+
aio.insecure_channel = orig_aio_insecure
|
|
880
|
+
aio.secure_channel = orig_aio_secure
|
|
881
|
+
except ImportError:
|
|
882
|
+
pass
|
|
883
|
+
self._original_grpc_aio_channel_fns = None
|
|
884
|
+
|
|
885
|
+
def _patch_aiohttp(self) -> None:
|
|
886
|
+
"""Monkey-patch aiohttp.ClientSession._request to record every call.
|
|
887
|
+
|
|
888
|
+
Like _patch_httpx/_patch_requests/_patch_grpc, the fixture is
|
|
889
|
+
resolved from self._active_fixture_var at call time (not closed
|
|
890
|
+
over at patch-install time) so concurrently active
|
|
891
|
+
start_trace(record=True) contexts each record into their own
|
|
892
|
+
fixture. See agent_trace/interceptor/aiohttp_hook.py's module
|
|
893
|
+
docstring for why this interceptor exists alongside the
|
|
894
|
+
httpx/requests ones.
|
|
895
|
+
"""
|
|
896
|
+
active_fixture_var = self._active_fixture_var
|
|
897
|
+
try:
|
|
898
|
+
import aiohttp
|
|
899
|
+
|
|
900
|
+
from agent_trace.interceptor.aiohttp_hook import _record_exchange
|
|
901
|
+
|
|
902
|
+
orig_request = aiohttp.ClientSession._request
|
|
903
|
+
|
|
904
|
+
async def _patched_request(
|
|
905
|
+
session_self: Any, method: str, str_or_url: Any, **kwargs: Any
|
|
906
|
+
) -> Any:
|
|
907
|
+
response = await orig_request(
|
|
908
|
+
session_self, method, str_or_url, **kwargs
|
|
909
|
+
)
|
|
910
|
+
fixture = active_fixture_var.get()
|
|
911
|
+
if fixture is not None:
|
|
912
|
+
await _record_exchange(
|
|
913
|
+
fixture, method, str_or_url, kwargs, response
|
|
914
|
+
)
|
|
915
|
+
return response
|
|
916
|
+
|
|
917
|
+
self._original_aiohttp_request = orig_request
|
|
918
|
+
setattr(aiohttp.ClientSession, "_request", _patched_request)
|
|
919
|
+
except ImportError:
|
|
920
|
+
pass
|
|
921
|
+
|
|
922
|
+
def _unpatch_aiohttp(self) -> None:
|
|
923
|
+
orig = self._original_aiohttp_request
|
|
924
|
+
if orig is None:
|
|
925
|
+
return
|
|
926
|
+
try:
|
|
927
|
+
import aiohttp
|
|
928
|
+
|
|
929
|
+
setattr(aiohttp.ClientSession, "_request", orig)
|
|
930
|
+
except ImportError:
|
|
931
|
+
pass
|
|
932
|
+
self._original_aiohttp_request = None
|
|
933
|
+
|
|
934
|
+
def _patch_botocore(self) -> None:
|
|
935
|
+
"""Monkey-patch botocore's HTTP-dispatch layer to record AWS SDK calls.
|
|
936
|
+
|
|
937
|
+
Every boto3 service client (bedrock-runtime, sagemaker-runtime, s3,
|
|
938
|
+
...) routes its outbound HTTP request through
|
|
939
|
+
``botocore.httpsession.URLLib3Session.send`` — a single class method
|
|
940
|
+
shared by every ``Endpoint`` instance, regardless of service —
|
|
941
|
+
analogous to how ``requests.Session.get_adapter`` is shared across
|
|
942
|
+
Sessions. Patching it here means every AWS SDK client is captured
|
|
943
|
+
without the caller needing to configure anything.
|
|
944
|
+
|
|
945
|
+
Like _patch_httpx/_patch_requests/_patch_grpc/_patch_aiohttp, the
|
|
946
|
+
fixture is resolved from self._active_fixture_var at call time (not
|
|
947
|
+
closed over at patch-install time) so concurrently active
|
|
948
|
+
start_trace(record=True) contexts each record into their own
|
|
949
|
+
fixture.
|
|
950
|
+
"""
|
|
951
|
+
active_fixture_var = self._active_fixture_var
|
|
952
|
+
try:
|
|
953
|
+
import botocore.httpsession
|
|
954
|
+
|
|
955
|
+
from agent_trace.interceptor.botocore_hook import RecordingSession
|
|
956
|
+
|
|
957
|
+
orig = botocore.httpsession.URLLib3Session.send
|
|
958
|
+
|
|
959
|
+
def _patched(session_self: Any, request: Any) -> Any:
|
|
960
|
+
fixture = active_fixture_var.get()
|
|
961
|
+
if fixture is None:
|
|
962
|
+
return orig(session_self, request)
|
|
963
|
+
# Bind the real (unpatched) send to *this* session instance
|
|
964
|
+
# so its own connection pools/proxy/SSL config are preserved,
|
|
965
|
+
# then hand it to RecordingSession as the "inner" sender —
|
|
966
|
+
# mirroring how _patch_requests wraps the adapter returned by
|
|
967
|
+
# the real get_adapter() dispatch instead of constructing a
|
|
968
|
+
# fresh HTTPAdapter.
|
|
969
|
+
inner = _BoundBotocoreSend(session_self, orig)
|
|
970
|
+
return RecordingSession(fixture, inner=inner).send(request)
|
|
971
|
+
|
|
972
|
+
self._original_botocore_session_send = orig
|
|
973
|
+
setattr(botocore.httpsession.URLLib3Session, "send", _patched)
|
|
974
|
+
except ImportError:
|
|
975
|
+
pass
|
|
976
|
+
|
|
977
|
+
def _unpatch_botocore(self) -> None:
|
|
978
|
+
orig = self._original_botocore_session_send
|
|
979
|
+
if orig is None:
|
|
980
|
+
return
|
|
981
|
+
try:
|
|
982
|
+
import botocore.httpsession
|
|
983
|
+
|
|
984
|
+
setattr(botocore.httpsession.URLLib3Session, "send", orig)
|
|
985
|
+
except ImportError:
|
|
986
|
+
pass
|
|
987
|
+
self._original_botocore_session_send = None
|
|
988
|
+
|
|
989
|
+
def _patch_websockets(self) -> None:
|
|
990
|
+
"""Monkey-patch websockets.connect to record every WS session.
|
|
991
|
+
|
|
992
|
+
Like _patch_httpx/_patch_requests/_patch_grpc/_patch_aiohttp/
|
|
993
|
+
_patch_botocore, the fixture is resolved from
|
|
994
|
+
self._active_fixture_var at call time (not closed over at patch-
|
|
995
|
+
install time) so concurrently active start_trace(record=True)
|
|
996
|
+
contexts each record into their own fixture. When no trace is
|
|
997
|
+
actively recording, the real connect is called unwrapped.
|
|
998
|
+
"""
|
|
999
|
+
active_fixture_var = self._active_fixture_var
|
|
1000
|
+
try:
|
|
1001
|
+
import websockets
|
|
1002
|
+
|
|
1003
|
+
from agent_trace.interceptor.websocket_hook import RecordingConnect
|
|
1004
|
+
|
|
1005
|
+
orig_connect = websockets.connect
|
|
1006
|
+
|
|
1007
|
+
def _patched_connect(uri: str, *args: Any, **kwargs: Any) -> Any:
|
|
1008
|
+
fixture = active_fixture_var.get()
|
|
1009
|
+
if fixture is None:
|
|
1010
|
+
return orig_connect(uri, *args, **kwargs)
|
|
1011
|
+
return RecordingConnect(orig_connect, fixture, uri, *args, **kwargs)
|
|
1012
|
+
|
|
1013
|
+
self._original_websockets_connect = orig_connect
|
|
1014
|
+
setattr(websockets, "connect", _patched_connect)
|
|
1015
|
+
except ImportError:
|
|
1016
|
+
pass
|
|
1017
|
+
|
|
1018
|
+
def _unpatch_websockets(self) -> None:
|
|
1019
|
+
orig = self._original_websockets_connect
|
|
1020
|
+
if orig is None:
|
|
1021
|
+
return
|
|
1022
|
+
try:
|
|
1023
|
+
import websockets
|
|
1024
|
+
|
|
1025
|
+
setattr(websockets, "connect", orig)
|
|
1026
|
+
except ImportError:
|
|
1027
|
+
pass
|
|
1028
|
+
self._original_websockets_connect = None
|
|
1029
|
+
|
|
1030
|
+
|
|
1031
|
+
# ---------------------------------------------------------------------------
|
|
1032
|
+
# Global singleton
|
|
1033
|
+
# ---------------------------------------------------------------------------
|
|
1034
|
+
|
|
1035
|
+
tracer: Tracer = Tracer()
|
|
1036
|
+
|
|
1037
|
+
|
|
1038
|
+
# ---------------------------------------------------------------------------
|
|
1039
|
+
# AGENT_TRACE_AUTO_RECORD — process-wide auto-record activation, read once at
|
|
1040
|
+
# import time. This is the supported mechanism for attaching agent-trace to
|
|
1041
|
+
# an externally-managed server process (e.g. `langgraph dev`/LangGraph
|
|
1042
|
+
# Studio) that the developer does not own the top-level invocation of: set
|
|
1043
|
+
# the env var (directly, or via `agent-trace run -- <command>`, see
|
|
1044
|
+
# agent_trace._cli.cmd_run) before the process that imports `agent_trace`
|
|
1045
|
+
# starts, and recording activates on the global `tracer` singleton the
|
|
1046
|
+
# moment this module is first imported — no `with tracer.start_trace(...)`
|
|
1047
|
+
# block required anywhere in the developer's own code.
|
|
1048
|
+
#
|
|
1049
|
+
# AGENT_TRACE_AUTO_RECORD: "1"/"true"/"yes"/"on" (case-insensitive) enables.
|
|
1050
|
+
# Anything else (unset, "0", "false", ...) leaves the tracer untouched —
|
|
1051
|
+
# this whole block is then a no-op with zero overhead.
|
|
1052
|
+
# AGENT_TRACE_RUN_ID: optional explicit run_id (default: random).
|
|
1053
|
+
# AGENT_TRACE_AUTO_RECORD_NAME: optional trace name (default: "auto-record").
|
|
1054
|
+
# AGENT_TRACE_TRACE_DIR: honoured indirectly — the CLI's `agent-trace run`
|
|
1055
|
+
# sets it explicitly; the module-level `tracer` singleton otherwise uses its
|
|
1056
|
+
# own default (~/.agent-trace/runs), same as every other entry point.
|
|
1057
|
+
# ---------------------------------------------------------------------------
|
|
1058
|
+
|
|
1059
|
+
_AUTO_RECORD_TRUE_VALUES = frozenset({"1", "true", "yes", "on"})
|
|
1060
|
+
|
|
1061
|
+
|
|
1062
|
+
def _auto_record_enabled_from_env() -> bool:
|
|
1063
|
+
raw = os.environ.get("AGENT_TRACE_AUTO_RECORD", "")
|
|
1064
|
+
return raw.strip().lower() in _AUTO_RECORD_TRUE_VALUES
|
|
1065
|
+
|
|
1066
|
+
|
|
1067
|
+
def _activate_auto_record_from_env() -> None:
|
|
1068
|
+
"""Best-effort AGENT_TRACE_AUTO_RECORD activation on the global
|
|
1069
|
+
`tracer` singleton. Never raises — a misconfigured env var must not
|
|
1070
|
+
break importing `agent_trace` itself."""
|
|
1071
|
+
if not _auto_record_enabled_from_env():
|
|
1072
|
+
return
|
|
1073
|
+
try:
|
|
1074
|
+
tracer.start_auto_record(
|
|
1075
|
+
name=os.environ.get("AGENT_TRACE_AUTO_RECORD_NAME", "auto-record"),
|
|
1076
|
+
run_id=os.environ.get("AGENT_TRACE_RUN_ID") or None,
|
|
1077
|
+
)
|
|
1078
|
+
except Exception:
|
|
1079
|
+
logger.warning(
|
|
1080
|
+
"agent-trace: AGENT_TRACE_AUTO_RECORD activation failed", exc_info=True
|
|
1081
|
+
)
|
|
1082
|
+
|
|
1083
|
+
|
|
1084
|
+
_activate_auto_record_from_env()
|
|
1085
|
+
|
|
1086
|
+
|
|
1087
|
+
# ---------------------------------------------------------------------------
|
|
1088
|
+
# ReplayContext
|
|
1089
|
+
# ---------------------------------------------------------------------------
|
|
1090
|
+
|
|
1091
|
+
|
|
1092
|
+
class ReplayContext:
|
|
1093
|
+
"""Context manager returned by :func:`replay`.
|
|
1094
|
+
|
|
1095
|
+
Delegates to :func:`agent_trace._replay.engine.replay_context` so that the
|
|
1096
|
+
:class:`~agent_trace.core.clock.FixtureClock` is installed and network
|
|
1097
|
+
calls are served from the fixture without touching real endpoints.
|
|
1098
|
+
"""
|
|
1099
|
+
|
|
1100
|
+
def __init__(self, fixture_path: Path) -> None:
|
|
1101
|
+
self._fixture_path: Path = fixture_path
|
|
1102
|
+
self._fixture: Fixture | None = None
|
|
1103
|
+
self._ctx_manager: Any = None
|
|
1104
|
+
|
|
1105
|
+
def __enter__(self) -> ReplayContext:
|
|
1106
|
+
from agent_trace._replay.engine import replay_context
|
|
1107
|
+
|
|
1108
|
+
cm = replay_context(self._fixture_path)
|
|
1109
|
+
self._fixture = cm.__enter__()
|
|
1110
|
+
self._ctx_manager = cm
|
|
1111
|
+
return self
|
|
1112
|
+
|
|
1113
|
+
def __exit__(
|
|
1114
|
+
self,
|
|
1115
|
+
exc_type: type[BaseException] | None,
|
|
1116
|
+
exc_val: BaseException | None,
|
|
1117
|
+
exc_tb: Any,
|
|
1118
|
+
) -> bool | None:
|
|
1119
|
+
result: bool | None = None
|
|
1120
|
+
if self._ctx_manager is not None:
|
|
1121
|
+
result = self._ctx_manager.__exit__(exc_type, exc_val, exc_tb)
|
|
1122
|
+
return result
|
|
1123
|
+
|
|
1124
|
+
@property
|
|
1125
|
+
def fixture(self) -> Fixture:
|
|
1126
|
+
"""The underlying :class:`~agent_trace._replay.fixture.Fixture`."""
|
|
1127
|
+
if self._fixture is None:
|
|
1128
|
+
raise RuntimeError("ReplayContext must be used as a context manager.")
|
|
1129
|
+
return self._fixture
|
|
1130
|
+
|
|
1131
|
+
def get_metadata(self, key: str) -> str | None:
|
|
1132
|
+
"""Look up a metadata value stored in the fixture."""
|
|
1133
|
+
return self.fixture.get_metadata(key)
|
|
1134
|
+
|
|
1135
|
+
|
|
1136
|
+
# ---------------------------------------------------------------------------
|
|
1137
|
+
# replay() factory
|
|
1138
|
+
# ---------------------------------------------------------------------------
|
|
1139
|
+
|
|
1140
|
+
|
|
1141
|
+
def replay(
|
|
1142
|
+
run_id_or_path: str | Path,
|
|
1143
|
+
trace_dir: Path | None = None,
|
|
1144
|
+
) -> ReplayContext:
|
|
1145
|
+
"""Return a :class:`ReplayContext` for the given run ID or path.
|
|
1146
|
+
|
|
1147
|
+
*run_id_or_path* may be either:
|
|
1148
|
+
|
|
1149
|
+
- A path to a ``fixture.db`` file directly (e.g. ``Path("fixtures/fixture.db")``).
|
|
1150
|
+
- A run directory path (absolute or relative) containing ``fixture.db``.
|
|
1151
|
+
- A run-ID string like ``run_abc123`` that is resolved relative to
|
|
1152
|
+
*trace_dir* (default: ``~/.agent-trace/runs``).
|
|
1153
|
+
|
|
1154
|
+
Example::
|
|
1155
|
+
|
|
1156
|
+
with replay("run_abc123") as ctx:
|
|
1157
|
+
value = ctx.get_metadata("input")
|
|
1158
|
+
|
|
1159
|
+
with replay(Path("fixtures/fixture.db")) as ctx:
|
|
1160
|
+
result = my_agent()
|
|
1161
|
+
"""
|
|
1162
|
+
p = Path(run_id_or_path)
|
|
1163
|
+
if not p.is_absolute():
|
|
1164
|
+
base = Path(trace_dir or (Path.home() / ".agent-trace" / "runs")).resolve()
|
|
1165
|
+
p = (base / p).resolve()
|
|
1166
|
+
try:
|
|
1167
|
+
p.relative_to(base)
|
|
1168
|
+
except ValueError:
|
|
1169
|
+
raise ValueError(
|
|
1170
|
+
f"Invalid run path {run_id_or_path!r}: path traversal detected"
|
|
1171
|
+
) from None
|
|
1172
|
+
else:
|
|
1173
|
+
p = p.resolve()
|
|
1174
|
+
|
|
1175
|
+
fixture_path = p if p.suffix == ".db" else p / "fixture.db"
|
|
1176
|
+
if not fixture_path.exists():
|
|
1177
|
+
raise FileNotFoundError(
|
|
1178
|
+
f"No fixture.db found at {fixture_path}. "
|
|
1179
|
+
"Did you record this run with record=True?"
|
|
1180
|
+
)
|
|
1181
|
+
|
|
1182
|
+
return ReplayContext(fixture_path)
|