sensorium-trace 0.18.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (135) hide show
  1. sensorium/__init__.py +0 -0
  2. sensorium/__main__.py +4 -0
  3. sensorium/_c/rt/sensorium_c_rt.c +1446 -0
  4. sensorium/_typescript/package-lock.json +68 -0
  5. sensorium/_typescript/package.json +39 -0
  6. sensorium/_typescript/src/bindings.mjs +792 -0
  7. sensorium/_typescript/src/cap.mjs +227 -0
  8. sensorium/_typescript/src/capabilities.mjs +40 -0
  9. sensorium/_typescript/src/dbg.mjs +348 -0
  10. sensorium/_typescript/src/escape.mjs +351 -0
  11. sensorium/_typescript/src/focus.mjs +135 -0
  12. sensorium/_typescript/src/hook.mjs +127 -0
  13. sensorium/_typescript/src/index.mjs +1 -0
  14. sensorium/_typescript/src/naming.mjs +63 -0
  15. sensorium/_typescript/src/positions.mjs +37 -0
  16. sensorium/_typescript/src/probe.mjs +246 -0
  17. sensorium/_typescript/src/qualname.mjs +236 -0
  18. sensorium/_typescript/src/redact.mjs +711 -0
  19. sensorium/_typescript/src/register.mjs +15 -0
  20. sensorium/_typescript/src/resolve.mjs +260 -0
  21. sensorium/_typescript/src/rt.mjs +798 -0
  22. sensorium/_typescript/src/setup.mjs +11 -0
  23. sensorium/_typescript/src/tally.mjs +164 -0
  24. sensorium/_typescript/src/tasks.mjs +175 -0
  25. sensorium/_typescript/src/transform.mjs +800 -0
  26. sensorium/_typescript/src/vite.mjs +99 -0
  27. sensorium/c/__init__.py +8 -0
  28. sensorium/c/cc.py +612 -0
  29. sensorium/c/cli.py +60 -0
  30. sensorium/c/convert.py +669 -0
  31. sensorium/c/decode.py +300 -0
  32. sensorium/c/driver.py +365 -0
  33. sensorium/c/dwarf.py +706 -0
  34. sensorium/c/format.py +271 -0
  35. sensorium/c/pkg.py +187 -0
  36. sensorium/c/plan.py +277 -0
  37. sensorium/cli.py +140 -0
  38. sensorium/driver.py +47 -0
  39. sensorium/exit.py +33 -0
  40. sensorium/invocations.py +83 -0
  41. sensorium/limits.py +253 -0
  42. sensorium/mcp/__init__.py +0 -0
  43. sensorium/mcp/audit.py +162 -0
  44. sensorium/mcp/child.py +240 -0
  45. sensorium/mcp/cmd.py +101 -0
  46. sensorium/mcp/jsonrpc.py +172 -0
  47. sensorium/mcp/result.py +187 -0
  48. sensorium/mcp/schema.py +390 -0
  49. sensorium/mcp/server.py +493 -0
  50. sensorium/mcp/tools.py +299 -0
  51. sensorium/paths.py +80 -0
  52. sensorium/query/__init__.py +0 -0
  53. sensorium/query/caps.py +150 -0
  54. sensorium/query/dbg_dialects.py +64 -0
  55. sensorium/query/diff_cmd.py +748 -0
  56. sensorium/query/diff_notes.py +117 -0
  57. sensorium/query/exceptions_c.py +263 -0
  58. sensorium/query/exceptions_cmd.py +788 -0
  59. sensorium/query/exceptions_group.py +571 -0
  60. sensorium/query/exceptions_invocation.py +487 -0
  61. sensorium/query/exceptions_rust.py +683 -0
  62. sensorium/query/exceptions_typescript.py +688 -0
  63. sensorium/query/expr.py +606 -0
  64. sensorium/query/failures_cmd.py +306 -0
  65. sensorium/query/flow_cmd.py +778 -0
  66. sensorium/query/flow_report.py +128 -0
  67. sensorium/query/flow_values.py +195 -0
  68. sensorium/query/fmt.py +280 -0
  69. sensorium/query/frame_cmd.py +237 -0
  70. sensorium/query/grep_cmd.py +156 -0
  71. sensorium/query/help.py +18 -0
  72. sensorium/query/info_cmd.py +527 -0
  73. sensorium/query/info_rust.py +214 -0
  74. sensorium/query/info_typescript.py +261 -0
  75. sensorium/query/js_inspect.py +394 -0
  76. sensorium/query/moves.py +254 -0
  77. sensorium/query/redact_cmd.py +304 -0
  78. sensorium/query/refocus_cmd.py +661 -0
  79. sensorium/query/refocus_env.py +493 -0
  80. sensorium/query/refocus_facts.py +269 -0
  81. sensorium/query/refocus_licence.py +187 -0
  82. sensorium/query/refocus_report.py +327 -0
  83. sensorium/query/refocus_rust.py +537 -0
  84. sensorium/query/refocus_threads.py +197 -0
  85. sensorium/query/refocus_typescript.py +756 -0
  86. sensorium/query/refocus_world.py +666 -0
  87. sensorium/query/runs_cmd.py +213 -0
  88. sensorium/query/rust_debug.py +176 -0
  89. sensorium/query/sites.py +171 -0
  90. sensorium/query/tree_cmd.py +511 -0
  91. sensorium/query/tree_threads.py +168 -0
  92. sensorium/query/vocab.py +723 -0
  93. sensorium/query/watch_cmd.py +657 -0
  94. sensorium/record/__init__.py +0 -0
  95. sensorium/record/boot.py +797 -0
  96. sensorium/record/boot_io.py +141 -0
  97. sensorium/record/budget.py +135 -0
  98. sensorium/record/capture.py +433 -0
  99. sensorium/record/children.py +755 -0
  100. sensorium/record/children_merge.py +179 -0
  101. sensorium/record/elide.py +55 -0
  102. sensorium/record/fingerprint.py +26 -0
  103. sensorium/record/thread_starts.py +261 -0
  104. sensorium/record/tracer.py +798 -0
  105. sensorium/record/tracer_exc.py +199 -0
  106. sensorium/record/tracer_frames.py +618 -0
  107. sensorium/redact.py +304 -0
  108. sensorium/redact_content.py +156 -0
  109. sensorium/redact_key.py +317 -0
  110. sensorium/redact_store.py +502 -0
  111. sensorium/redact_values.py +433 -0
  112. sensorium/store/__init__.py +0 -0
  113. sensorium/store/db.py +284 -0
  114. sensorium/store/gc.py +360 -0
  115. sensorium/store/payloads.py +69 -0
  116. sensorium/store/reader.py +438 -0
  117. sensorium/store/writer.py +283 -0
  118. sensorium/ts/__init__.py +10 -0
  119. sensorium/ts/build.py +783 -0
  120. sensorium/ts/cli.py +130 -0
  121. sensorium/ts/driver.py +472 -0
  122. sensorium/ts/focus.py +146 -0
  123. sensorium/ts/harness.py +314 -0
  124. sensorium/ts/ingest.py +380 -0
  125. sensorium/ts/invocation.py +252 -0
  126. sensorium/ts/pkg.py +167 -0
  127. sensorium/ts/redaction.py +64 -0
  128. sensorium/ts/spool.py +168 -0
  129. sensorium/ts/truncation.py +81 -0
  130. sensorium/ts/wrapper.py +255 -0
  131. sensorium_trace-0.18.2.dist-info/METADATA +245 -0
  132. sensorium_trace-0.18.2.dist-info/RECORD +135 -0
  133. sensorium_trace-0.18.2.dist-info/WHEEL +4 -0
  134. sensorium_trace-0.18.2.dist-info/entry_points.txt +3 -0
  135. sensorium_trace-0.18.2.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,798 @@
1
+ """sys.monitoring-based recorder.
2
+
3
+ Default tier: CALL/RETURN/RAISE/HANDLED for user code (files under root,
4
+ excluding stdlib/site-packages/sensorium itself). Focus tier adds LINE events
5
+ carrying local-variable deltas for focused code. Re-entrancy: recorder frames
6
+ are never traced and capture runs behind a thread-local in_hook flag.
7
+
8
+ LINE events, and every value captured anywhere, are NEVER fingerprinted: a
9
+ fingerprint must depend only on the (code object, causal kind) sequence, which
10
+ is what lets `refocus` re-run a program with deeper capture and prove it was
11
+ the same execution. `_on_line` must never touch `_fp`.
12
+
13
+ A LINE event fires BEFORE its line runs. So the deltas on a LINE event describe
14
+ state produced by the PRECEDING executed line, while the event's `line` column
15
+ is the line that is about to execute. This is deliberate: read a run of LINE
16
+ events as a state timeline, where each row says "just before line N, these
17
+ locals had just become these values". `deltas` holds only the names whose
18
+ captured value differs from the previous capture in that same frame, and no
19
+ event is written at all when nothing changed.
20
+
21
+ Names that went away carry their own sibling key: `payload["unbound"]` is a
22
+ sorted list of names bound at the previous line and gone at this one, present
23
+ only when non-empty. It is kept out of `deltas` so every delta value stays a
24
+ `capture_value` result and consumers need no type check. Both `del x` and the
25
+ implicit unbind that ends an `except ... as x` handler are reported this way,
26
+ and an unbind alone is enough to emit an event -- otherwise a `del` whose line
27
+ changes nothing else would leave no trace, and a consumer folding deltas would
28
+ keep a dead binding alive for the rest of the frame.
29
+
30
+ A LINE event's `frame_id` is always set. Every traced code object opens a
31
+ frame, generators and coroutines included (arc 2); a suspendable frame keeps
32
+ its entry across its suspensions -- parked in a tracer-level table between
33
+ YIELD and RESUME, so the thread that resumes it need not be the one that
34
+ opened it -- and leaves through PY_RETURN/PY_UNWIND like any other. The
35
+ dropped-while-suspended case arrives as PY_THROW(GeneratorExit) + PY_UNWIND.
36
+ Some point releases close a simple generator without those events; install
37
+ measures that once and records the same RESUME and unwind when it happens.
38
+ Each suspension is recorded: YIELD names the TYPE being awaited and RESUME
39
+ carries the exception thrown in, if any. Neither is fingerprinted --
40
+ how often a coroutine parks is the event loop's business, not the program's.
41
+ An exception thrown into a suspended TRACED frame is `identify`d at PY_THROW
42
+ so its RESUME row and the PY_UNWIND that follows carry ONE serial; that
43
+ equality, not the type, is what lets a reader call a frame cancelled or
44
+ abandoned rather than merely raised. Untraced frames mint nothing: PY_THROW
45
+ fires on the frame that will unwind, so an untraced one writes no row at
46
+ either end and a serial for it would only evict a live one. A call whose
47
+ caller has no open frame gets `parent_id NULL` and `caller_code` / `caller`
48
+ naming who called it: the parent is never guessed.
49
+
50
+ Every RAISE/HANDLED payload carries an exception `serial`: an exact per-thread
51
+ identity, minted the first time this recorder sees an exception object and
52
+ reused for every later event of that same object. It exists because `oid`
53
+ (`id(exc)`) is NOT an identity -- CPython recycles addresses, and a plain retry
54
+ loop measurably gives three distinct exceptions one address. Serials never
55
+ reach a fingerprint: `Fingerprint.update` takes only (file, qualname, kind),
56
+ and payloads are not hashed, which is what keeps `refocus` verdicts stable.
57
+
58
+ Serials live in a per-thread TABLE (`_ExcRefs`), never in a "current serial"
59
+ slot. Slots cannot express "several exceptions are alive and any of them may
60
+ come back", and both shapes that broke earlier attempts are exactly that:
61
+
62
+ * an exception stored by a handler (`except E as e: return e`) and raised
63
+ again after some *other* exception has been handled in between, and
64
+ * an exception in flight through a `finally` that raises and handles an
65
+ exception of its own before the original resumes.
66
+
67
+ With one slot the first loses its serial and the second inherits the
68
+ interloper's -- one object under two identities, or two objects under one.
69
+ A table keyed by the object has neither failure.
70
+
71
+ The one place this recorder measurably outlives the program's own references:
72
+ that table holds a STRONG reference to each exception it remembers. It has to
73
+ be strong -- `BaseException` does not support weak references at all -- and
74
+ being strong is also what makes `id(exc)` a sound key, since a retained
75
+ address cannot be recycled while we hold the object. The cost is real and is
76
+ bounded on purpose: each retained exception pins its traceback and therefore
77
+ the frames and locals reachable from it, so at most `_RETAIN_MAX` per live
78
+ thread are kept, the oldest being dropped when a newer one arrives. Past that
79
+ bound the recorder simply forgets, a later raise of a forgotten object mints a
80
+ fresh serial, and `sensorium exceptions` refuses to call such an exception
81
+ swallowed rather than pretending the two are unrelated. Control-flow
82
+ exceptions thrown INTO a frame are serialled from a second, much smaller
83
+ table (`_CONTROL_RETAIN_MAX`), so that dropping generators -- which ordinary
84
+ code does constantly -- cannot evict the real exceptions this machinery
85
+ exists to link. A thread's tables die with the thread, and `uninstall` clears
86
+ every live thread's tables and in-flight slot, after which this recorder
87
+ holds no exception object at all.
88
+ """
89
+ import inspect
90
+ import sys
91
+ import threading
92
+ import time
93
+ import weakref
94
+ from pathlib import Path
95
+
96
+ from sensorium import redact_values as rv
97
+ from sensorium.record.capture import (capture_exc, capture_value, plain_str,
98
+ type_name)
99
+ from sensorium.record.fingerprint import Fingerprint
100
+ # The exception-retention layer, split out at this file's 800-line ceiling.
101
+ # Re-exported so `tracer.<name>` keeps resolving: these are one recorder's
102
+ # internals across its files, not separate modules with surfaces of their
103
+ # own.
104
+ from sensorium.record.tracer_exc import ( # noqa: F401
105
+ _CONTROL_RETAIN_MAX, _RETAIN_MAX, _ExcRefs, _TLS, _is_control_flow)
106
+ # The frame-classification layer, split out at the same ceiling, re-exported
107
+ # for the same reason.
108
+ from sensorium.record.elide import materialize_elided
109
+ from sensorium.record.tracer_frames import ( # noqa: F401
110
+ FocusSpec, WindowSpec, _FrameDecisions, detect_silent_generator_close,
111
+ module_name_for)
112
+
113
+ M = sys.monitoring
114
+ TOOL = M.PROFILER_ID
115
+
116
+
117
+ def _call_arg_names(code):
118
+ """Every parameter name, including `*args` and `**kwargs`.
119
+
120
+ `co_varnames` lays them out as positional, then keyword-only, then the
121
+ variadic names when the flags say they exist, then the other locals.
122
+ The slice that stops at `co_argcount + co_kwonlyargcount` drops the
123
+ variadic bindings, so a call's `*args` and `**kwargs` never reached
124
+ the trace.
125
+ """
126
+ n = code.co_argcount + code.co_kwonlyargcount
127
+ flags = code.co_flags
128
+ if flags & inspect.CO_VARARGS:
129
+ n += 1
130
+ if flags & inspect.CO_VARKEYWORDS:
131
+ n += 1
132
+ return code.co_varnames[:n]
133
+
134
+
135
+ def locals_snapshot(frame) -> dict | None:
136
+ """`frame.f_locals` as a plain dict with exact-`str` keys, or None.
137
+
138
+ Reading a frame's locals looks like reading interpreter data. Two parts
139
+ of it are the observed program's own code:
140
+
141
+ * **`f_locals` may BE a mapping the program supplied.** `exec(code,
142
+ globals, mapping)` and a metaclass `__prepare__` both hand the frame an
143
+ arbitrary object, so `.items()` is an overridable method. Measured: a
144
+ `dict` subclass whose `items()` raises killed a traced program that
145
+ runs clean standalone, at this line.
146
+ * **The KEYS may be `str` subclasses.** A class body's `locals()[K("x")]
147
+ = 1`, or the 3.13+ write-through `f_locals` proxy, puts a program-
148
+ defined object where a name should be -- and the recorder then hashes
149
+ it, compares it (`prev.get(name)`, the set difference) and SORTS it
150
+ (`sorted(gone)`). Measured: two such keys going out of scope on one
151
+ line reached `sorted`, whose `__lt__` killed the program and was then
152
+ reported by `exceptions` as the program's own uncaught bug, at a line
153
+ that never raised.
154
+
155
+ So: one guarded read of `.items()`, then exact-`str` keys built by
156
+ `plain_str`, which cannot raise. Nothing downstream touches a key the
157
+ program owns. None means the locals could not be read at all -- the
158
+ callers record that, they do not paper over it.
159
+
160
+ Two names that normalise to the SAME string collapse, last one wins.
161
+ That is the honest outcome: the plain string is the only name the trace
162
+ can report, and a program that binds both `x` and `K("x")` has two
163
+ things the trace cannot tell apart by name anyway.
164
+ """
165
+ try:
166
+ items = list(frame.f_locals.items())
167
+ except BaseException:
168
+ return None
169
+ return {plain_str(k): v for k, v in items}
170
+
171
+
172
+ class Tracer(_FrameDecisions):
173
+ def __init__(self, writer, root: Path, focus: FocusSpec,
174
+ include=(), exclude=(), window: str | None = None,
175
+ limits=None) -> None:
176
+ self.writer = writer
177
+ self.root = Path(root).resolve()
178
+ self.focus = focus
179
+ self.include = tuple(include)
180
+ self.exclude = tuple(exclude)
181
+ self.window = WindowSpec(window)
182
+ # id(code) -> (traced, fp_file, qualname, focused, kind, win_key)
183
+ self._decisions: dict[int, tuple] = {}
184
+ # id(frame) -> the live entry of a frame SUSPENDED right now, for
185
+ # every thread at once. A suspended frame runs on no thread, so the
186
+ # thread that resumes it need not be the one that opened it: a
187
+ # generator first `next`ed where it was made and stepped from a
188
+ # thread pool afterwards (Starlette's `iterate_in_threadpool`, which
189
+ # is what `StreamingResponse(sync_gen)` does) resumes on a worker,
190
+ # where a per-thread map cannot possibly hold its entry. An entry is
191
+ # owned HERE or by exactly one thread's `live`, never by both.
192
+ self._parked: dict[int, list] = {}
193
+ self._parked_lock = threading.Lock()
194
+ # Set in install, from a one-shot probe. True only when this
195
+ # interpreter closes a simple generator without PY_UNWIND.
196
+ self._silent_gen_close = False
197
+ self._silent_closed: set[int] = set()
198
+ self._seen_codes: list = [] # pins code objects so id() cannot recycle
199
+ self._fps: dict[int, Fingerprint] = {}
200
+ self._task_fps: dict[int, Fingerprint] = {} # task serial -> fp
201
+ self._fp_lock = threading.Lock()
202
+ # Weak values: a thread's table dies with the thread, so a program
203
+ # that spawns thousands of short-lived threads does not accumulate
204
+ # their retained exceptions here.
205
+ self._exc_refs: weakref.WeakValueDictionary = weakref.WeakValueDictionary()
206
+ self._refs_lock = threading.Lock()
207
+ # Recorded thread identity: a monotonic serial minted once per distinct
208
+ # thread, NOT `threading.get_ident()`, which the OS recycles once a
209
+ # thread ends -- reuse would merge two short-lived threads' events,
210
+ # frames and fingerprints under one id.
211
+ self._serial_lock = threading.Lock()
212
+ self._next_serial = 0
213
+ # asyncio task identity: a minted serial per task object, weakly held
214
+ # so finished tasks do not accumulate. Bound lazily from sys.modules
215
+ # (never imported here) so a program that never uses asyncio pays one
216
+ # dict probe per event and sees its sys.modules untouched.
217
+ self._asyncio: tuple | None = None
218
+ self._task_serials: weakref.WeakKeyDictionary = weakref.WeakKeyDictionary()
219
+ self._task_lock = threading.Lock()
220
+ self._next_task = 0
221
+ self.task_errors = 0 # lookups a hostile task object broke
222
+ # None until `thread_starts.arm`. Set before the first mint: the
223
+ # main thread's serial is assigned by the `_TLS` call below, and
224
+ # that mint must not look up a missing attribute.
225
+ self._thread_born = None
226
+ # Set in the child after os.fork. Inherited frames are still on the
227
+ # stack; their returns belong to the parent and must not land in the
228
+ # child's spool. New calls after the fork are in `live`.
229
+ self._fork_child = False
230
+ self._tls = _TLS(self._assign_thread_serial, self._register_refs)
231
+ # The thread constructing the Tracer runs `_TLS.__init__` above, so it
232
+ # is the first serial (1). `run_target` runs the target on this same
233
+ # thread, which is why this is the run's main-thread identity.
234
+ self.main_thread_serial: int = self._tls.thread_serial
235
+ if limits is None:
236
+ from sensorium.limits import resolve
237
+ limits = resolve()
238
+ self.limits = limits
239
+ self._budget = None
240
+ if limits.engaged:
241
+ from sensorium.record.budget import Budget
242
+ self._budget = Budget(limits, writer)
243
+
244
+ def _parked_has(self, key: int) -> bool:
245
+ with self._parked_lock:
246
+ return key in self._parked
247
+
248
+ def _assign_thread_serial(self) -> int:
249
+ with self._serial_lock:
250
+ self._next_serial += 1
251
+ serial = self._next_serial
252
+ # Mint is the first moment a new thread exists inside this recorder.
253
+ # The hook joins the serial to the Thread that started it. The OS
254
+ # ident cannot: it is recycled, and events carry this serial. The
255
+ # hook runs from a monitoring callback, so a failure must not escape.
256
+ hook = self._thread_born
257
+ if hook is not None:
258
+ try:
259
+ hook(serial)
260
+ except BaseException:
261
+ pass
262
+ return serial
263
+
264
+ def _register_refs(self, serial: int, refs: _ExcRefs) -> None:
265
+ with self._refs_lock:
266
+ self._exc_refs[serial] = refs
267
+
268
+ def _live_exc_refs(self) -> list:
269
+ with self._refs_lock:
270
+ return list(self._exc_refs.values())
271
+
272
+ # -- callbacks ---------------------------------------------------------
273
+ def _on_start(self, code, offset):
274
+ tls = self._tls
275
+ if tls.in_hook:
276
+ return None
277
+ budget = self._budget
278
+ if budget is not None:
279
+ slot = budget.slot(id(code))
280
+ if slot is not None:
281
+ budget.bump(slot)
282
+ return None
283
+ traced, fp_file, qual, focused, kind, win_key = self._decide(code)
284
+ if not traced:
285
+ return M.DISABLE
286
+ if budget is not None and budget.should_elide(
287
+ id(code), code.co_filename, qual, code.co_firstlineno):
288
+ return None
289
+ tls.in_hook = True
290
+ try:
291
+ frame = sys._getframe(1)
292
+ names = _call_arg_names(code)
293
+ # Through the same snapshot as `_on_line`, for the same reason.
294
+ # A frame with parameters always has interpreter-built locals, so
295
+ # this path is not the one that was reachable -- but "reachable
296
+ # only because of the order things happen in" is the argument
297
+ # this project has already watched rot twice, and the snapshot
298
+ # makes it structural instead.
299
+ loc = locals_snapshot(frame)
300
+ # Rule v1 applies HERE, before the payload exists: an argument
301
+ # bound to a secret-shaped NAME is taken whole, and every text
302
+ # in every other one is scanned. `capture.py` stays pure -- it
303
+ # decides what a value looks like, never what may be stored.
304
+ args = ({} if loc is None else
305
+ {n: rv.named(n, capture_value(loc[n]))
306
+ for n in names if n in loc})
307
+ payload = {"args": args} if loc is not None else {
308
+ "args": {}, "unread": ["locals"]}
309
+ tid = tls.thread_serial
310
+ task = self._task_serial(tls)
311
+ cid = self.writer.intern_code(code.co_filename, qual,
312
+ code.co_firstlineno)
313
+ caller = frame.f_back
314
+ parent = self._parent_of(tls, caller)
315
+ if parent is None:
316
+ self._note_caller(payload, caller)
317
+ # `values` counts what the TRACE HOLDS (B3), so it is counted
318
+ # at the WRITE: one per capture in the payload carrying a
319
+ # `redacted` object. The rule's transforms count nothing, and a
320
+ # capture the recorder builds but does not store -- every
321
+ # unchanged local, at every line -- is never counted.
322
+ rv.stats["values"] += rv.count(payload)
323
+ eid = self.writer.add_event(time.monotonic_ns(), tid, "CALL",
324
+ None, cid, code.co_firstlineno,
325
+ payload, task_id=task)
326
+ pfid = parent[0] if parent is not None else None
327
+ depth = parent[4] + 1 if parent is not None else 0
328
+ fid = self.writer.open_frame(pfid, cid, eid, depth, tid, kind)
329
+ in_window = bool(win_key is not None
330
+ or (parent is not None and parent[5]))
331
+ tls.live[id(frame)] = [fid, code, cid, {}, depth, in_window,
332
+ False]
333
+ self._fp_for(tid, task).update(fp_file, qual, "CALL")
334
+ if budget is not None:
335
+ budget.note_kept(id(code), code.co_filename, qual,
336
+ code.co_firstlineno)
337
+ finally:
338
+ tls.in_hook = False
339
+ return None
340
+
341
+ def _on_return(self, code, offset, retval):
342
+ tls = self._tls
343
+ if tls.in_hook:
344
+ return None
345
+ budget = self._budget
346
+ if budget is not None and budget.slot(id(code)) is not None:
347
+ frame = sys._getframe(1)
348
+ key = id(frame)
349
+ if key not in tls.live and not self._parked_has(key):
350
+ return None
351
+ traced, fp_file, qual, focused, _kind, _win_key = self._decide(code)
352
+ if not traced:
353
+ return M.DISABLE
354
+ tls.in_hook = True
355
+ try:
356
+ tid = tls.thread_serial
357
+ frame = sys._getframe(1)
358
+ entry = self._live_entry(tls, frame, code)
359
+ if entry is None and self._fork_child:
360
+ return None
361
+ fid = None
362
+ if entry is not None:
363
+ del tls.live[id(frame)]
364
+ fid = entry[0]
365
+ cid = self.writer.intern_code(code.co_filename, qual,
366
+ code.co_firstlineno)
367
+ task = self._task_serial(tls)
368
+ payload = {"value": rv.named_return(qual,
369
+ capture_value(retval))}
370
+ rv.stats["values"] += rv.count(payload)
371
+ eid = self.writer.add_event(time.monotonic_ns(), tid, "RETURN",
372
+ fid, cid, None, payload,
373
+ task_id=task)
374
+ if fid is not None:
375
+ self.writer.close_frame(fid, eid, "return")
376
+ self._fp_for(tid, task).update(fp_file, qual, "RETURN")
377
+ finally:
378
+ tls.in_hook = False
379
+ return None
380
+
381
+ def _on_unwind(self, code, offset, exc):
382
+ tls = self._tls
383
+ if tls.in_hook:
384
+ return None
385
+ traced, fp_file, qual, focused, _kind, _win_key = self._decide(code)
386
+ if not traced:
387
+ return None # exception events can't DISABLE
388
+ tls.in_hook = True
389
+ try:
390
+ frame = sys._getframe(1)
391
+ materialize_elided(self, tls, frame)
392
+ entry = self._live_entry(tls, frame, code)
393
+ if entry is not None:
394
+ del tls.live[id(frame)]
395
+ unwind_exc = rv.exc(capture_exc(exc, self.serial_of(exc)))
396
+ rv.stats["values"] += rv.count_capture(unwind_exc)
397
+ self.writer.close_frame(entry[0], None, "unwind", unwind_exc)
398
+ finally:
399
+ tls.in_hook = False
400
+ return None
401
+
402
+ # -- suspension --------------------------------------------------------
403
+ def _suspension(self, code, frame, kind, payload_factory, disable_ok=True):
404
+ """Shared body of the YIELD/RESUME callbacks. `frame` is the
405
+ triggering frame, read as `sys._getframe(1)` OF THE REGISTERED
406
+ CALLBACK and passed down -- the same convention `_on_raise` and
407
+ `_on_handled` follow, and for the same reason: read one frame deeper,
408
+ here, it would be tracer.py's own.
409
+
410
+ `payload_factory` is a callable, not a payload. Building the payload
411
+ runs the observed program (`capture_exc` calls its `__str__`,
412
+ `type_name` reads a possibly-computed `__name__`), so it must happen
413
+ INSIDE the `in_hook` region -- otherwise a traced helper called from
414
+ that `__str__` is recorded as real execution: a phantom CALL/RETURN
415
+ pair, a live entry, and a fingerprint update minted from hook time.
416
+
417
+ Never fingerprinted. A fingerprint must depend only on the (code
418
+ object, causal kind) sequence -- suspending is not a causal step, and
419
+ how many times a coroutine parks depends on the event loop's
420
+ scheduling, so hashing it would make two identical runs disagree.
421
+ """
422
+ tls = self._tls
423
+ if tls.in_hook:
424
+ return None
425
+ budget = self._budget
426
+ if budget is not None and budget.slot(id(code)) is not None:
427
+ key = id(frame)
428
+ if key not in tls.live and not self._parked_has(key):
429
+ return None
430
+ traced, _fp_file, _qual, _focused, _kind, _win = self._decide(code)
431
+ if not traced:
432
+ # PY_YIELD/PY_RESUME may be disabled per code object like any
433
+ # other local event, and untraced code suspends constantly (the
434
+ # event loop, every stdlib generator), so saying so is worth
435
+ # real time. PY_THROW may NOT: CPython refuses to disable it,
436
+ # exactly as it refuses RAISE and PY_UNWIND, and a DISABLE from
437
+ # that callback raises ValueError *into the traced program* and
438
+ # unregisters the callback -- measured, not assumed.
439
+ return M.DISABLE if disable_ok else None
440
+ tls.in_hook = True
441
+ try:
442
+ entry = self._live_entry(tls, frame, code)
443
+ if entry is None:
444
+ return None
445
+ suspending = kind == "YIELD"
446
+ entry[6] = suspending
447
+ payload = payload_factory()
448
+ rv.stats["values"] += rv.count(payload)
449
+ self.writer.add_event(time.monotonic_ns(), tls.thread_serial, kind,
450
+ entry[0], entry[2], frame.f_lineno, payload,
451
+ task_id=self._task_serial(tls))
452
+ if suspending:
453
+ self._arm_silent_owner(entry, frame)
454
+ self._park(tls, frame, entry)
455
+ finally:
456
+ tls.in_hook = False
457
+ return None
458
+
459
+ def _on_yield(self, code, offset, value):
460
+ # The TYPE name, never `repr(value)`: a repr is program code run from
461
+ # a hook, unbounded in size, and different on every run (addresses),
462
+ # which would make the column useless for grouping and diffing.
463
+ # `type_name` is the capture module's guarded funnel and runs inside
464
+ # the hook region -- a metaclass `__name__` is program code too.
465
+ return self._suspension(code, sys._getframe(1), "YIELD",
466
+ lambda: {"awaiting": type_name(value)})
467
+
468
+ def _on_resume(self, code, offset):
469
+ return self._suspension(code, sys._getframe(1), "RESUME", lambda: None)
470
+
471
+ def _on_throw(self, code, offset, exc):
472
+ """An exception is being thrown into a suspended frame.
473
+
474
+ Decide BEFORE minting. `identify` retains a strong reference in a
475
+ bounded table, so minting for untraced code would pour every
476
+ generator finalisation and every asyncio cancellation in the process
477
+ into it and evict serials a stored re-raise still wants. Unlike RAISE
478
+ (where the raising frame may be untraced while the frame that unwinds
479
+ is traced), PY_THROW fires on the very frame that will PY_UNWIND: if
480
+ it is untraced, no row is written at either end and the serial buys
481
+ nothing.
482
+
483
+ Which table, for the same reason at a finer grain. `_exc_event`
484
+ deliberately keeps control-flow types out of the real table, and this
485
+ door is the one they would otherwise arrive by: a `for ... break`, a
486
+ short-circuiting `any(genexpr)`, any dropped generator throws
487
+ GeneratorExit into a frame this recorder may well be tracing. They
488
+ still need a serial -- `abandoned` is the RESUME serial matching the
489
+ unwind's, nothing else -- so they get one from `cf_exc`, a small
490
+ table of their own (`_CONTROL_RETAIN_MAX`).
491
+
492
+ Knowingly given up: an exception thrown into an UNTRACED generator
493
+ that escapes into a traced caller's unwind has no serial there, so
494
+ that frame reads `raised` rather than `thrown` -- no D2 state on a
495
+ traced frame depends on it.
496
+
497
+ For a traced frame the serial is minted inside the hook region, so
498
+ the RESUME row, the RAISE the interpreter fires next, and the UNWIND
499
+ all carry ONE serial. That equality -- not the type -- is what lets
500
+ the reader say "cancelled" or "abandoned".
501
+ """
502
+ tls = self._tls
503
+ if tls.in_hook or not self._decide(code)[0]:
504
+ return None # PY_THROW may not DISABLE
505
+ refs = tls.cf_exc if _is_control_flow(exc) else tls.exc
506
+ return self._suspension(
507
+ code, sys._getframe(1), "RESUME",
508
+ lambda: {"thrown": rv.exc(capture_exc(exc,
509
+ refs.identify(exc)))},
510
+ disable_ok=False)
511
+
512
+ # The triggering frame is sys._getframe(1) *of the registered callback*,
513
+ # so it is read here and passed down rather than inside _exc_event, which
514
+ # sits one frame deeper and would otherwise report tracer.py's own lines.
515
+ def _on_raise(self, code, offset, exc):
516
+ return self._exc_event(code, exc, "RAISE", sys._getframe(1))
517
+
518
+ def _on_handled(self, code, offset, exc):
519
+ return self._exc_event(code, exc, "HANDLED", sys._getframe(1))
520
+
521
+ def _on_reraise(self, code, offset, exc):
522
+ """RERAISE is not a recorded kind; it only maintains in-flight state.
523
+
524
+ A bare ``raise``, and the implicit re-raise that ends a ``finally`` or
525
+ a ``__exit__``, put the same exception back in flight. Without this,
526
+ the EXCEPTION_HANDLED that CPython fires on *entry* to a finally block
527
+ -- which happens even when nothing is caught, because ``finally`` is
528
+ compiled as an implicit handler -- would disarm the de-dupe
529
+ mid-propagation and the next frame's RAISE would be recorded as a
530
+ second origin.
531
+
532
+ It deliberately does not touch ``origin_recorded``: re-arming resumes a
533
+ propagation, it never reopens the origin. Only a fresh RAISE of a
534
+ different object does that.
535
+ """
536
+ tls = self._tls
537
+ if not tls.in_hook and not _is_control_flow(exc):
538
+ tls.exc.last_exc = exc
539
+ return None
540
+
541
+ def _exc_event(self, code, exc, kind, frame):
542
+ tls = self._tls
543
+ if tls.in_hook:
544
+ return None
545
+ # Control-flow exceptions are neither recorded nor allowed to disturb
546
+ # in-flight state: a generator finishing during cleanup must not clear
547
+ # the real exception that is propagating. Matched by TYPE, not name, so
548
+ # a user class merely sharing a control-flow name is recorded rather
549
+ # than silently dropped; see `_is_control_flow`.
550
+ if _is_control_flow(exc):
551
+ return None
552
+ # In-flight bookkeeping runs whether or not this frame is traced --
553
+ # handlers and cleanup blocks are frequently foreign code. Being in
554
+ # flight is tracked separately from having recorded an origin: library
555
+ # code routinely raises, handles and re-raises internally before the
556
+ # exception ever surfaces in user code, so "already in flight" must not
557
+ # by itself suppress the row. The origin is the first *traced* frame the
558
+ # exception reaches. HANDLED ends the flight -- which is what makes a
559
+ # later raise of the same object a new origin -- but note it does not
560
+ # imply anything was caught; see _on_reraise.
561
+ # The serial rides along on decisions this machine already takes; it
562
+ # never changes one. It is looked up per event, from the object, so
563
+ # that an exception handled between this one's raise and its re-raise
564
+ # -- or raised and handled inside a `finally` while this one is still
565
+ # in flight -- cannot displace it. Nothing here is a "current serial":
566
+ # `refs.identify` answers for the object it is given and no other.
567
+ refs = tls.exc
568
+ if kind == "RAISE":
569
+ if refs.last_exc is not exc:
570
+ refs.last_exc = exc
571
+ tls.origin_recorded = False # a fresh raise reopens the origin
572
+ if tls.origin_recorded:
573
+ return None # propagating an origin we logged
574
+ serial = refs.identify(exc)
575
+ else:
576
+ # Remembered (or recognised) here as well as on RAISE, so a later
577
+ # `raise e` of this same object resumes its serial; see _ExcRefs
578
+ # for why the reference is strong and why it is bounded.
579
+ serial = refs.identify(exc)
580
+ refs.last_exc = None
581
+ traced, fp_file, qual, focused, _fkind, _win_key = self._decide(code)
582
+ if not traced:
583
+ return None
584
+ if kind == "RAISE":
585
+ tls.origin_recorded = True
586
+ tls.in_hook = True
587
+ try:
588
+ if kind == "RAISE":
589
+ materialize_elided(self, tls, frame)
590
+ tid = tls.thread_serial
591
+ entry = tls.live.get(id(frame))
592
+ fid = entry[0] if (entry is not None and entry[1] is code) else None
593
+ cid = self.writer.intern_code(code.co_filename, qual,
594
+ code.co_firstlineno)
595
+ task = self._task_serial(tls)
596
+ payload = {"exc": rv.exc(capture_exc(exc, serial))}
597
+ rv.stats["values"] += rv.count(payload)
598
+ self.writer.add_event(time.monotonic_ns(), tid, kind, fid, cid,
599
+ frame.f_lineno, payload, task_id=task)
600
+ self._fp_for(tid, task).update(fp_file, qual, kind)
601
+ finally:
602
+ tls.in_hook = False
603
+ return None
604
+
605
+ def _on_line(self, code, line):
606
+ """Record the locals that changed since this frame's previous line.
607
+
608
+ Fires *before* `line` executes, so the deltas belong to the line before
609
+ it; see the module docstring. Never fingerprints.
610
+ """
611
+ tls = self._tls
612
+ if tls.in_hook:
613
+ return None
614
+ traced, _fp_file, _qual, focused, _kind, _win_key = self._decide(code)
615
+ if not traced or not focused:
616
+ return M.DISABLE # nothing here will ever be worth recording
617
+ frame = sys._getframe(1)
618
+ entry = self._live_entry(tls, frame, code)
619
+ if entry is None:
620
+ return None # no open frame for this activation
621
+ if self.window and not entry[5]:
622
+ # Outside the window: no ancestor of THIS activation is the
623
+ # window target. Not DISABLE -- another activation of the same
624
+ # code may be inside it later.
625
+ return None
626
+ tls.in_hook = True # capture_value runs user __repr__ code
627
+ try:
628
+ prev = entry[3]
629
+ snap = locals_snapshot(frame)
630
+ if snap is None:
631
+ # The locals could not be read at all. Record the site with
632
+ # that said plainly rather than skip it: a site nobody
633
+ # checked must not look like a site where nothing changed,
634
+ # and `prev` is deliberately left in place, because this
635
+ # step establishes nothing about what went out of scope.
636
+ # Not counted: this payload holds no capture at all.
637
+ self.writer.add_event(time.monotonic_ns(),
638
+ tls.thread_serial, "LINE",
639
+ entry[0], entry[2], line,
640
+ {"deltas": {}, "unread": ["locals"]},
641
+ task_id=self._task_serial(tls))
642
+ return None
643
+ cur, deltas = {}, {}
644
+ for name, val in snap.items():
645
+ # The NAME rule runs BEFORE the comparison below, not
646
+ # after it: a redacted value is what this frame now holds,
647
+ # so two sightings of one unchanged secret must compare
648
+ # equal (their digests do) and be ONE delta, exactly as
649
+ # they were before the rule.
650
+ cap = rv.named(name, capture_value(val))
651
+ cur[name] = cap
652
+ # Captures, never live objects -- and NAMES that are exact
653
+ # `str`, never the program's own key objects. Both halves
654
+ # were false once: a capture EMBEDDED `str`/`int`/`float`
655
+ # subclass instances until `capture_value` normalised them,
656
+ # and the keys came straight out of `f_locals` until
657
+ # `locals_snapshot` did. Either one turns the dict lookup
658
+ # below, the set difference, or the `sorted` further down
659
+ # into a call into the observed program, from a hook, with
660
+ # no guard anywhere on the path.
661
+ if prev.get(name) != cap:
662
+ deltas[name] = cap
663
+ gone = prev.keys() - cur.keys()
664
+ entry[3] = cur
665
+ if deltas or gone:
666
+ payload = {"deltas": deltas}
667
+ if gone:
668
+ # Sibling key, never a sentinel inside deltas: that would
669
+ # widen capture_value's codomain and force a type check on
670
+ # every value. Readers that ignore it lose nothing else.
671
+ payload["unbound"] = sorted(gone)
672
+ # Only the deltas this row stores are counted -- the rule ran
673
+ # over every local in scope, and most of them changed nothing.
674
+ rv.stats["values"] += rv.count(payload)
675
+ self.writer.add_event(time.monotonic_ns(),
676
+ tls.thread_serial, "LINE",
677
+ entry[0], entry[2], line, payload,
678
+ task_id=self._task_serial(tls))
679
+ finally:
680
+ tls.in_hook = False
681
+ return None
682
+
683
+ def serial_of(self, exc) -> int | None:
684
+ """The serial this thread holds for `exc`, or None if it has none.
685
+
686
+ `boot` calls this from its own `except BaseException` clause. That
687
+ clause is untraced code, so the EXCEPTION_HANDLED it fires cleared
688
+ `last_exc` -- but the identity table still holds the object, which is
689
+ why the uncaught record can be tied to the RAISE row that produced it.
690
+
691
+ Both of this thread's tables are asked, because a frame closed by a
692
+ GeneratorExit thrown into it must read the serial its RESUME row
693
+ carries -- that equality is the whole of `abandoned`. The two are
694
+ disjoint by construction (`_exc_event` never records a control-flow
695
+ type, `_on_throw` sends only control-flow types to `cf_exc`), so
696
+ asking both cannot return the wrong one.
697
+
698
+ Never mints: a caller asking after the fact must be told "no serial"
699
+ rather than handed a fresh one that matches no recorded row.
700
+ """
701
+ tls = self._tls
702
+ serial = tls.exc.serial_of(exc)
703
+ return serial if serial is not None else tls.cf_exc.serial_of(exc)
704
+
705
+ # -- lifecycle ---------------------------------------------------------
706
+ def install(self) -> None:
707
+ E = M.events
708
+ # Before this tool id is taken: the probe borrows another id and
709
+ # frees it. A silent close is a property of this interpreter, not
710
+ # of the program about to run.
711
+ self._silent_gen_close = detect_silent_generator_close()
712
+ try:
713
+ M.use_tool_id(TOOL, "sensorium")
714
+ except ValueError as e:
715
+ owner = M.get_tool(TOOL)
716
+ raise RuntimeError(
717
+ f"cannot install monitoring: tool id {TOOL} is already in use"
718
+ f" by {owner!r} ({e}). Another profiler or debugger is "
719
+ "active; stop it and re-run.") from None
720
+ M.register_callback(TOOL, E.PY_START, self._on_start)
721
+ M.register_callback(TOOL, E.PY_RETURN, self._on_return)
722
+ M.register_callback(TOOL, E.PY_UNWIND, self._on_unwind)
723
+ M.register_callback(TOOL, E.RAISE, self._on_raise)
724
+ M.register_callback(TOOL, E.RERAISE, self._on_reraise)
725
+ M.register_callback(TOOL, E.EXCEPTION_HANDLED, self._on_handled)
726
+ M.register_callback(TOOL, E.LINE, self._on_line)
727
+ M.register_callback(TOOL, E.PY_YIELD, self._on_yield)
728
+ M.register_callback(TOOL, E.PY_RESUME, self._on_resume)
729
+ M.register_callback(TOOL, E.PY_THROW, self._on_throw)
730
+ events = (E.PY_START | E.PY_RETURN | E.PY_UNWIND
731
+ | E.RAISE | E.RERAISE | E.EXCEPTION_HANDLED
732
+ | E.PY_YIELD | E.PY_RESUME | E.PY_THROW)
733
+ if self.focus:
734
+ events |= E.LINE
735
+ M.set_events(TOOL, events)
736
+ M.restart_events()
737
+
738
+ def uninstall(self) -> None:
739
+ E = M.events
740
+ # Events off FIRST, and only then the tables. Clearing another
741
+ # thread's retention table while its callbacks are still live races
742
+ # the eviction loop's unguarded `next(iter(...))`: a worker between
743
+ # the `len()` check and the `next()` would raise StopIteration from
744
+ # inside a monitoring callback -- the recorder killing a traced
745
+ # thread, which is the one thing it must never do.
746
+ M.set_events(TOOL, 0)
747
+ # Drop every exception this recorder is holding, on every live thread
748
+ # -- not just the one calling uninstall, whose table is the only one
749
+ # `self._tls` can reach. Nothing can resume a serial once recording
750
+ # has stopped, and a worker parked in a `finally` must not keep its
751
+ # last exception (and that exception's frames and locals) alive for
752
+ # the rest of the process.
753
+ # Before the retention tables are cleared: a synthesized GeneratorExit
754
+ # takes its serial from this thread's control-flow table.
755
+ self._finish_silent_parked()
756
+ for refs in self._live_exc_refs():
757
+ refs.clear()
758
+ # Whatever is still parked was suspended when recording stopped --
759
+ # the one residual state (spec D2). Nothing is written for it: the
760
+ # reader derives "suspended at end of recording" from the frame's
761
+ # last YIELD row, which is evidence the trace already holds.
762
+ # Dropping the entries here is what stops this recorder holding a
763
+ # dead frame's locals for the rest of the process.
764
+ with self._parked_lock:
765
+ self._parked.clear()
766
+ self._tls.task_cache = None
767
+ for ev in (E.PY_START, E.PY_RETURN, E.PY_UNWIND, E.RAISE,
768
+ E.RERAISE, E.EXCEPTION_HANDLED, E.LINE,
769
+ E.PY_YIELD, E.PY_RESUME, E.PY_THROW):
770
+ M.register_callback(TOOL, ev, None)
771
+ M.free_tool_id(TOOL)
772
+ # Snapshot under the lock, as every other access to `_fps` does. Events
773
+ # are off, but a callback already dispatched on another thread can still
774
+ # be mid-flight and reach `_fp` to insert its first entry -- iterating
775
+ # the live dict here would raise `dictionary changed size during
776
+ # iteration` out of `run_target`'s `finally`, before `w.close()`,
777
+ # leaking the connection and leaving the trace `incomplete`. A thread
778
+ # that inserts after this snapshot was still alive when recording
779
+ # stopped, and is already reported as such (`_recording_gaps`).
780
+ with self._fp_lock:
781
+ fps = list(self._fps.items())
782
+ tfps = list(self._task_fps.items())
783
+ for tid, fp in fps:
784
+ self.writer.write_fingerprint(tid, fp.hexdigest(), fp.count)
785
+ # A task's row is written whatever state the task was left in: a
786
+ # stream that was recorded is a stream, and one still parked at
787
+ # uninstall is exactly the case a comparison most wants to see.
788
+ # ALL of them in one transaction: a commit is an fsync, this runs
789
+ # after the recorded program has finished, and one per task made
790
+ # exit cost scale with the task count (see
791
+ # `TraceWriter.write_task_fingerprints`).
792
+ self.writer.write_task_fingerprints(
793
+ [(task, fp.hexdigest(), fp.count) for task, fp in tfps])
794
+ if self._budget is not None:
795
+ report = self._budget.report()
796
+ if report:
797
+ for key, value in report.items():
798
+ self.writer.set_meta(key, value)