sensorium-trace 0.18.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (135) hide show
  1. sensorium/__init__.py +0 -0
  2. sensorium/__main__.py +4 -0
  3. sensorium/_c/rt/sensorium_c_rt.c +1446 -0
  4. sensorium/_typescript/package-lock.json +68 -0
  5. sensorium/_typescript/package.json +39 -0
  6. sensorium/_typescript/src/bindings.mjs +792 -0
  7. sensorium/_typescript/src/cap.mjs +227 -0
  8. sensorium/_typescript/src/capabilities.mjs +40 -0
  9. sensorium/_typescript/src/dbg.mjs +348 -0
  10. sensorium/_typescript/src/escape.mjs +351 -0
  11. sensorium/_typescript/src/focus.mjs +135 -0
  12. sensorium/_typescript/src/hook.mjs +127 -0
  13. sensorium/_typescript/src/index.mjs +1 -0
  14. sensorium/_typescript/src/naming.mjs +63 -0
  15. sensorium/_typescript/src/positions.mjs +37 -0
  16. sensorium/_typescript/src/probe.mjs +246 -0
  17. sensorium/_typescript/src/qualname.mjs +236 -0
  18. sensorium/_typescript/src/redact.mjs +711 -0
  19. sensorium/_typescript/src/register.mjs +15 -0
  20. sensorium/_typescript/src/resolve.mjs +260 -0
  21. sensorium/_typescript/src/rt.mjs +798 -0
  22. sensorium/_typescript/src/setup.mjs +11 -0
  23. sensorium/_typescript/src/tally.mjs +164 -0
  24. sensorium/_typescript/src/tasks.mjs +175 -0
  25. sensorium/_typescript/src/transform.mjs +800 -0
  26. sensorium/_typescript/src/vite.mjs +99 -0
  27. sensorium/c/__init__.py +8 -0
  28. sensorium/c/cc.py +612 -0
  29. sensorium/c/cli.py +60 -0
  30. sensorium/c/convert.py +669 -0
  31. sensorium/c/decode.py +300 -0
  32. sensorium/c/driver.py +365 -0
  33. sensorium/c/dwarf.py +706 -0
  34. sensorium/c/format.py +271 -0
  35. sensorium/c/pkg.py +187 -0
  36. sensorium/c/plan.py +277 -0
  37. sensorium/cli.py +140 -0
  38. sensorium/driver.py +47 -0
  39. sensorium/exit.py +33 -0
  40. sensorium/invocations.py +83 -0
  41. sensorium/limits.py +253 -0
  42. sensorium/mcp/__init__.py +0 -0
  43. sensorium/mcp/audit.py +162 -0
  44. sensorium/mcp/child.py +240 -0
  45. sensorium/mcp/cmd.py +101 -0
  46. sensorium/mcp/jsonrpc.py +172 -0
  47. sensorium/mcp/result.py +187 -0
  48. sensorium/mcp/schema.py +390 -0
  49. sensorium/mcp/server.py +493 -0
  50. sensorium/mcp/tools.py +299 -0
  51. sensorium/paths.py +80 -0
  52. sensorium/query/__init__.py +0 -0
  53. sensorium/query/caps.py +150 -0
  54. sensorium/query/dbg_dialects.py +64 -0
  55. sensorium/query/diff_cmd.py +748 -0
  56. sensorium/query/diff_notes.py +117 -0
  57. sensorium/query/exceptions_c.py +263 -0
  58. sensorium/query/exceptions_cmd.py +788 -0
  59. sensorium/query/exceptions_group.py +571 -0
  60. sensorium/query/exceptions_invocation.py +487 -0
  61. sensorium/query/exceptions_rust.py +683 -0
  62. sensorium/query/exceptions_typescript.py +688 -0
  63. sensorium/query/expr.py +606 -0
  64. sensorium/query/failures_cmd.py +306 -0
  65. sensorium/query/flow_cmd.py +778 -0
  66. sensorium/query/flow_report.py +128 -0
  67. sensorium/query/flow_values.py +195 -0
  68. sensorium/query/fmt.py +280 -0
  69. sensorium/query/frame_cmd.py +237 -0
  70. sensorium/query/grep_cmd.py +156 -0
  71. sensorium/query/help.py +18 -0
  72. sensorium/query/info_cmd.py +527 -0
  73. sensorium/query/info_rust.py +214 -0
  74. sensorium/query/info_typescript.py +261 -0
  75. sensorium/query/js_inspect.py +394 -0
  76. sensorium/query/moves.py +254 -0
  77. sensorium/query/redact_cmd.py +304 -0
  78. sensorium/query/refocus_cmd.py +661 -0
  79. sensorium/query/refocus_env.py +493 -0
  80. sensorium/query/refocus_facts.py +269 -0
  81. sensorium/query/refocus_licence.py +187 -0
  82. sensorium/query/refocus_report.py +327 -0
  83. sensorium/query/refocus_rust.py +537 -0
  84. sensorium/query/refocus_threads.py +197 -0
  85. sensorium/query/refocus_typescript.py +756 -0
  86. sensorium/query/refocus_world.py +666 -0
  87. sensorium/query/runs_cmd.py +213 -0
  88. sensorium/query/rust_debug.py +176 -0
  89. sensorium/query/sites.py +171 -0
  90. sensorium/query/tree_cmd.py +511 -0
  91. sensorium/query/tree_threads.py +168 -0
  92. sensorium/query/vocab.py +723 -0
  93. sensorium/query/watch_cmd.py +657 -0
  94. sensorium/record/__init__.py +0 -0
  95. sensorium/record/boot.py +797 -0
  96. sensorium/record/boot_io.py +141 -0
  97. sensorium/record/budget.py +135 -0
  98. sensorium/record/capture.py +433 -0
  99. sensorium/record/children.py +755 -0
  100. sensorium/record/children_merge.py +179 -0
  101. sensorium/record/elide.py +55 -0
  102. sensorium/record/fingerprint.py +26 -0
  103. sensorium/record/thread_starts.py +261 -0
  104. sensorium/record/tracer.py +798 -0
  105. sensorium/record/tracer_exc.py +199 -0
  106. sensorium/record/tracer_frames.py +618 -0
  107. sensorium/redact.py +304 -0
  108. sensorium/redact_content.py +156 -0
  109. sensorium/redact_key.py +317 -0
  110. sensorium/redact_store.py +502 -0
  111. sensorium/redact_values.py +433 -0
  112. sensorium/store/__init__.py +0 -0
  113. sensorium/store/db.py +284 -0
  114. sensorium/store/gc.py +360 -0
  115. sensorium/store/payloads.py +69 -0
  116. sensorium/store/reader.py +438 -0
  117. sensorium/store/writer.py +283 -0
  118. sensorium/ts/__init__.py +10 -0
  119. sensorium/ts/build.py +783 -0
  120. sensorium/ts/cli.py +130 -0
  121. sensorium/ts/driver.py +472 -0
  122. sensorium/ts/focus.py +146 -0
  123. sensorium/ts/harness.py +314 -0
  124. sensorium/ts/ingest.py +380 -0
  125. sensorium/ts/invocation.py +252 -0
  126. sensorium/ts/pkg.py +167 -0
  127. sensorium/ts/redaction.py +64 -0
  128. sensorium/ts/spool.py +168 -0
  129. sensorium/ts/truncation.py +81 -0
  130. sensorium/ts/wrapper.py +255 -0
  131. sensorium_trace-0.18.2.dist-info/METADATA +245 -0
  132. sensorium_trace-0.18.2.dist-info/RECORD +135 -0
  133. sensorium_trace-0.18.2.dist-info/WHEEL +4 -0
  134. sensorium_trace-0.18.2.dist-info/entry_points.txt +3 -0
  135. sensorium_trace-0.18.2.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,797 @@
1
+ """Resolve and execute a target program in-process under recording.
2
+
3
+ The target runs inside this interpreter, not a child: that is what makes its
4
+ frames visible to sys.monitoring at all. Everything here therefore has to put
5
+ the interpreter back the way it found it -- streams, argv, sys.path, the
6
+ monitoring tool id, the audit hook -- because `refocus` re-runs a program a
7
+ second time in the same process and must not inherit the first run's state.
8
+
9
+ Two orderings matter and are deliberate:
10
+
11
+ * The target is resolved *before* monitoring is installed, so import
12
+ machinery, entry-point loading and runpy's own setup never land in the
13
+ trace.
14
+ * The tracer is uninstalled *before* the writer closes, and every write goes
15
+ through `_LateWriteGuard` so a callback a thread was already inside cannot
16
+ write into a closed database.
17
+
18
+ A run that cannot record is a failed run: if monitoring will not install, or
19
+ the target cannot be resolved, nothing reports success. `incomplete` is
20
+ written true up front and cleared only after a clean finish, so a trace that
21
+ died mid-run says so.
22
+ """
23
+ import hashlib
24
+ import importlib.metadata
25
+ import json
26
+ import os
27
+ import runpy
28
+ import subprocess
29
+ import sys
30
+ import threading
31
+ import time
32
+ import traceback
33
+ from pathlib import Path
34
+
35
+ from sensorium import paths, redact
36
+ from sensorium import redact_values as rv
37
+ from sensorium.record import capture
38
+ # The stream interception, split out at this file's 800-line ceiling.
39
+ # Re-exported so `boot._Tee` and `boot._StdinProxy` keep resolving:
40
+ # these are one recorder's internals across two files, not separate
41
+ # modules with surfaces of their own.
42
+ from sensorium.record.boot_io import _StdinProxy, _Tee # noqa: F401
43
+ from sensorium.record.children import (arm_children, children_enabled,
44
+ disarm_children, merge_children)
45
+ from sensorium.record.thread_starts import (arm_thread_starts,
46
+ finish_thread_starts)
47
+ from sensorium.record.tracer import FocusSpec, Tracer
48
+ from sensorium.store.writer import TraceWriter
49
+
50
+
51
+ class TargetError(Exception):
52
+ pass
53
+
54
+
55
+ # What this recorder produces, declared into every trace (spec §5.2). Every
56
+ # query command reads the declaration of the trace it was given; a
57
+ # recorder that cannot produce something says so here, and the command
58
+ # refuses instead of rendering an empty result as a finding.
59
+ CAPABILITIES = {"line": True, "locals": True, "return_value": True,
60
+ "tasks": True, "threads": True, "children": True,
61
+ "stdin": True, "output": True, "object_identity": True,
62
+ "refocus": True}
63
+
64
+
65
+ def _recorder_id() -> str:
66
+ try:
67
+ return f"sensorium {importlib.metadata.version('sensorium-trace')}"
68
+ except importlib.metadata.PackageNotFoundError:
69
+ return "sensorium (uninstalled source tree)"
70
+
71
+
72
+ # -- target resolution -----------------------------------------------------
73
+ def resolve_target(argv: list[str]):
74
+ """Return a zero-argument callable that runs `argv`, or raise TargetError.
75
+
76
+ Called before monitoring is installed, so nothing it imports is traced.
77
+ """
78
+ if not argv:
79
+ raise TargetError("no target command given")
80
+ cmd = argv[0]
81
+ if cmd == "-m":
82
+ return _module_target(argv)
83
+ if cmd.endswith(".py"):
84
+ return _script_target(argv)
85
+ return _console_script_target(argv)
86
+
87
+
88
+ def _module_target(argv: list[str]):
89
+ if len(argv) < 2:
90
+ raise TargetError("-m requires a module name")
91
+ mod, rest = argv[1], argv[2:]
92
+
93
+ def run_module():
94
+ sys.path.insert(0, os.getcwd()) # what `python -m` puts first
95
+ sys.argv = [mod, *rest]
96
+ runpy.run_module(mod, run_name="__main__", alter_sys=True)
97
+ return run_module
98
+
99
+
100
+ def _script_target(argv: list[str]):
101
+ p = Path(argv[0]).resolve()
102
+ if not p.exists():
103
+ raise TargetError(f"cannot resolve target: no such file: {argv[0]}")
104
+
105
+ def run_file():
106
+ sys.path.insert(0, str(p.parent)) # what `python script.py` does
107
+ sys.argv = [str(p), *argv[1:]]
108
+ runpy.run_path(str(p), run_name="__main__")
109
+ return run_file
110
+
111
+
112
+ def _console_script_target(argv: list[str]):
113
+ cmd = argv[0]
114
+ for ep in importlib.metadata.entry_points(group="console_scripts",
115
+ name=cmd):
116
+ fn = ep.load()
117
+
118
+ def run_script():
119
+ sys.argv = list(argv)
120
+ # Exactly what pip's generated wrapper does: `sys.exit(main())`.
121
+ # A console script's `main` usually RETURNS its status (assay,
122
+ # pytest's own wrapper, every `return 2` refusal) rather than
123
+ # raising SystemExit; dropping that return recorded exit 0 for
124
+ # a program that exited 2, and `runs`/`info`/refocus's "ended
125
+ # differently" check all read the recorded status. None -> 0,
126
+ # an int -> that int, anything else -> CPython's own exit-1
127
+ # answer, all via `_exit_status_of`.
128
+ raise SystemExit(fn())
129
+ return run_script
130
+ raise TargetError(
131
+ f"cannot resolve target {cmd!r}: not a .py file, -m module, "
132
+ "or installed console script")
133
+
134
+
135
+ # -- writes from threads that outlive the target ---------------------------
136
+ class _LateWriteGuard:
137
+ """Serialise every trace write against close.
138
+
139
+ Uninstalling the tracer stops new callbacks, but a thread already inside
140
+ one runs to completion and writes afterwards -- a daemon thread parked in
141
+ a captured value's `__repr__` when the target's main function returns is
142
+ the reachable case. Sealing under the same lock that guards each
143
+ delegated write makes such a write either land before the close or be
144
+ dropped and counted; it can never reach a closed sqlite connection.
145
+ (Measured: with the writer's default batching a late write is merely
146
+ buffered and lost, but any flush it triggers raises
147
+ `sqlite3.ProgrammingError` from inside the monitoring callback and kills
148
+ that thread.)
149
+
150
+ Dropping is the only available answer -- the database has to close, and
151
+ the thread may run forever -- so nothing is dropped quietly: `seal()`
152
+ stops accepting writes while the connection is still open, which is what
153
+ lets the drop count reach `late_writes` in the run metadata, and
154
+ `run_target` reports both that count and the threads still alive.
155
+ """
156
+
157
+ def __init__(self, writer: TraceWriter) -> None:
158
+ self._w = writer
159
+ self._lock = threading.Lock()
160
+ self._sealed = False
161
+ self._closed = False
162
+ self.late_writes = 0
163
+
164
+ @property
165
+ def last_event_id(self) -> int:
166
+ return self._w.last_event_id
167
+
168
+ @property
169
+ def bytes_estimate(self) -> int:
170
+ return self._w.bytes_estimate
171
+
172
+ def _call(self, name, *args, **kwargs):
173
+ with self._lock:
174
+ if self._sealed:
175
+ self.late_writes += 1
176
+ return 0
177
+ return getattr(self._w, name)(*args, **kwargs)
178
+
179
+ def intern_code(self, *a, **k):
180
+ return self._call("intern_code", *a, **k)
181
+
182
+ def add_event(self, *a, **k):
183
+ return self._call("add_event", *a, **k)
184
+
185
+ def open_frame(self, *a, **k):
186
+ return self._call("open_frame", *a, **k)
187
+
188
+ def close_frame(self, *a, **k):
189
+ return self._call("close_frame", *a, **k)
190
+
191
+ def add_task(self, *a, **k):
192
+ return self._call("add_task", *a, **k)
193
+
194
+ def add_output(self, *a, **k):
195
+ return self._call("add_output", *a, **k)
196
+
197
+ def set_meta(self, *a, **k):
198
+ return self._call("set_meta", *a, **k)
199
+
200
+ def write_fingerprint(self, *a, **k):
201
+ return self._call("write_fingerprint", *a, **k)
202
+
203
+ def write_task_fingerprint(self, *a, **k):
204
+ return self._call("write_task_fingerprint", *a, **k)
205
+
206
+ def write_task_fingerprints(self, *a, **k):
207
+ return self._call("write_task_fingerprints", *a, **k)
208
+
209
+ def seal(self) -> None:
210
+ """Stop accepting writes; the connection stays open to finalize."""
211
+ with self._lock:
212
+ self._sealed = True
213
+
214
+ def set_meta_final(self, key, value) -> None:
215
+ """Write metadata after the seal: the finalizer is not a late write."""
216
+ with self._lock:
217
+ if not self._closed:
218
+ self._w.set_meta(key, value)
219
+
220
+ def interned_files(self) -> list[str]:
221
+ """Deliberately not routed through `_call`: this is a read, and a
222
+ sealed writer must still be able to answer it for the finalizer."""
223
+ return self._w.interned_files()
224
+
225
+ def close(self) -> None:
226
+ with self._lock:
227
+ if self._closed:
228
+ return
229
+ self._sealed = self._closed = True
230
+ self._w.close()
231
+
232
+
233
+ # -- child processes -------------------------------------------------------
234
+ _audit_lock = threading.Lock()
235
+ _audit_sink: list | None = None # the `children` list of the run in progress
236
+ _audit_threads: list | None = None # one entry per thread the run created
237
+ _audit_errors: list | None = None # one entry per hook malfunction
238
+ _audit_spawns: list | None = None # one entry per low-level spawn syscall
239
+ _audit_installs = 0 # audit hooks added, ever, by this process
240
+
241
+ # Thread creation, which the recorder cannot otherwise see. A thread whose
242
+ # body is entirely stdlib runs no traced code, so it gets no fingerprint row,
243
+ # and if it is joined before the run ends it is absent from `live_threads`
244
+ # too -- invisible on both counts while doing file I/O of its own. Every
245
+ # Python-level thread goes through one of these, measured:
246
+ #
247
+ # threading.Thread(...).start() -> _thread.start_joinable_thread
248
+ # _thread.start_new_thread(...) -> _thread.start_new_thread
249
+ #
250
+ # so counting them is sound rather than a heuristic. A thread started by a C
251
+ # extension without going through `_thread` still cannot be seen; that gap is
252
+ # named in refocus's blind spots rather than papered over.
253
+ _THREAD_EVENTS = ("_thread.start_new_thread", "_thread.start_joinable_thread")
254
+
255
+ # The syscall layer under every spawn, counted SEPARATELY from `children`.
256
+ # Measured:
257
+ #
258
+ # subprocess.run(...) -> subprocess.Popen, os.posix_spawn
259
+ # subprocess.run(preexec_fn=...) -> subprocess.Popen, ...fork_exec
260
+ # multiprocessing 'spawn' -> _posixsubprocess.fork_exec (no Popen)
261
+ # multiprocessing 'forkserver' -> _posixsubprocess.fork_exec (no Popen)
262
+ # multiprocessing 'fork' -> os.fork
263
+ #
264
+ # so these cannot join `_CHILD_EVENTS`: Popen nests one of them and would be
265
+ # counted twice. They go in their own list instead. Each list may over-count
266
+ # within itself, but "was ANY child witnessed" -- the only question the
267
+ # licence gate asks -- is answered correctly by either being non-empty, and
268
+ # `multiprocessing` with spawn/forkserver is visible here and nowhere else --
269
+ # but ONLY from 3.14 on. See `_SPAWN_WITNESSED`.
270
+ _SPAWN_SYSCALLS = ("os.posix_spawn", "_posixsubprocess.fork_exec")
271
+
272
+ # The `_posixsubprocess.fork_exec` audit event -- the ONLY parent-side signal a
273
+ # `multiprocessing` spawn/forkserver child raises -- was added in CPython 3.14.
274
+ # Measured on this project: a spawn is counted in `_SPAWN_SYSCALLS` on 3.14,
275
+ # and NOTHING on 3.12/3.13 (the syscall runs, but silently). So on an older
276
+ # interpreter `spawn_syscalls == 0` is not evidence a spawn did not happen, and
277
+ # a consumer must not read it as one. The recorder records the capability so
278
+ # `refocus` can withhold the "no child witnessed" claim it cannot support here.
279
+ _SPAWN_WITNESSED = sys.version_info >= (3, 14)
280
+
281
+
282
+ # Audit events that start another process, mapped to the positional argument
283
+ # carrying the command (None where the event carries no command at all).
284
+ #
285
+ # `subprocess.Popen` alone was not enough: `os.system` spawns a shell without
286
+ # ever touching `subprocess`, so a run that shelled out recorded
287
+ # `children == []` and was reported as having witnessed everything it did.
288
+ #
289
+ # The set is chosen from measurement, not from the audit-event list, because
290
+ # the events nest. Measured on this platform:
291
+ #
292
+ # subprocess.run(...) -> subprocess.Popen, os.posix_spawn
293
+ # subprocess.run(shell=1) -> subprocess.Popen, os.posix_spawn
294
+ # os.system(...) -> os.system
295
+ # os.spawnv(...) -> os.fork
296
+ # os.posix_spawn(...) -> os.posix_spawn
297
+ #
298
+ # so `os.posix_spawn` is absent from THIS table: it is how CPython implements
299
+ # `subprocess.Popen` here, and listing it beside Popen would count every
300
+ # subprocess twice. It is not unwatched -- it is counted in `_SPAWN_SYSCALLS`
301
+ # below, a separate list that is never summed with this one. `os.exec*` is
302
+ # absent for a different reason: it replaces this process rather than
303
+ # starting a child, so there is no run left to attach the note to.
304
+ _CHILD_EVENTS = {
305
+ "subprocess.Popen": 1, # (executable, args, cwd, env)
306
+ "os.system": 0, # (command,)
307
+ "os.fork": None, # no args; also how os.spawn* reaches the OS
308
+ }
309
+
310
+
311
+ def _audit(event, args) -> None:
312
+ """Note child processes and thread creation. One hook for the whole
313
+ process, armed per run.
314
+
315
+ `sys.addaudithook` cannot be removed, so a hook per run would accumulate:
316
+ after `refocus` re-runs a program, the first run's hook would still be
317
+ appending into a list nobody reads for every subprocess anything spawns
318
+ for the rest of the process's life. One module-level hook, armed and
319
+ disarmed around each run, has neither problem.
320
+
321
+ It must never raise: an audit hook that raises breaks the operation being
322
+ audited, which would mean the recorder killing the program it is meant to
323
+ observe. But swallowing silently is how a hook bug becomes an empty
324
+ `children` list that reads as "no subprocess ran" -- a false all-clear for
325
+ the licence gate downstream. So failures are swallowed AND counted, and
326
+ `refocus` withholds on a non-zero count.
327
+ """
328
+ try:
329
+ if event in _THREAD_EVENTS:
330
+ threads = _audit_threads
331
+ if threads is not None:
332
+ threads.append(1) # list.append is atomic; no lock
333
+ return
334
+ if event in _SPAWN_SYSCALLS:
335
+ spawns = _audit_spawns
336
+ if spawns is not None:
337
+ spawns.append(1)
338
+ return
339
+ if event not in _CHILD_EVENTS:
340
+ return
341
+ sink = _audit_sink
342
+ if sink is None:
343
+ return
344
+ index = _CHILD_EVENTS[event]
345
+ if index is None:
346
+ sink.append([f"<{event}>"]) # a child with no command to name
347
+ return
348
+ # Deliberately unguarded: an index past the end of `args` means the
349
+ # table above is wrong about this event, and returning quietly would
350
+ # leave `children == []` reading as "no subprocess ran". Let it raise
351
+ # into the counter below, where a wrong table becomes visible.
352
+ cmd = args[index]
353
+ if not cmd:
354
+ return
355
+ if isinstance(cmd, (str, bytes, os.PathLike)):
356
+ cmd = [cmd] # a bare command line, as Windows reports it
357
+ # Rule v1's CONTENT half over each element, at the write (R25). A
358
+ # child's command line is text this recorder STORES -- `meta.children`,
359
+ # which `info` prints back verbatim -- so
360
+ # `subprocess.run(["curl", "-H", f"Authorization: Bearer {tok}"])`
361
+ # would otherwise put a live token in run metadata. Only after the
362
+ # slice: `values` counts what the trace HOLDS (B3), and the ninth
363
+ # argument is not held.
364
+ sink.append([_content_ruled(a)
365
+ for a in [_as_text(a) for a in cmd][:8]])
366
+ # BaseException, not Exception. The arguments in `args` are the program's
367
+ # own objects -- `isinstance` consults `__class__`, `str()` runs
368
+ # `__str__`, and `for a in cmd` runs `__iter__` -- so a dunder raising
369
+ # anything at all would escape a narrower clause and break the very
370
+ # operation being audited. The docstring above already says this hook must
371
+ # never raise; this is what makes that true rather than nearly true.
372
+ except BaseException:
373
+ errors = _audit_errors
374
+ if errors is not None:
375
+ errors.append(1) # swallowed, but never silently
376
+
377
+
378
+ def _as_text(arg) -> str:
379
+ """Every recorded argument must be a str, or json.dumps kills the run.
380
+
381
+ `Popen` takes bytes anywhere a str is allowed -- `Popen(b"...",
382
+ shell=True)` reaches the hook as `['/bin/sh', '-c', b'...']` -- and
383
+ `str(b"x")` would record the literal `"b'x'"`, so decode rather than
384
+ stringify. `db.set_meta` json-encodes `children` inside the finalizer,
385
+ which runs before the writer closes: an unserializable entry there aborts
386
+ finalization, leaks the connection, and masks the program's exit status.
387
+ """
388
+ if isinstance(arg, bytes):
389
+ return arg.decode("utf-8", "replace")
390
+ # `str()` honours a `__str__` override and can hand back a `str` SUBCLASS
391
+ # (measured), which would put a live object into `children` and keep it
392
+ # alive in run metadata until the finalizer. Same normalisation as every
393
+ # other payload; found by the item-7 sweep.
394
+ return capture.plain_str(str(arg))
395
+
396
+
397
+ def _content_ruled(text: str) -> str:
398
+ """One element of a spawned command line, under rule v1's CONTENT half.
399
+
400
+ There is no NAME to read on a command line -- a child's argv has
401
+ positions, not bindings -- so the span operation is the whole of the rule
402
+ at this site (ruling R25): `--token sk-live-…` becomes
403
+ `--token <redacted>` and the shape of the invocation survives, which is
404
+ what `meta.children` is FOR.
405
+
406
+ Counted per element that CHANGED, by the hand that writes it, exactly as
407
+ the tee counts a chunk it stores (B3). `rv.text` honours the `off` knob
408
+ itself and never raises, which is what lets it be called from inside an
409
+ audit hook that must not.
410
+ """
411
+ after, hit = rv.text(text)
412
+ if hit:
413
+ rv.stats["values"] += 1
414
+ return after
415
+
416
+
417
+ def _arm_audit(sink: list, threads: list, errors: list,
418
+ spawns: list) -> None:
419
+ global _audit_sink, _audit_threads, _audit_errors, _audit_spawns
420
+ global _audit_installs
421
+ with _audit_lock:
422
+ if _audit_installs == 0:
423
+ sys.addaudithook(_audit)
424
+ _audit_installs += 1
425
+ _audit_sink, _audit_threads = sink, threads
426
+ _audit_errors, _audit_spawns = errors, spawns
427
+
428
+
429
+ def _disarm_audit() -> None:
430
+ global _audit_sink, _audit_threads, _audit_errors, _audit_spawns
431
+ with _audit_lock:
432
+ _audit_sink = _audit_threads = None
433
+ _audit_errors = _audit_spawns = None
434
+
435
+
436
+ # -- run metadata ----------------------------------------------------------
437
+ def git_info(cwd: Path) -> dict:
438
+ """Repository context for the run. NOT a change detector.
439
+
440
+ `git_dirty_hash` hashes the OUTPUT of `git status --porcelain`, which is
441
+ a list of paths and their status letters -- not file contents. A file
442
+ that is already dirty when a run is recorded can then be edited
443
+ arbitrarily, including the program being executed, and this hash does not
444
+ move; nor does it see gitignored or out-of-repo files. It answers "which
445
+ paths were dirty", and nothing that needs "did the code change" may be
446
+ built on it. `source_hashes` below is the check with that meaning.
447
+ """
448
+ def _git(*args):
449
+ r = subprocess.run(["git", *args], cwd=cwd,
450
+ capture_output=True, text=True)
451
+ return r.stdout.strip() if r.returncode == 0 else None
452
+ sha = _git("rev-parse", "HEAD")
453
+ if sha is None:
454
+ return {"git_sha": None, "git_dirty_hash": None}
455
+ status = _git("status", "--porcelain") or ""
456
+ return {"git_sha": sha,
457
+ "git_dirty_hash": hashlib.sha256(status.encode()).hexdigest()[:16]}
458
+
459
+
460
+ def hash_file(path: str) -> str | None:
461
+ """sha256 of a file's bytes, or None if it cannot be read now."""
462
+ try:
463
+ with open(path, "rb") as fh:
464
+ return hashlib.sha256(fh.read()).hexdigest()[:16]
465
+ except OSError:
466
+ return None
467
+
468
+
469
+ def _source_hashes(files) -> dict:
470
+ """Content digests for every file this run executed traced code from.
471
+
472
+ This is what makes "the source is unchanged" a claim the tool has
473
+ actually verified. Its limit is precise and worth knowing: it covers the
474
+ files the ORIGINAL run interned code from, plus the entry target. Code
475
+ that was never traced -- stdlib, site-packages, anything `--include` /
476
+ `--exclude` filtered out -- is not in the map and its changes are
477
+ invisible here.
478
+ """
479
+ return {f: hash_file(f) for f in sorted(files)}
480
+
481
+
482
+ def _write_run_meta(w, run_id, argv, focus, include, exclude, window,
483
+ refocus_of, *, key, knobs) -> dict:
484
+ # Rule v1 applies HERE, at the writer, before anything reaches disk:
485
+ # `env` is what the trace holds and there is no moment at which the
486
+ # plaintext was in the file. `env_hash` is taken over that stored
487
+ # environment and not over `os.environ`, so two runs of one unchanged
488
+ # shell hash alike whatever their secrets were -- a hash of the
489
+ # plaintext would report a world change whose evidence the trace no
490
+ # longer holds. `table` is the name -> digest map; `redact.meta` is the
491
+ # only thing that decides the `redaction` object's shape.
492
+ env, table = redact.env(os.environ, key, knobs)
493
+ redaction = redact.meta(key, knobs, table)
494
+ w.set_meta("run_id", run_id)
495
+ w.set_meta("argv", list(argv))
496
+ w.set_meta("cwd", str(Path.cwd()))
497
+ w.set_meta("env", env)
498
+ w.set_meta("env_hash", hashlib.sha256(
499
+ json.dumps(env, sort_keys=True).encode()).hexdigest()[:16])
500
+ w.set_meta("redaction", redaction)
501
+ w.set_meta("python", sys.version.split()[0])
502
+ w.set_meta("recorder", _recorder_id())
503
+ w.set_meta("lang", "python")
504
+ w.set_meta("capabilities", dict(CAPABILITIES))
505
+ for k, v in git_info(Path.cwd()).items():
506
+ w.set_meta(k, v)
507
+ # main_thread_ident is written after the Tracer exists (its serial for the
508
+ # calling thread IS the value); see run_target.
509
+ w.set_meta("focus", list(focus))
510
+ w.set_meta("include", list(include))
511
+ w.set_meta("exclude", list(exclude))
512
+ w.set_meta("window", window)
513
+ w.set_meta("caps", capture.CAPS)
514
+ # Which definition this recorder's per-thread fingerprints follow (spec
515
+ # D6): events that ran in an asyncio task go to that task's own
516
+ # fingerprint, so the thread's covers only the rest. A trace without
517
+ # this key was recorded before the distinction existed.
518
+ w.set_meta("fingerprint_basis", "per-task")
519
+ w.set_meta("start_ts", time.time())
520
+ w.set_meta("incomplete", True) # cleared only after a clean finish
521
+ if refocus_of:
522
+ w.set_meta("refocus_of", refocus_of)
523
+ # Returned, not re-read: `_finalize_meta` rewrites this same object with
524
+ # the run's `values` count, and a second `redact.meta(...)` there could
525
+ # drift from the one the trace already holds.
526
+ return redaction
527
+
528
+
529
+ def _finalize_meta(w, *, exit_status, uncaught, stdin_consumed, children,
530
+ truncated_count, live_threads, entry, threads_started,
531
+ audit_errors, spawn_syscalls, task_errors,
532
+ redaction, values, thread_starts, child_stats) -> None:
533
+ """Close out the run. Runs after `w.seal()`, hence `set_meta_final`."""
534
+ # How many values rule v1 took, counted by the hand that wrote them (B3)
535
+ # -- captures taken by name or by content, output chunks and exception
536
+ # messages alike. Written only under `mode: on`: a recording made with
537
+ # the rule off counted nothing, and a `values: 0` there would read as
538
+ # "the rule ran and found none" (B26).
539
+ if redaction.get("mode") == "on":
540
+ w.set_meta_final("redaction", {**redaction, "values": values})
541
+ w.set_meta_final("uncaught", uncaught)
542
+ w.set_meta_final("stdin_consumed", stdin_consumed)
543
+ w.set_meta_final("children", children)
544
+ w.set_meta_final("truncated_count", truncated_count)
545
+ # Threads still running when recording stopped. Already computed for the
546
+ # stderr gap warning; stored because a thread that ran only stdlib code
547
+ # produces NO fingerprint row, so counting fingerprints alone reports a
548
+ # single-threaded run while a second thread is doing file I/O.
549
+ w.set_meta_final("live_threads", live_threads)
550
+ # Threads CREATED, from the audit hook. `live_threads` answers "still
551
+ # running at seal time", which a worker that finished has already left --
552
+ # and if it ran only stdlib it has no fingerprint either, so it would be
553
+ # invisible on both counts.
554
+ w.set_meta_final("threads_started", threads_started)
555
+ # Which frame called Thread.start / Executor.submit, keyed by the tracer
556
+ # serial -- not the OS ident, which events do not use. Always written,
557
+ # even empty, so `tree` can tell "this recorder saw no starts" from "this
558
+ # trace predates the key". An absent key must not grow a heading.
559
+ w.set_meta_final("thread_starts", thread_starts)
560
+ # Present even when zero, so a reader can tell this recorder followed
561
+ # Python children from a trace that predates that merge.
562
+ w.set_meta_final("python_children", child_stats["python_children"])
563
+ w.set_meta_final("child_record_errors", child_stats["child_record_errors"])
564
+ w.set_meta_final("non_python_children", child_stats["non_python_children"])
565
+ # Times the audit hook malfunctioned. Non-zero means `children` above is
566
+ # not to be trusted, and refocus withholds rather than reading a short
567
+ # list as "nothing was spawned".
568
+ w.set_meta_final("audit_errors", audit_errors)
569
+ # Low-level spawns, counted apart from `children` because Popen nests one
570
+ # and would otherwise be counted twice. A Python child is also recorded
571
+ # into this run (children.py); this count stays the licence signal.
572
+ w.set_meta_final("spawn_syscalls", spawn_syscalls)
573
+ # Times the task IDENTITY lookup raised: `current_task()` itself, or a
574
+ # Task subclass's `__hash__`/`__eq__` during the serial lookup. Those
575
+ # events carry `task_id NULL`, which there means "could not tell", not
576
+ # "no task" -- which is why the count has to be on the run. An unreadable
577
+ # task NAME is NOT counted here: the serial was still minted and the
578
+ # events are still attributed, and the fact is already on the record as a
579
+ # NULL `name` in `tasks`.
580
+ w.set_meta_final("task_errors", task_errors)
581
+ w.set_meta_final("spawn_witnessing", _SPAWN_WITNESSED)
582
+ files = set(w.interned_files())
583
+ if entry:
584
+ files.add(entry) # the target itself, even if it traced nothing
585
+ w.set_meta_final("source_hashes", _source_hashes(files))
586
+ w.set_meta_final("exit_status", exit_status)
587
+ w.set_meta_final("end_ts", time.time())
588
+ w.set_meta_final("late_writes", w.late_writes) # read as late as possible
589
+ w.set_meta_final("incomplete", False)
590
+
591
+
592
+ # -- running ---------------------------------------------------------------
593
+ def _exit_status_of(exc: SystemExit) -> int:
594
+ """The status CPython would exit with, computed without letting the code
595
+ object's own dunders crash the recorder.
596
+
597
+ `sys.exit(obj)` arrives as `exc.code`. `isinstance(obj, int)` consults
598
+ `obj.__class__` for a non-int, which a property can make raise -- the
599
+ recorder killing the program from inside its own SystemExit handler. The
600
+ fast path is dunder-free for genuine ints (and bool/IntEnum, real int
601
+ subclasses); the guards cover a hostile `__class__`, a hostile `__int__` on
602
+ an int subclass, and a hostile `__str__` on the printed fallback. Any
603
+ failure takes the interpreter's own non-int answer, exit 1, not a traceback.
604
+ """
605
+ code = exc.code
606
+ if code is None:
607
+ return 0
608
+ try:
609
+ if isinstance(code, int):
610
+ return int(code)
611
+ except BaseException:
612
+ return 1
613
+ try:
614
+ print(code, file=sys.stderr) # what the interpreter itself does
615
+ except BaseException:
616
+ pass
617
+ return 1
618
+
619
+
620
+ def _live_thread_names() -> list[str]:
621
+ # `plain_str`, because a thread name is a str the PROGRAM chose: a str
622
+ # subclass whose `__str__` returns self survives threading's own
623
+ # `str(name)`, and its `__lt__`/`__eq__` then run when these names are
624
+ # sorted or stored. Normalise to an exact str at the boundary so no
625
+ # downstream consumer runs a program dunder.
626
+ return [capture.plain_str(t.name) for t in threading.enumerate()
627
+ if t is not threading.current_thread() and t.is_alive()]
628
+
629
+
630
+ def _recording_gaps(live: list[str], dropped: int) -> str | None:
631
+ """Say what the trace does not contain. Silence would be the failure."""
632
+ parts = []
633
+ if live:
634
+ names = ", ".join(sorted(live)[:4])
635
+ parts.append(f"{len(live)} thread(s) still alive when recording "
636
+ f"stopped ({names}); anything they run after this point "
637
+ "is not in the trace")
638
+ if dropped:
639
+ parts.append(f"{dropped} trace write(s) arrived after the recorder "
640
+ "stopped accepting them and were dropped")
641
+ if not parts:
642
+ return None
643
+ return "sensorium: " + "; ".join(parts) + "."
644
+
645
+
646
+ def run_target(argv, *, focus=(), include=(), exclude=(), window=None,
647
+ run_id=None, refocus_of=None, limits=None,
648
+ record_children=None):
649
+ """Record one execution of `argv`. Returns (run_id, exit_status)."""
650
+ run_id = run_id or paths.new_run_id()
651
+ if not paths.is_valid_run_id(run_id):
652
+ raise TargetError(
653
+ f"invalid run id {run_id!r}: a run id names one file in the trace "
654
+ "store, so it may not contain a path separator or '..'")
655
+ trace_path = paths.traces_dir() / f"{run_id}.db"
656
+ if trace_path.exists():
657
+ # Only reachable via an explicit --run-id (minted ids are unique).
658
+ # Without this, create_trace's executescript hits "table meta already
659
+ # exists" and escapes cli.main as a raw sqlite traceback.
660
+ raise TargetError(
661
+ f"run id {run_id!r} already has a trace at {trace_path}")
662
+ # The key and the knobs are read from the store and the environment as
663
+ # they are NOW, before the target is even resolved: a program that
664
+ # chdirs, or that rewrites `SENSORIUM_DIR` or one of the knobs in its
665
+ # own `os.environ` during import, cannot move the key the digests are
666
+ # taken under or change the rule the recording was made under.
667
+ key = redact.Key.load_or_create(paths.trace_root())
668
+ knobs = redact.Knobs.from_environ(os.environ)
669
+ # The value half of the rule reads the same key and the same knobs, from
670
+ # the same moment: one recording is made under one rule.
671
+ rv.install(key, knobs)
672
+ target = resolve_target(list(argv)) # resolve before hooks: never traced
673
+ # Resolved here, before the program can chdir underneath us.
674
+ entry = (str(Path(argv[0]).resolve())
675
+ if argv and str(argv[0]).endswith(".py") else None)
676
+ w = _LateWriteGuard(TraceWriter(trace_path))
677
+ redaction = _write_run_meta(w, run_id, argv, focus, include, exclude,
678
+ window, refocus_of, key=key, knobs=knobs)
679
+ truncated_before = capture.capture_stats["truncated"]
680
+ values_before = rv.stats["values"]
681
+ if limits is None:
682
+ from sensorium.limits import resolve
683
+ limits = resolve()
684
+ tracer = Tracer(w, root=Path.cwd(), focus=FocusSpec(list(focus)),
685
+ include=include, exclude=exclude, window=window,
686
+ limits=limits)
687
+ # The thread constructing the Tracer -- this one, which runs `target()`
688
+ # below synchronously -- is its first thread serial, and that serial is the
689
+ # identity all of this thread's events carry. It names the thread that
690
+ # actually ran the target, not the interpreter's global main thread: a
691
+ # caller invoking run_target from a worker gets that worker's identity,
692
+ # which is the point. `Trace.main_thread_id()` prefers this key.
693
+ w.set_meta("main_thread_ident", tracer.main_thread_serial)
694
+ if record_children is None:
695
+ record_children = children_enabled()
696
+ w.set_meta("record_children", bool(record_children))
697
+ try:
698
+ tracer.install()
699
+ except BaseException:
700
+ w.set_meta("end_ts", time.time())
701
+ w.close() # a trace still marked incomplete, and a loud raise
702
+ raise
703
+
704
+ children: list[list[str]] = []
705
+ started: list[int] = []
706
+ audit_errors: list[int] = []
707
+ spawns: list[int] = []
708
+ _arm_audit(children, started, audit_errors, spawns)
709
+ # Armed even when children are not recorded: a fork would otherwise
710
+ # keep this tracer and write the child into this trace file.
711
+ child_dir = None
712
+ child_arm_failed = False
713
+ try:
714
+ child_dir = arm_children(
715
+ tracer, record=bool(record_children), focus=focus,
716
+ include=include, exclude=exclude, window=window,
717
+ limits=limits)
718
+ except Exception:
719
+ child_arm_failed = True
720
+ saved = (sys.stdin, sys.stdout, sys.stderr, sys.argv, list(sys.path))
721
+ stdin_proxy = _StdinProxy(sys.stdin)
722
+ sys.stdin = stdin_proxy
723
+ sys.stdout = _Tee(saved[1], "stdout", w)
724
+ sys.stderr = _Tee(saved[2], "stderr", w)
725
+ exit_status, uncaught = 0, None
726
+ try:
727
+ # Before the target, so a Thread.start in the program is wrapped,
728
+ # and restored in the finally: refocus runs another target in this
729
+ # same process and must not wrap `start` a second time.
730
+ arm_thread_starts(tracer)
731
+ target()
732
+ except SystemExit as e:
733
+ exit_status = _exit_status_of(e)
734
+ except BaseException as e:
735
+ exit_status = 1
736
+ # Ask the tracer for this object's serial before uninstalling it, so
737
+ # the uncaught record names the exact RAISE row that produced it
738
+ # rather than being matched back by a recyclable address.
739
+ uncaught = rv.exc(capture.capture_exc(e, tracer.serial_of(e)))
740
+ rv.stats["values"] += rv.count_capture(uncaught)
741
+ traceback.print_exception(e) # tee'd: the trace holds what was shown
742
+ finally:
743
+ # `finish` puts `Thread.start` back even when uninstall itself
744
+ # fails: a wrapped start left installed would wrap again on the
745
+ # next run in this process.
746
+ try:
747
+ tracer.uninstall() # stop callbacks before closing the db
748
+ _disarm_audit()
749
+ finally:
750
+ thread_starts = finish_thread_starts(tracer)
751
+ child_stats = {"python_children": 0, "child_record_errors": 0,
752
+ "non_python_children": 0}
753
+ if child_dir is not None:
754
+ try:
755
+ child_stats = merge_children(w, child_dir)
756
+ except Exception:
757
+ child_stats["child_record_errors"] += 1
758
+ if child_arm_failed:
759
+ child_stats["child_record_errors"] += 1
760
+ disarm_children()
761
+ sys.stdin, sys.stdout, sys.stderr, sys.argv = saved[:4]
762
+ sys.path[:] = saved[4]
763
+ w.seal() # in-flight callbacks stop writing here
764
+ live = _live_thread_names()
765
+ try:
766
+ _finalize_meta(
767
+ w, exit_status=exit_status, uncaught=uncaught,
768
+ stdin_consumed=stdin_proxy.consumed, children=children,
769
+ truncated_count=(capture.capture_stats["truncated"]
770
+ - truncated_before),
771
+ live_threads=live, entry=entry,
772
+ threads_started=len(started),
773
+ audit_errors=len(audit_errors),
774
+ spawn_syscalls=len(spawns),
775
+ task_errors=tracer.task_errors,
776
+ redaction=redaction,
777
+ values=rv.stats["values"] - values_before,
778
+ thread_starts=thread_starts, child_stats=child_stats)
779
+ finally:
780
+ w.close() # never skipped: no leaked connection
781
+ gaps = _recording_gaps(live, w.late_writes)
782
+ if gaps:
783
+ print(gaps, file=sys.stderr)
784
+ _auto_gc()
785
+ return run_id, exit_status
786
+
787
+
788
+ def _auto_gc() -> None:
789
+ """The store's own budget. A failure here does not change the exit."""
790
+ try:
791
+ from sensorium.store.gc import maybe_auto
792
+ note = maybe_auto()
793
+ except Exception as e:
794
+ print(f"sensorium: gc failed: {e}", file=sys.stderr)
795
+ return
796
+ if note:
797
+ print(note, file=sys.stderr)