sensorium-trace 0.18.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sensorium/__init__.py +0 -0
- sensorium/__main__.py +4 -0
- sensorium/_c/rt/sensorium_c_rt.c +1446 -0
- sensorium/_typescript/package-lock.json +68 -0
- sensorium/_typescript/package.json +39 -0
- sensorium/_typescript/src/bindings.mjs +792 -0
- sensorium/_typescript/src/cap.mjs +227 -0
- sensorium/_typescript/src/capabilities.mjs +40 -0
- sensorium/_typescript/src/dbg.mjs +348 -0
- sensorium/_typescript/src/escape.mjs +351 -0
- sensorium/_typescript/src/focus.mjs +135 -0
- sensorium/_typescript/src/hook.mjs +127 -0
- sensorium/_typescript/src/index.mjs +1 -0
- sensorium/_typescript/src/naming.mjs +63 -0
- sensorium/_typescript/src/positions.mjs +37 -0
- sensorium/_typescript/src/probe.mjs +246 -0
- sensorium/_typescript/src/qualname.mjs +236 -0
- sensorium/_typescript/src/redact.mjs +711 -0
- sensorium/_typescript/src/register.mjs +15 -0
- sensorium/_typescript/src/resolve.mjs +260 -0
- sensorium/_typescript/src/rt.mjs +798 -0
- sensorium/_typescript/src/setup.mjs +11 -0
- sensorium/_typescript/src/tally.mjs +164 -0
- sensorium/_typescript/src/tasks.mjs +175 -0
- sensorium/_typescript/src/transform.mjs +800 -0
- sensorium/_typescript/src/vite.mjs +99 -0
- sensorium/c/__init__.py +8 -0
- sensorium/c/cc.py +612 -0
- sensorium/c/cli.py +60 -0
- sensorium/c/convert.py +669 -0
- sensorium/c/decode.py +300 -0
- sensorium/c/driver.py +365 -0
- sensorium/c/dwarf.py +706 -0
- sensorium/c/format.py +271 -0
- sensorium/c/pkg.py +187 -0
- sensorium/c/plan.py +277 -0
- sensorium/cli.py +140 -0
- sensorium/driver.py +47 -0
- sensorium/exit.py +33 -0
- sensorium/invocations.py +83 -0
- sensorium/limits.py +253 -0
- sensorium/mcp/__init__.py +0 -0
- sensorium/mcp/audit.py +162 -0
- sensorium/mcp/child.py +240 -0
- sensorium/mcp/cmd.py +101 -0
- sensorium/mcp/jsonrpc.py +172 -0
- sensorium/mcp/result.py +187 -0
- sensorium/mcp/schema.py +390 -0
- sensorium/mcp/server.py +493 -0
- sensorium/mcp/tools.py +299 -0
- sensorium/paths.py +80 -0
- sensorium/query/__init__.py +0 -0
- sensorium/query/caps.py +150 -0
- sensorium/query/dbg_dialects.py +64 -0
- sensorium/query/diff_cmd.py +748 -0
- sensorium/query/diff_notes.py +117 -0
- sensorium/query/exceptions_c.py +263 -0
- sensorium/query/exceptions_cmd.py +788 -0
- sensorium/query/exceptions_group.py +571 -0
- sensorium/query/exceptions_invocation.py +487 -0
- sensorium/query/exceptions_rust.py +683 -0
- sensorium/query/exceptions_typescript.py +688 -0
- sensorium/query/expr.py +606 -0
- sensorium/query/failures_cmd.py +306 -0
- sensorium/query/flow_cmd.py +778 -0
- sensorium/query/flow_report.py +128 -0
- sensorium/query/flow_values.py +195 -0
- sensorium/query/fmt.py +280 -0
- sensorium/query/frame_cmd.py +237 -0
- sensorium/query/grep_cmd.py +156 -0
- sensorium/query/help.py +18 -0
- sensorium/query/info_cmd.py +527 -0
- sensorium/query/info_rust.py +214 -0
- sensorium/query/info_typescript.py +261 -0
- sensorium/query/js_inspect.py +394 -0
- sensorium/query/moves.py +254 -0
- sensorium/query/redact_cmd.py +304 -0
- sensorium/query/refocus_cmd.py +661 -0
- sensorium/query/refocus_env.py +493 -0
- sensorium/query/refocus_facts.py +269 -0
- sensorium/query/refocus_licence.py +187 -0
- sensorium/query/refocus_report.py +327 -0
- sensorium/query/refocus_rust.py +537 -0
- sensorium/query/refocus_threads.py +197 -0
- sensorium/query/refocus_typescript.py +756 -0
- sensorium/query/refocus_world.py +666 -0
- sensorium/query/runs_cmd.py +213 -0
- sensorium/query/rust_debug.py +176 -0
- sensorium/query/sites.py +171 -0
- sensorium/query/tree_cmd.py +511 -0
- sensorium/query/tree_threads.py +168 -0
- sensorium/query/vocab.py +723 -0
- sensorium/query/watch_cmd.py +657 -0
- sensorium/record/__init__.py +0 -0
- sensorium/record/boot.py +797 -0
- sensorium/record/boot_io.py +141 -0
- sensorium/record/budget.py +135 -0
- sensorium/record/capture.py +433 -0
- sensorium/record/children.py +755 -0
- sensorium/record/children_merge.py +179 -0
- sensorium/record/elide.py +55 -0
- sensorium/record/fingerprint.py +26 -0
- sensorium/record/thread_starts.py +261 -0
- sensorium/record/tracer.py +798 -0
- sensorium/record/tracer_exc.py +199 -0
- sensorium/record/tracer_frames.py +618 -0
- sensorium/redact.py +304 -0
- sensorium/redact_content.py +156 -0
- sensorium/redact_key.py +317 -0
- sensorium/redact_store.py +502 -0
- sensorium/redact_values.py +433 -0
- sensorium/store/__init__.py +0 -0
- sensorium/store/db.py +284 -0
- sensorium/store/gc.py +360 -0
- sensorium/store/payloads.py +69 -0
- sensorium/store/reader.py +438 -0
- sensorium/store/writer.py +283 -0
- sensorium/ts/__init__.py +10 -0
- sensorium/ts/build.py +783 -0
- sensorium/ts/cli.py +130 -0
- sensorium/ts/driver.py +472 -0
- sensorium/ts/focus.py +146 -0
- sensorium/ts/harness.py +314 -0
- sensorium/ts/ingest.py +380 -0
- sensorium/ts/invocation.py +252 -0
- sensorium/ts/pkg.py +167 -0
- sensorium/ts/redaction.py +64 -0
- sensorium/ts/spool.py +168 -0
- sensorium/ts/truncation.py +81 -0
- sensorium/ts/wrapper.py +255 -0
- sensorium_trace-0.18.2.dist-info/METADATA +245 -0
- sensorium_trace-0.18.2.dist-info/RECORD +135 -0
- sensorium_trace-0.18.2.dist-info/WHEEL +4 -0
- sensorium_trace-0.18.2.dist-info/entry_points.txt +3 -0
- sensorium_trace-0.18.2.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,755 @@
|
|
|
1
|
+
"""Record Python child processes into the run that spawned them.
|
|
2
|
+
|
|
3
|
+
`subprocess` and `multiprocessing` (fork, and spawn via fork_exec) start a
|
|
4
|
+
new interpreter or a forked copy of this one. The child's calls never appear
|
|
5
|
+
in the parent trace: the parent only sees the spawn. This module follows a
|
|
6
|
+
child when the executable is Python, writes a side spool the parent does not
|
|
7
|
+
share, and merges that spool back under the frame that spawned it.
|
|
8
|
+
|
|
9
|
+
A child that is not Python is counted and left alone. `SENSORIUM_NO_CHILDREN`
|
|
10
|
+
and `--no-children` skip the whole thing. A failure inside a child, or while
|
|
11
|
+
merging one, is counted and does not fail the parent.
|
|
12
|
+
|
|
13
|
+
The side directory lives outside the working directory on purpose. The
|
|
14
|
+
tracer records every file under that directory, and a hook placed inside it
|
|
15
|
+
would record itself.
|
|
16
|
+
"""
|
|
17
|
+
import atexit
|
|
18
|
+
import json
|
|
19
|
+
import os
|
|
20
|
+
import shutil
|
|
21
|
+
import subprocess
|
|
22
|
+
import sys
|
|
23
|
+
import tempfile
|
|
24
|
+
import threading
|
|
25
|
+
from pathlib import Path
|
|
26
|
+
|
|
27
|
+
from sensorium.record.children_merge import (
|
|
28
|
+
merge_one as _merge_one,
|
|
29
|
+
process_alive as _process_alive,
|
|
30
|
+
spool_status as _spool_status,
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
_OFF = {"1", "true", "yes", "on"}
|
|
34
|
+
_SELF = os.path.realpath(__file__)
|
|
35
|
+
_SUBPROCESS = os.path.realpath(subprocess.__file__)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class _State:
|
|
39
|
+
def __init__(self) -> None:
|
|
40
|
+
self.armed = False
|
|
41
|
+
self.record = False
|
|
42
|
+
self.tracer = None
|
|
43
|
+
self.spool_dir: Path | None = None
|
|
44
|
+
self.hook_dir: Path | None = None
|
|
45
|
+
self.non_python = 0
|
|
46
|
+
self.popen_orig = None
|
|
47
|
+
self.fork_exec_orig = None
|
|
48
|
+
self.fork_hook = False
|
|
49
|
+
self.forked = False
|
|
50
|
+
self.pending_frame: int | None = None
|
|
51
|
+
self.flushed = False
|
|
52
|
+
self.finish_registered = False
|
|
53
|
+
self.leaked: list = []
|
|
54
|
+
self.exit_orig = None
|
|
55
|
+
self.spawn_orig = None
|
|
56
|
+
self.execve_orig = None
|
|
57
|
+
self.focus = ()
|
|
58
|
+
self.include = ()
|
|
59
|
+
self.exclude = ()
|
|
60
|
+
self.window = None
|
|
61
|
+
self.limits = None
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
_STATE = _State()
|
|
65
|
+
# os._exit must call whichever spool is current. A forkserver worker
|
|
66
|
+
# inherits the wrapper and then points the tracer at its own file; a
|
|
67
|
+
# closure captured in the forkserver would flush the server instead.
|
|
68
|
+
_EXECING: set[int] = set()
|
|
69
|
+
_PENDING: dict[int, int | None] = {}
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def children_enabled(env=None) -> bool:
|
|
73
|
+
"""True unless the process asked not to follow children."""
|
|
74
|
+
env = os.environ if env is None else env
|
|
75
|
+
return str(env.get("SENSORIUM_NO_CHILDREN", "")).lower() not in _OFF
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def arm_children(tracer, *, record=True, focus=(), include=(), exclude=(),
|
|
79
|
+
window=None, limits=None) -> Path | None:
|
|
80
|
+
"""Start following children for this run.
|
|
81
|
+
|
|
82
|
+
Returns the spool directory, or None when this run is not recording
|
|
83
|
+
children. The fork hook is installed either way: a forked interpreter
|
|
84
|
+
otherwise keeps the parent's monitoring callbacks and writes the
|
|
85
|
+
child's calls into the parent's trace file.
|
|
86
|
+
"""
|
|
87
|
+
_STATE.tracer = tracer
|
|
88
|
+
_STATE.focus = tuple(focus or ())
|
|
89
|
+
_STATE.include = tuple(include or ())
|
|
90
|
+
_STATE.exclude = tuple(exclude or ())
|
|
91
|
+
_STATE.window = window
|
|
92
|
+
_STATE.limits = limits
|
|
93
|
+
_STATE.non_python = 0
|
|
94
|
+
_STATE.flushed = False
|
|
95
|
+
_STATE.armed = True
|
|
96
|
+
_STATE.record = bool(record)
|
|
97
|
+
_install_fork_hook()
|
|
98
|
+
if not record:
|
|
99
|
+
return None
|
|
100
|
+
spool = Path(tempfile.mkdtemp(prefix="sensorium-children-"))
|
|
101
|
+
os.chmod(spool, 0o700)
|
|
102
|
+
hook = spool / "hook"
|
|
103
|
+
hook.mkdir(mode=0o700)
|
|
104
|
+
_write_hook(hook)
|
|
105
|
+
_STATE.spool_dir = spool
|
|
106
|
+
_STATE.hook_dir = hook
|
|
107
|
+
_install_popen()
|
|
108
|
+
_install_fork_exec()
|
|
109
|
+
_install_spawn_exec()
|
|
110
|
+
return spool
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def disarm_children() -> None:
|
|
114
|
+
"""Stop following. The fork hook cannot be removed; the flag no-ops it."""
|
|
115
|
+
_STATE.armed = False
|
|
116
|
+
_STATE.record = False
|
|
117
|
+
_STATE.tracer = None
|
|
118
|
+
if _STATE.popen_orig is not None:
|
|
119
|
+
subprocess.Popen.__init__ = _STATE.popen_orig
|
|
120
|
+
_STATE.popen_orig = None
|
|
121
|
+
if _STATE.fork_exec_orig is not None:
|
|
122
|
+
try:
|
|
123
|
+
import _posixsubprocess
|
|
124
|
+
_posixsubprocess.fork_exec = _STATE.fork_exec_orig
|
|
125
|
+
except Exception:
|
|
126
|
+
pass
|
|
127
|
+
_STATE.fork_exec_orig = None
|
|
128
|
+
if _STATE.spawn_orig is not None:
|
|
129
|
+
os.posix_spawn = _STATE.spawn_orig
|
|
130
|
+
_STATE.spawn_orig = None
|
|
131
|
+
if _STATE.execve_orig is not None:
|
|
132
|
+
os.execve = _STATE.execve_orig
|
|
133
|
+
_STATE.execve_orig = None
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def merge_children(guard, spool_dir) -> dict:
|
|
137
|
+
"""Copy finished child spools into `guard`. Never raises.
|
|
138
|
+
|
|
139
|
+
Incomplete spools are skipped. A resource-tracker process is still alive
|
|
140
|
+
at finalize and must not count as a failed child. A spool that cannot be
|
|
141
|
+
read is counted in `child_record_errors`.
|
|
142
|
+
"""
|
|
143
|
+
stats = {
|
|
144
|
+
"python_children": 0,
|
|
145
|
+
"child_record_errors": 0,
|
|
146
|
+
"non_python_children": _STATE.non_python,
|
|
147
|
+
}
|
|
148
|
+
directory = Path(spool_dir)
|
|
149
|
+
try:
|
|
150
|
+
if not directory.is_dir():
|
|
151
|
+
return stats
|
|
152
|
+
for err in directory.glob("*.err"):
|
|
153
|
+
if err.is_file():
|
|
154
|
+
stats["child_record_errors"] += 1
|
|
155
|
+
for path in sorted(directory.glob("*.db")):
|
|
156
|
+
try:
|
|
157
|
+
done, pid = _spool_status(path)
|
|
158
|
+
except Exception:
|
|
159
|
+
stats["child_record_errors"] += 1
|
|
160
|
+
continue
|
|
161
|
+
# A live child (forkserver, resource tracker) has not finished
|
|
162
|
+
# its spool. Skipping it is not a failed record.
|
|
163
|
+
if not done and _process_alive(pid):
|
|
164
|
+
continue
|
|
165
|
+
if not done:
|
|
166
|
+
stats["child_record_errors"] += 1
|
|
167
|
+
try:
|
|
168
|
+
if _merge_one(guard, path):
|
|
169
|
+
stats["python_children"] += 1
|
|
170
|
+
except Exception:
|
|
171
|
+
stats["child_record_errors"] += 1
|
|
172
|
+
except Exception:
|
|
173
|
+
stats["child_record_errors"] += 1
|
|
174
|
+
finally:
|
|
175
|
+
shutil.rmtree(directory, ignore_errors=True)
|
|
176
|
+
_STATE.spool_dir = None
|
|
177
|
+
return stats
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def record_this_process() -> None:
|
|
181
|
+
"""Entry point for a spawned interpreter's sitecustomize.
|
|
182
|
+
|
|
183
|
+
Forked children do not come through here: they already have a tracer,
|
|
184
|
+
which `os.register_at_fork` points at a fresh spool.
|
|
185
|
+
"""
|
|
186
|
+
if not children_enabled() or _STATE.flushed:
|
|
187
|
+
return
|
|
188
|
+
spool = os.environ.get("SENSORIUM_CHILD_SPOOL")
|
|
189
|
+
if not spool:
|
|
190
|
+
return
|
|
191
|
+
try:
|
|
192
|
+
_record_spawned(Path(spool))
|
|
193
|
+
except Exception:
|
|
194
|
+
_note_error(Path(spool), "start")
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def _record_spawned(spool_dir: Path) -> None:
|
|
198
|
+
from sensorium.limits import Limits
|
|
199
|
+
from sensorium.record.tracer import FocusSpec, Tracer
|
|
200
|
+
from sensorium.store.writer import TraceWriter
|
|
201
|
+
|
|
202
|
+
_install_redact()
|
|
203
|
+
cfg = _spawn_cfg()
|
|
204
|
+
hook = os.environ.get("SENSORIUM_CHILD_HOOK")
|
|
205
|
+
if hook:
|
|
206
|
+
_STATE.hook_dir = Path(hook)
|
|
207
|
+
path = spool_dir / f"{os.getpid()}.db"
|
|
208
|
+
writer = TraceWriter(path)
|
|
209
|
+
frame = _env_frame()
|
|
210
|
+
writer.set_meta("parent_frame", frame)
|
|
211
|
+
writer.set_meta("incomplete", True)
|
|
212
|
+
root = Path(os.environ.get("SENSORIUM_CHILD_ROOT") or os.getcwd())
|
|
213
|
+
lim = cfg.get("limits")
|
|
214
|
+
limits = (Limits(int(lim[0]), int(lim[1]), int(lim[2]))
|
|
215
|
+
if isinstance(lim, list) and len(lim) == 3 else None)
|
|
216
|
+
tracer = Tracer(
|
|
217
|
+
writer, root=root, focus=FocusSpec(list(cfg.get("focus") or ())),
|
|
218
|
+
include=tuple(cfg.get("include") or ()),
|
|
219
|
+
exclude=tuple(cfg.get("exclude") or ()),
|
|
220
|
+
window=cfg.get("window"), limits=limits)
|
|
221
|
+
tracer.install()
|
|
222
|
+
_STATE.tracer = tracer
|
|
223
|
+
_STATE.spool_dir = spool_dir
|
|
224
|
+
_STATE.armed = True
|
|
225
|
+
_STATE.record = True
|
|
226
|
+
_STATE.flushed = False
|
|
227
|
+
_STATE.focus = tuple(cfg.get("focus") or ())
|
|
228
|
+
_STATE.include = tuple(cfg.get("include") or ())
|
|
229
|
+
_STATE.exclude = tuple(cfg.get("exclude") or ())
|
|
230
|
+
_STATE.window = cfg.get("window")
|
|
231
|
+
_STATE.limits = limits
|
|
232
|
+
# This process may itself fork (a forkserver). The hook is per
|
|
233
|
+
# interpreter, and exec does not keep the parent's registration.
|
|
234
|
+
_install_fork_hook()
|
|
235
|
+
_install_popen()
|
|
236
|
+
_install_fork_exec()
|
|
237
|
+
_install_spawn_exec()
|
|
238
|
+
_arm_finish()
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def _redirect_fork_child(frame: int | None) -> None:
|
|
242
|
+
"""Point the inherited tracer at a new spool.
|
|
243
|
+
|
|
244
|
+
The copied writer still holds the parent's SQLite connection. Closing it
|
|
245
|
+
here would checkpoint the parent's database from the child. It is kept
|
|
246
|
+
alive and unused instead: multiprocessing leaves via os._exit, which
|
|
247
|
+
does not run that destructor. Calls that were already open belong to the
|
|
248
|
+
parent, so their returns are dropped (`Tracer._fork_child`).
|
|
249
|
+
"""
|
|
250
|
+
from sensorium.store.writer import TraceWriter
|
|
251
|
+
|
|
252
|
+
tracer = _STATE.tracer
|
|
253
|
+
spool = _STATE.spool_dir
|
|
254
|
+
if tracer is None or spool is None:
|
|
255
|
+
return
|
|
256
|
+
_STATE.flushed = False
|
|
257
|
+
_STATE.leaked.append(tracer.writer)
|
|
258
|
+
_STATE.leaked.append(tracer)
|
|
259
|
+
writer = TraceWriter(spool / f"{os.getpid()}.db")
|
|
260
|
+
writer.set_meta("parent_frame", frame)
|
|
261
|
+
writer.set_meta("incomplete", True)
|
|
262
|
+
tracer.writer = writer
|
|
263
|
+
# The copied budget still holds the parent's writer. Counting from
|
|
264
|
+
# here keeps this child's calls on this spool, and a later report
|
|
265
|
+
# reads this connection rather than the parent's.
|
|
266
|
+
if getattr(tracer, "_budget", None) is not None:
|
|
267
|
+
from sensorium.record.budget import Budget
|
|
268
|
+
tracer._budget = Budget(tracer.limits, writer)
|
|
269
|
+
tracer._fork_child = True
|
|
270
|
+
tracer._tls.live.clear()
|
|
271
|
+
with tracer._parked_lock:
|
|
272
|
+
tracer._parked.clear()
|
|
273
|
+
with tracer._fp_lock:
|
|
274
|
+
tracer._fps.clear()
|
|
275
|
+
tracer._task_fps.clear()
|
|
276
|
+
_arm_finish()
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def _flush_current() -> None:
|
|
280
|
+
"""Close this process's spool. Looks up the tracer at call time.
|
|
281
|
+
|
|
282
|
+
A forkserver worker inherits the parent's os._exit wrapper. The wrapper
|
|
283
|
+
has to flush the worker's writer, which was installed after the fork,
|
|
284
|
+
not the server's writer the wrapper was first created beside.
|
|
285
|
+
"""
|
|
286
|
+
if _STATE.flushed:
|
|
287
|
+
return
|
|
288
|
+
_STATE.flushed = True
|
|
289
|
+
tracer = _STATE.tracer
|
|
290
|
+
if tracer is None:
|
|
291
|
+
return
|
|
292
|
+
writer = tracer.writer
|
|
293
|
+
try:
|
|
294
|
+
tracer.uninstall()
|
|
295
|
+
writer.set_meta("incomplete", False)
|
|
296
|
+
except Exception:
|
|
297
|
+
spool = _STATE.spool_dir
|
|
298
|
+
if spool is not None:
|
|
299
|
+
_note_error(spool, "flush")
|
|
300
|
+
try:
|
|
301
|
+
writer.set_meta("incomplete", True)
|
|
302
|
+
except Exception:
|
|
303
|
+
pass
|
|
304
|
+
finally:
|
|
305
|
+
try:
|
|
306
|
+
writer.close()
|
|
307
|
+
except Exception:
|
|
308
|
+
pass
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def _arm_finish() -> None:
|
|
312
|
+
_wrap_exit()
|
|
313
|
+
if _STATE.finish_registered:
|
|
314
|
+
return
|
|
315
|
+
atexit.register(_flush_current)
|
|
316
|
+
_STATE.finish_registered = True
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def _silence_inherited() -> None:
|
|
320
|
+
"""Drop a forked tracer without writing through its connection.
|
|
321
|
+
|
|
322
|
+
`Tracer.uninstall` stores fingerprints. On an opt-out fork that would
|
|
323
|
+
write the child's calls into the parent's trace. Events are turned off
|
|
324
|
+
and the writer is pinned so it is not closed onto the parent's file.
|
|
325
|
+
"""
|
|
326
|
+
tracer = _STATE.tracer
|
|
327
|
+
if tracer is not None:
|
|
328
|
+
_STATE.leaked.append(tracer)
|
|
329
|
+
writer = getattr(tracer, "writer", None)
|
|
330
|
+
if writer is not None:
|
|
331
|
+
_STATE.leaked.append(writer)
|
|
332
|
+
try:
|
|
333
|
+
mon = sys.monitoring
|
|
334
|
+
tool = mon.PROFILER_ID
|
|
335
|
+
mon.set_events(tool, 0)
|
|
336
|
+
ev = mon.events
|
|
337
|
+
for kind in (ev.PY_START, ev.PY_RETURN, ev.PY_UNWIND, ev.RAISE,
|
|
338
|
+
ev.RERAISE, ev.EXCEPTION_HANDLED, ev.LINE,
|
|
339
|
+
ev.PY_YIELD, ev.PY_RESUME, ev.PY_THROW):
|
|
340
|
+
try:
|
|
341
|
+
mon.register_callback(tool, kind, None)
|
|
342
|
+
except Exception:
|
|
343
|
+
pass
|
|
344
|
+
try:
|
|
345
|
+
mon.free_tool_id(tool)
|
|
346
|
+
except Exception:
|
|
347
|
+
pass
|
|
348
|
+
except Exception:
|
|
349
|
+
pass
|
|
350
|
+
_STATE.tracer = None
|
|
351
|
+
_STATE.record = False
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def _install_redact() -> None:
|
|
355
|
+
"""Same key and knobs the parent recorded under. A spawned interpreter
|
|
356
|
+
does not go through `run_target`, so nothing has installed the rule."""
|
|
357
|
+
try:
|
|
358
|
+
from sensorium import paths, redact
|
|
359
|
+
from sensorium import redact_values as rv
|
|
360
|
+
rv.install(redact.Key.load_or_create(paths.trace_root()),
|
|
361
|
+
redact.Knobs.from_environ(os.environ))
|
|
362
|
+
except Exception:
|
|
363
|
+
pass
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def _spawn_cfg() -> dict:
|
|
367
|
+
raw = os.environ.get("SENSORIUM_CHILD_CFG") or ""
|
|
368
|
+
if not raw:
|
|
369
|
+
return {}
|
|
370
|
+
try:
|
|
371
|
+
cfg = json.loads(raw)
|
|
372
|
+
except json.JSONDecodeError:
|
|
373
|
+
return {}
|
|
374
|
+
return cfg if isinstance(cfg, dict) else {}
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def _note_error(spool: Path, what: str) -> None:
|
|
378
|
+
try:
|
|
379
|
+
spool.mkdir(parents=True, exist_ok=True)
|
|
380
|
+
(spool / f"{os.getpid()}.err").write_text(what)
|
|
381
|
+
except Exception:
|
|
382
|
+
pass
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def _env_frame() -> int | None:
|
|
386
|
+
raw = os.environ.get("SENSORIUM_PARENT_FRAME", "")
|
|
387
|
+
if raw.isdigit():
|
|
388
|
+
return int(raw)
|
|
389
|
+
return None
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
def _is_python(argv0) -> bool:
|
|
393
|
+
if argv0 is None:
|
|
394
|
+
return False
|
|
395
|
+
try:
|
|
396
|
+
text = os.fsdecode(argv0)
|
|
397
|
+
except Exception:
|
|
398
|
+
return False
|
|
399
|
+
try:
|
|
400
|
+
if os.path.realpath(text) == os.path.realpath(sys.executable):
|
|
401
|
+
return True
|
|
402
|
+
except OSError:
|
|
403
|
+
pass
|
|
404
|
+
base = os.path.basename(text).lower()
|
|
405
|
+
if base.endswith(".exe"):
|
|
406
|
+
base = base[:-4]
|
|
407
|
+
if base in {"python", "python3"}:
|
|
408
|
+
return True
|
|
409
|
+
if base.startswith("python3."):
|
|
410
|
+
rest = base[len("python3."):]
|
|
411
|
+
if rest.isdigit():
|
|
412
|
+
return True
|
|
413
|
+
return False
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
def _machinery(filename: str) -> bool:
|
|
417
|
+
try:
|
|
418
|
+
real = os.path.realpath(filename)
|
|
419
|
+
except OSError:
|
|
420
|
+
real = filename
|
|
421
|
+
if real in {_SELF, _SUBPROCESS}:
|
|
422
|
+
return True
|
|
423
|
+
norm = real.replace("\\", "/")
|
|
424
|
+
return ("/multiprocessing/" in norm
|
|
425
|
+
or "/concurrent/futures/" in norm
|
|
426
|
+
or norm.endswith("/concurrent/futures.py"))
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
def _starter_frame(tracer) -> int | None:
|
|
430
|
+
"""The traced frame that asked for this child, or None.
|
|
431
|
+
|
|
432
|
+
stdlib and this module sit between that frame and the spawn. A frame
|
|
433
|
+
this run did not trace is not a parent the trace can name.
|
|
434
|
+
"""
|
|
435
|
+
if tracer is None:
|
|
436
|
+
return None
|
|
437
|
+
tls = tracer._tls
|
|
438
|
+
frame = sys._getframe()
|
|
439
|
+
while frame is not None:
|
|
440
|
+
if not _machinery(frame.f_code.co_filename):
|
|
441
|
+
entry = tls.live.get(id(frame))
|
|
442
|
+
if entry is not None and type(entry[0]) is int:
|
|
443
|
+
return entry[0]
|
|
444
|
+
return None
|
|
445
|
+
frame = frame.f_back
|
|
446
|
+
return None
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
def _cfg_blob() -> str:
|
|
450
|
+
limits = _STATE.limits
|
|
451
|
+
lim = None
|
|
452
|
+
if limits is not None:
|
|
453
|
+
lim = [int(limits.max_events), int(limits.max_bytes),
|
|
454
|
+
int(limits.max_calls)]
|
|
455
|
+
return json.dumps({
|
|
456
|
+
"focus": list(_STATE.focus),
|
|
457
|
+
"include": list(_STATE.include),
|
|
458
|
+
"exclude": list(_STATE.exclude),
|
|
459
|
+
"window": _STATE.window,
|
|
460
|
+
"limits": lim,
|
|
461
|
+
}, separators=(",", ":"))
|
|
462
|
+
|
|
463
|
+
|
|
464
|
+
def _child_env(base, frame: int | None) -> dict:
|
|
465
|
+
env = dict(os.environ if base is None else base)
|
|
466
|
+
# An explicit env would otherwise drop the store and the redaction
|
|
467
|
+
# knobs the child has to read. Do not copy SENSORIUM_NO_CHILDREN:
|
|
468
|
+
# a recorded parent records its grandchildren too.
|
|
469
|
+
for key in ("SENSORIUM_DIR", "SENSORIUM_NO_REDACT",
|
|
470
|
+
"SENSORIUM_REDACT_NAMES", "SENSORIUM_REDACT_ALLOW"):
|
|
471
|
+
if key not in env and key in os.environ:
|
|
472
|
+
env[key] = os.environ[key]
|
|
473
|
+
hook = str(_STATE.hook_dir) if _STATE.hook_dir is not None else ""
|
|
474
|
+
prev = env.get("PYTHONPATH", "")
|
|
475
|
+
if hook and (not prev or not prev.startswith(hook)):
|
|
476
|
+
env["PYTHONPATH"] = hook + (os.pathsep + prev if prev else "")
|
|
477
|
+
if hook:
|
|
478
|
+
env["SENSORIUM_CHILD_HOOK"] = hook
|
|
479
|
+
if _STATE.spool_dir is not None:
|
|
480
|
+
env["SENSORIUM_CHILD_SPOOL"] = str(_STATE.spool_dir)
|
|
481
|
+
root = getattr(_STATE.tracer, "root", None) if _STATE.tracer else None
|
|
482
|
+
env["SENSORIUM_CHILD_ROOT"] = str(root) if root else os.getcwd()
|
|
483
|
+
env["SENSORIUM_CHILD_CFG"] = _cfg_blob()
|
|
484
|
+
if frame is None:
|
|
485
|
+
env.pop("SENSORIUM_PARENT_FRAME", None)
|
|
486
|
+
else:
|
|
487
|
+
env["SENSORIUM_PARENT_FRAME"] = str(frame)
|
|
488
|
+
return env
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
def _as_env_list(env: dict) -> list:
|
|
492
|
+
rows = []
|
|
493
|
+
for key, value in env.items():
|
|
494
|
+
if value is None:
|
|
495
|
+
continue
|
|
496
|
+
rows.append(os.fsencode(str(key)) + b"=" + os.fsencode(os.fsdecode(value)
|
|
497
|
+
if isinstance(value, bytes) else str(value)))
|
|
498
|
+
return rows
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
def _env_marked(env) -> bool:
|
|
502
|
+
if not env:
|
|
503
|
+
return False
|
|
504
|
+
try:
|
|
505
|
+
items = list(env)
|
|
506
|
+
except Exception:
|
|
507
|
+
return False
|
|
508
|
+
for item in items:
|
|
509
|
+
try:
|
|
510
|
+
if isinstance(item, bytes):
|
|
511
|
+
text = item.decode("utf-8", "replace")
|
|
512
|
+
else:
|
|
513
|
+
text = str(item)
|
|
514
|
+
except Exception:
|
|
515
|
+
continue
|
|
516
|
+
if text.startswith("SENSORIUM_CHILD_SPOOL="):
|
|
517
|
+
return True
|
|
518
|
+
return False
|
|
519
|
+
|
|
520
|
+
|
|
521
|
+
def _popen_shell(args, kwargs) -> bool:
|
|
522
|
+
if "shell" in kwargs:
|
|
523
|
+
return bool(kwargs.get("shell"))
|
|
524
|
+
# Popen(args, bufsize, executable, stdin, stdout, stderr, preexec_fn,
|
|
525
|
+
# close_fds, shell, ...). shell is the 9th argument after args.
|
|
526
|
+
if len(args) > 8:
|
|
527
|
+
return bool(args[8])
|
|
528
|
+
return False
|
|
529
|
+
|
|
530
|
+
|
|
531
|
+
def _popen_head(args, kwargs):
|
|
532
|
+
executable = kwargs.get("executable")
|
|
533
|
+
if executable is None and len(args) > 2 and "executable" not in kwargs:
|
|
534
|
+
# Only trust a positional executable when the caller actually
|
|
535
|
+
# passed that many arguments. Index 2 is executable.
|
|
536
|
+
if len(args) > 2:
|
|
537
|
+
executable = args[2]
|
|
538
|
+
if executable:
|
|
539
|
+
return executable
|
|
540
|
+
argv = args[0] if args else kwargs.get("args")
|
|
541
|
+
if isinstance(argv, (list, tuple)):
|
|
542
|
+
return argv[0] if argv else None
|
|
543
|
+
return argv
|
|
544
|
+
|
|
545
|
+
|
|
546
|
+
def _inject_popen(args, kwargs):
|
|
547
|
+
if not (_STATE.armed and _STATE.record):
|
|
548
|
+
return args, kwargs
|
|
549
|
+
if _popen_shell(args, kwargs):
|
|
550
|
+
_STATE.non_python += 1
|
|
551
|
+
return args, kwargs
|
|
552
|
+
head = _popen_head(args, kwargs)
|
|
553
|
+
if head is None or not _is_python(head):
|
|
554
|
+
if head is not None:
|
|
555
|
+
_STATE.non_python += 1
|
|
556
|
+
return args, kwargs
|
|
557
|
+
frame = _starter_frame(_STATE.tracer)
|
|
558
|
+
if "env" in kwargs:
|
|
559
|
+
kwargs = dict(kwargs)
|
|
560
|
+
kwargs["env"] = _child_env(kwargs.get("env"), frame)
|
|
561
|
+
return args, kwargs
|
|
562
|
+
if len(args) > 10:
|
|
563
|
+
copied = list(args)
|
|
564
|
+
copied[10] = _child_env(copied[10], frame)
|
|
565
|
+
return tuple(copied), kwargs
|
|
566
|
+
kwargs = dict(kwargs)
|
|
567
|
+
kwargs["env"] = _child_env(None, frame)
|
|
568
|
+
return args, kwargs
|
|
569
|
+
|
|
570
|
+
|
|
571
|
+
def _install_popen() -> None:
|
|
572
|
+
if _STATE.popen_orig is not None:
|
|
573
|
+
return
|
|
574
|
+
orig = subprocess.Popen.__init__
|
|
575
|
+
|
|
576
|
+
def _init(self, *args, **kwargs):
|
|
577
|
+
try:
|
|
578
|
+
args, kwargs = _inject_popen(args, kwargs)
|
|
579
|
+
except Exception:
|
|
580
|
+
pass
|
|
581
|
+
return orig(self, *args, **kwargs)
|
|
582
|
+
|
|
583
|
+
subprocess.Popen.__init__ = _init
|
|
584
|
+
_STATE.popen_orig = orig
|
|
585
|
+
|
|
586
|
+
|
|
587
|
+
def _install_fork_exec() -> None:
|
|
588
|
+
if _STATE.fork_exec_orig is not None:
|
|
589
|
+
return
|
|
590
|
+
try:
|
|
591
|
+
import _posixsubprocess
|
|
592
|
+
except ImportError:
|
|
593
|
+
return
|
|
594
|
+
orig = _posixsubprocess.fork_exec
|
|
595
|
+
|
|
596
|
+
def _fork_exec(args, exe_list, *rest, **kwargs):
|
|
597
|
+
# vfork does not run at-fork hooks. A real fork inside fork_exec
|
|
598
|
+
# does, and that child is about to exec: it must not open a spool.
|
|
599
|
+
ident = threading.get_ident()
|
|
600
|
+
_EXECING.add(ident)
|
|
601
|
+
try:
|
|
602
|
+
if not (_STATE.armed and _STATE.record):
|
|
603
|
+
return orig(args, exe_list, *rest, **kwargs)
|
|
604
|
+
head = None
|
|
605
|
+
if isinstance(exe_list, (list, tuple)) and exe_list:
|
|
606
|
+
head = exe_list[0]
|
|
607
|
+
elif isinstance(args, (list, tuple)) and args:
|
|
608
|
+
head = args[0]
|
|
609
|
+
if not _is_python(head):
|
|
610
|
+
return orig(args, exe_list, *rest, **kwargs)
|
|
611
|
+
# env is the 4th argument after executable_list: close_fds,
|
|
612
|
+
# pass_fds, cwd, env. Replacing None is what the child sees.
|
|
613
|
+
# Mutating os.environ races other threads and is restored too
|
|
614
|
+
# late to be the only path.
|
|
615
|
+
if len(rest) >= 4 and not _env_marked(rest[3]):
|
|
616
|
+
frame = _starter_frame(_STATE.tracer)
|
|
617
|
+
if frame is None:
|
|
618
|
+
frame = _env_frame()
|
|
619
|
+
updated = list(rest)
|
|
620
|
+
if updated[3] is None:
|
|
621
|
+
updated[3] = _as_env_list(_child_env(None, frame))
|
|
622
|
+
return orig(args, exe_list, *updated, **kwargs)
|
|
623
|
+
return orig(args, exe_list, *rest, **kwargs)
|
|
624
|
+
finally:
|
|
625
|
+
_EXECING.discard(ident)
|
|
626
|
+
|
|
627
|
+
_posixsubprocess.fork_exec = _fork_exec
|
|
628
|
+
_STATE.fork_exec_orig = orig
|
|
629
|
+
|
|
630
|
+
|
|
631
|
+
def _install_spawn_exec() -> None:
|
|
632
|
+
"""posix_spawn and execve snapshot the environment before the audit
|
|
633
|
+
hook runs, so the hook cannot add variables. Inject before the call.
|
|
634
|
+
"""
|
|
635
|
+
if _STATE.spawn_orig is None and hasattr(os, "posix_spawn"):
|
|
636
|
+
orig_spawn = os.posix_spawn
|
|
637
|
+
|
|
638
|
+
def _posix_spawn(path, argv, env, *args, **kwargs):
|
|
639
|
+
try:
|
|
640
|
+
if (_STATE.armed and _STATE.record
|
|
641
|
+
and _is_python(path or (argv[0] if argv else None))):
|
|
642
|
+
env = _child_env(env, _starter_frame(_STATE.tracer))
|
|
643
|
+
except Exception:
|
|
644
|
+
pass
|
|
645
|
+
return orig_spawn(path, argv, env, *args, **kwargs)
|
|
646
|
+
|
|
647
|
+
os.posix_spawn = _posix_spawn
|
|
648
|
+
_STATE.spawn_orig = orig_spawn
|
|
649
|
+
if _STATE.execve_orig is None:
|
|
650
|
+
orig_exec = os.execve
|
|
651
|
+
|
|
652
|
+
def _execve(path, argv, env):
|
|
653
|
+
try:
|
|
654
|
+
if _STATE.armed and _STATE.record and _is_python(path):
|
|
655
|
+
env = _child_env(env, _starter_frame(_STATE.tracer))
|
|
656
|
+
except Exception:
|
|
657
|
+
pass
|
|
658
|
+
return orig_exec(path, argv, env)
|
|
659
|
+
|
|
660
|
+
os.execve = _execve
|
|
661
|
+
_STATE.execve_orig = orig_exec
|
|
662
|
+
|
|
663
|
+
|
|
664
|
+
def _install_fork_hook() -> None:
|
|
665
|
+
if _STATE.fork_hook:
|
|
666
|
+
return
|
|
667
|
+
if not hasattr(os, "register_at_fork"):
|
|
668
|
+
return
|
|
669
|
+
|
|
670
|
+
def _before() -> None:
|
|
671
|
+
ident = threading.get_ident()
|
|
672
|
+
if ident in _EXECING or not _STATE.armed:
|
|
673
|
+
return
|
|
674
|
+
if not _STATE.record:
|
|
675
|
+
_PENDING[ident] = None
|
|
676
|
+
return
|
|
677
|
+
# A forkserver worker's stack is stdlib, so the user frame is the
|
|
678
|
+
# one this process was spawned under, carried in the environment.
|
|
679
|
+
frame = _starter_frame(_STATE.tracer)
|
|
680
|
+
if frame is None:
|
|
681
|
+
frame = _env_frame()
|
|
682
|
+
_PENDING[ident] = frame
|
|
683
|
+
|
|
684
|
+
def _after_child() -> None:
|
|
685
|
+
ident = threading.get_ident()
|
|
686
|
+
if ident in _EXECING or not _STATE.armed:
|
|
687
|
+
return
|
|
688
|
+
frame = _PENDING.pop(ident, None)
|
|
689
|
+
if not _STATE.record:
|
|
690
|
+
_silence_inherited()
|
|
691
|
+
return
|
|
692
|
+
if frame is None:
|
|
693
|
+
frame = _env_frame()
|
|
694
|
+
try:
|
|
695
|
+
_redirect_fork_child(frame)
|
|
696
|
+
except Exception:
|
|
697
|
+
if _STATE.spool_dir is not None:
|
|
698
|
+
_note_error(_STATE.spool_dir, "fork")
|
|
699
|
+
|
|
700
|
+
def _after_parent() -> None:
|
|
701
|
+
_PENDING.pop(threading.get_ident(), None)
|
|
702
|
+
|
|
703
|
+
os.register_at_fork(before=_before, after_in_child=_after_child,
|
|
704
|
+
after_in_parent=_after_parent)
|
|
705
|
+
_STATE.fork_hook = True
|
|
706
|
+
|
|
707
|
+
|
|
708
|
+
def _wrap_exit() -> None:
|
|
709
|
+
if _STATE.exit_orig is not None:
|
|
710
|
+
return
|
|
711
|
+
orig = os._exit
|
|
712
|
+
|
|
713
|
+
def _exit(code=0):
|
|
714
|
+
try:
|
|
715
|
+
_flush_current()
|
|
716
|
+
except Exception:
|
|
717
|
+
pass
|
|
718
|
+
orig(code)
|
|
719
|
+
|
|
720
|
+
os._exit = _exit
|
|
721
|
+
_STATE.exit_orig = orig
|
|
722
|
+
|
|
723
|
+
|
|
724
|
+
def _write_hook(hook: Path) -> None:
|
|
725
|
+
# Chain any sitecustomize already on the path, then record. The hook
|
|
726
|
+
# directory is outside the traced tree, so this file is not a frame.
|
|
727
|
+
hook.joinpath("sitecustomize.py").write_text(
|
|
728
|
+
"import os, sys\n"
|
|
729
|
+
"_here = os.path.dirname(os.path.abspath(__file__))\n"
|
|
730
|
+
"def _chain():\n"
|
|
731
|
+
" try:\n"
|
|
732
|
+
" import importlib.util\n"
|
|
733
|
+
" for entry in list(sys.path):\n"
|
|
734
|
+
" if os.path.abspath(entry) == _here:\n"
|
|
735
|
+
" continue\n"
|
|
736
|
+
" cand = os.path.join(entry, 'sitecustomize.py')\n"
|
|
737
|
+
" if not os.path.isfile(cand):\n"
|
|
738
|
+
" continue\n"
|
|
739
|
+
" spec = importlib.util.spec_from_file_location(\n"
|
|
740
|
+
" '_sensorium_user_sitecustomize', cand)\n"
|
|
741
|
+
" if spec is None or spec.loader is None:\n"
|
|
742
|
+
" continue\n"
|
|
743
|
+
" mod = importlib.util.module_from_spec(spec)\n"
|
|
744
|
+
" spec.loader.exec_module(mod)\n"
|
|
745
|
+
" return\n"
|
|
746
|
+
" except Exception:\n"
|
|
747
|
+
" pass\n"
|
|
748
|
+
"_chain()\n"
|
|
749
|
+
"try:\n"
|
|
750
|
+
" if os.environ.get('SENSORIUM_CHILD_SPOOL'):\n"
|
|
751
|
+
" from sensorium.record.children import record_this_process\n"
|
|
752
|
+
" record_this_process()\n"
|
|
753
|
+
"except Exception:\n"
|
|
754
|
+
" pass\n"
|
|
755
|
+
)
|