sensorium-trace 0.18.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sensorium/__init__.py +0 -0
- sensorium/__main__.py +4 -0
- sensorium/_c/rt/sensorium_c_rt.c +1446 -0
- sensorium/_typescript/package-lock.json +68 -0
- sensorium/_typescript/package.json +39 -0
- sensorium/_typescript/src/bindings.mjs +792 -0
- sensorium/_typescript/src/cap.mjs +227 -0
- sensorium/_typescript/src/capabilities.mjs +40 -0
- sensorium/_typescript/src/dbg.mjs +348 -0
- sensorium/_typescript/src/escape.mjs +351 -0
- sensorium/_typescript/src/focus.mjs +135 -0
- sensorium/_typescript/src/hook.mjs +127 -0
- sensorium/_typescript/src/index.mjs +1 -0
- sensorium/_typescript/src/naming.mjs +63 -0
- sensorium/_typescript/src/positions.mjs +37 -0
- sensorium/_typescript/src/probe.mjs +246 -0
- sensorium/_typescript/src/qualname.mjs +236 -0
- sensorium/_typescript/src/redact.mjs +711 -0
- sensorium/_typescript/src/register.mjs +15 -0
- sensorium/_typescript/src/resolve.mjs +260 -0
- sensorium/_typescript/src/rt.mjs +798 -0
- sensorium/_typescript/src/setup.mjs +11 -0
- sensorium/_typescript/src/tally.mjs +164 -0
- sensorium/_typescript/src/tasks.mjs +175 -0
- sensorium/_typescript/src/transform.mjs +800 -0
- sensorium/_typescript/src/vite.mjs +99 -0
- sensorium/c/__init__.py +8 -0
- sensorium/c/cc.py +612 -0
- sensorium/c/cli.py +60 -0
- sensorium/c/convert.py +669 -0
- sensorium/c/decode.py +300 -0
- sensorium/c/driver.py +365 -0
- sensorium/c/dwarf.py +706 -0
- sensorium/c/format.py +271 -0
- sensorium/c/pkg.py +187 -0
- sensorium/c/plan.py +277 -0
- sensorium/cli.py +140 -0
- sensorium/driver.py +47 -0
- sensorium/exit.py +33 -0
- sensorium/invocations.py +83 -0
- sensorium/limits.py +253 -0
- sensorium/mcp/__init__.py +0 -0
- sensorium/mcp/audit.py +162 -0
- sensorium/mcp/child.py +240 -0
- sensorium/mcp/cmd.py +101 -0
- sensorium/mcp/jsonrpc.py +172 -0
- sensorium/mcp/result.py +187 -0
- sensorium/mcp/schema.py +390 -0
- sensorium/mcp/server.py +493 -0
- sensorium/mcp/tools.py +299 -0
- sensorium/paths.py +80 -0
- sensorium/query/__init__.py +0 -0
- sensorium/query/caps.py +150 -0
- sensorium/query/dbg_dialects.py +64 -0
- sensorium/query/diff_cmd.py +748 -0
- sensorium/query/diff_notes.py +117 -0
- sensorium/query/exceptions_c.py +263 -0
- sensorium/query/exceptions_cmd.py +788 -0
- sensorium/query/exceptions_group.py +571 -0
- sensorium/query/exceptions_invocation.py +487 -0
- sensorium/query/exceptions_rust.py +683 -0
- sensorium/query/exceptions_typescript.py +688 -0
- sensorium/query/expr.py +606 -0
- sensorium/query/failures_cmd.py +306 -0
- sensorium/query/flow_cmd.py +778 -0
- sensorium/query/flow_report.py +128 -0
- sensorium/query/flow_values.py +195 -0
- sensorium/query/fmt.py +280 -0
- sensorium/query/frame_cmd.py +237 -0
- sensorium/query/grep_cmd.py +156 -0
- sensorium/query/help.py +18 -0
- sensorium/query/info_cmd.py +527 -0
- sensorium/query/info_rust.py +214 -0
- sensorium/query/info_typescript.py +261 -0
- sensorium/query/js_inspect.py +394 -0
- sensorium/query/moves.py +254 -0
- sensorium/query/redact_cmd.py +304 -0
- sensorium/query/refocus_cmd.py +661 -0
- sensorium/query/refocus_env.py +493 -0
- sensorium/query/refocus_facts.py +269 -0
- sensorium/query/refocus_licence.py +187 -0
- sensorium/query/refocus_report.py +327 -0
- sensorium/query/refocus_rust.py +537 -0
- sensorium/query/refocus_threads.py +197 -0
- sensorium/query/refocus_typescript.py +756 -0
- sensorium/query/refocus_world.py +666 -0
- sensorium/query/runs_cmd.py +213 -0
- sensorium/query/rust_debug.py +176 -0
- sensorium/query/sites.py +171 -0
- sensorium/query/tree_cmd.py +511 -0
- sensorium/query/tree_threads.py +168 -0
- sensorium/query/vocab.py +723 -0
- sensorium/query/watch_cmd.py +657 -0
- sensorium/record/__init__.py +0 -0
- sensorium/record/boot.py +797 -0
- sensorium/record/boot_io.py +141 -0
- sensorium/record/budget.py +135 -0
- sensorium/record/capture.py +433 -0
- sensorium/record/children.py +755 -0
- sensorium/record/children_merge.py +179 -0
- sensorium/record/elide.py +55 -0
- sensorium/record/fingerprint.py +26 -0
- sensorium/record/thread_starts.py +261 -0
- sensorium/record/tracer.py +798 -0
- sensorium/record/tracer_exc.py +199 -0
- sensorium/record/tracer_frames.py +618 -0
- sensorium/redact.py +304 -0
- sensorium/redact_content.py +156 -0
- sensorium/redact_key.py +317 -0
- sensorium/redact_store.py +502 -0
- sensorium/redact_values.py +433 -0
- sensorium/store/__init__.py +0 -0
- sensorium/store/db.py +284 -0
- sensorium/store/gc.py +360 -0
- sensorium/store/payloads.py +69 -0
- sensorium/store/reader.py +438 -0
- sensorium/store/writer.py +283 -0
- sensorium/ts/__init__.py +10 -0
- sensorium/ts/build.py +783 -0
- sensorium/ts/cli.py +130 -0
- sensorium/ts/driver.py +472 -0
- sensorium/ts/focus.py +146 -0
- sensorium/ts/harness.py +314 -0
- sensorium/ts/ingest.py +380 -0
- sensorium/ts/invocation.py +252 -0
- sensorium/ts/pkg.py +167 -0
- sensorium/ts/redaction.py +64 -0
- sensorium/ts/spool.py +168 -0
- sensorium/ts/truncation.py +81 -0
- sensorium/ts/wrapper.py +255 -0
- sensorium_trace-0.18.2.dist-info/METADATA +245 -0
- sensorium_trace-0.18.2.dist-info/RECORD +135 -0
- sensorium_trace-0.18.2.dist-info/WHEEL +4 -0
- sensorium_trace-0.18.2.dist-info/entry_points.txt +3 -0
- sensorium_trace-0.18.2.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
"""Batched, thread-safe writer for trace files. Buffers rows in memory and
|
|
2
|
+
flushes in one transaction per batch; on an unclean death, everything
|
|
3
|
+
already flushed is a valid partial trace.
|
|
4
|
+
|
|
5
|
+
That is the DURABLE mode, and it is the default because it is the recorder's:
|
|
6
|
+
a recorder writes a trace live, and a trace cut short is still worth reading.
|
|
7
|
+
A caller that builds a file nobody can see until it is renamed into place --
|
|
8
|
+
the converter (`sensorium.ts.ingest.convert`) -- passes `durable=False` and
|
|
9
|
+
gets one transaction for the whole trace under `synchronous=NORMAL`: the
|
|
10
|
+
partial trace it would be buying has no reader, so the fsyncs that make one
|
|
11
|
+
are a cost with nothing on the other side.
|
|
12
|
+
"""
|
|
13
|
+
import json
|
|
14
|
+
import threading
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
from sensorium.store import db, payloads
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class TraceWriter:
|
|
21
|
+
def __init__(self, path: Path, batch: int = 512,
|
|
22
|
+
durable: bool = True) -> None:
|
|
23
|
+
self.path = Path(path)
|
|
24
|
+
self._conn = db.create_trace(self.path)
|
|
25
|
+
self._durable = durable
|
|
26
|
+
if not durable:
|
|
27
|
+
# NORMAL under WAL: no fsync per commit. The file is renamed into
|
|
28
|
+
# place only after close() commits, so there is no earlier
|
|
29
|
+
# committed state a reader could be shown; synchronous below that
|
|
30
|
+
# point buys speed, not correctness (the Rust converter's rule).
|
|
31
|
+
self._conn.execute("PRAGMA synchronous=NORMAL")
|
|
32
|
+
self._lock = threading.Lock()
|
|
33
|
+
self._batch = batch
|
|
34
|
+
self._events: list[tuple] = []
|
|
35
|
+
self._frames: list[tuple] = []
|
|
36
|
+
self._closes: list[tuple] = []
|
|
37
|
+
self._outputs: list[tuple] = []
|
|
38
|
+
self._tasks: list[tuple] = []
|
|
39
|
+
self._codes: dict[tuple, int] = {}
|
|
40
|
+
self._new_codes: list[tuple] = []
|
|
41
|
+
self._next_event = 1
|
|
42
|
+
self._next_frame = 1
|
|
43
|
+
self._bytes_estimate = 0
|
|
44
|
+
# body -> 0 while only the inline copy exists, else the strings id.
|
|
45
|
+
self._interned: dict[str, int] = {}
|
|
46
|
+
self._string_rows: list[tuple[int, str]] = []
|
|
47
|
+
self._next_string = 1
|
|
48
|
+
self._strings_ready = False
|
|
49
|
+
|
|
50
|
+
@property
|
|
51
|
+
def bytes_estimate(self) -> int:
|
|
52
|
+
"""Payload bytes accepted so far, including the intern table."""
|
|
53
|
+
return self._bytes_estimate
|
|
54
|
+
|
|
55
|
+
def _intern_locked(self, text: str | None) -> str | None:
|
|
56
|
+
"""First copy of a long payload stays JSON. Later copies are ``#id``."""
|
|
57
|
+
if text is None:
|
|
58
|
+
return None
|
|
59
|
+
stored = text
|
|
60
|
+
if len(text) >= payloads.INTERN_MIN:
|
|
61
|
+
sid = self._interned.get(text)
|
|
62
|
+
if sid is None:
|
|
63
|
+
self._interned[text] = 0
|
|
64
|
+
elif sid == 0:
|
|
65
|
+
sid = self._next_string
|
|
66
|
+
self._next_string += 1
|
|
67
|
+
self._interned[text] = sid
|
|
68
|
+
self._string_rows.append((sid, text))
|
|
69
|
+
stored = f"#{sid}"
|
|
70
|
+
self._bytes_estimate += len(text)
|
|
71
|
+
else:
|
|
72
|
+
stored = f"#{sid}"
|
|
73
|
+
self._bytes_estimate += len(stored)
|
|
74
|
+
return stored
|
|
75
|
+
|
|
76
|
+
def _commit(self) -> None:
|
|
77
|
+
"""Every commit in this class goes through here. Durable: each batch
|
|
78
|
+
is its own transaction, so a killed recorder leaves what it flushed.
|
|
79
|
+
Non-durable: nothing commits before close() -- one transaction."""
|
|
80
|
+
if self._durable:
|
|
81
|
+
self._conn.commit()
|
|
82
|
+
|
|
83
|
+
@property
|
|
84
|
+
def last_event_id(self) -> int:
|
|
85
|
+
return self._next_event - 1
|
|
86
|
+
|
|
87
|
+
def intern_code(self, file: str, qualname: str, firstlineno: int) -> int:
|
|
88
|
+
key = (file, qualname, firstlineno)
|
|
89
|
+
with self._lock:
|
|
90
|
+
cid = self._codes.get(key)
|
|
91
|
+
if cid is None:
|
|
92
|
+
cid = len(self._codes) + 1
|
|
93
|
+
self._codes[key] = cid
|
|
94
|
+
self._new_codes.append((cid, file, qualname, firstlineno))
|
|
95
|
+
return cid
|
|
96
|
+
|
|
97
|
+
def add_event(self, ts_ns, thread_id, kind, frame_id, code_id, line,
|
|
98
|
+
payload: dict | None, task_id: int | None = None) -> int:
|
|
99
|
+
p = (None if payload is None
|
|
100
|
+
else json.dumps(payload, separators=(",", ":"), default=repr))
|
|
101
|
+
with self._lock:
|
|
102
|
+
eid = self._next_event
|
|
103
|
+
self._next_event += 1
|
|
104
|
+
p = self._intern_locked(p)
|
|
105
|
+
self._events.append(
|
|
106
|
+
(eid, ts_ns, thread_id, kind, frame_id, code_id, line, p,
|
|
107
|
+
task_id))
|
|
108
|
+
if len(self._events) >= self._batch:
|
|
109
|
+
self._flush_locked()
|
|
110
|
+
return eid
|
|
111
|
+
|
|
112
|
+
def add_task(self, task_id: int, name: str | None, thread_id: int) -> None:
|
|
113
|
+
"""One row per asyncio task, written when its serial is minted."""
|
|
114
|
+
with self._lock:
|
|
115
|
+
self._tasks.append((task_id, name, thread_id))
|
|
116
|
+
|
|
117
|
+
def open_frame(self, parent_id, code_id, call_event_id, depth,
|
|
118
|
+
thread_id, kind: str = "function") -> int:
|
|
119
|
+
with self._lock:
|
|
120
|
+
fid = self._next_frame
|
|
121
|
+
self._next_frame += 1
|
|
122
|
+
self._frames.append(
|
|
123
|
+
(fid, parent_id, code_id, call_event_id, depth, thread_id, kind))
|
|
124
|
+
return fid
|
|
125
|
+
|
|
126
|
+
def close_frame(self, frame_id, return_event_id=None, closed_by="return",
|
|
127
|
+
unwind_exc: dict | None = None) -> None:
|
|
128
|
+
exc = (None if unwind_exc is None
|
|
129
|
+
else json.dumps(unwind_exc, separators=(",", ":")))
|
|
130
|
+
with self._lock:
|
|
131
|
+
self._closes.append((return_event_id, closed_by, exc, frame_id))
|
|
132
|
+
|
|
133
|
+
def add_output(self, after_event_id, stream, data) -> None:
|
|
134
|
+
with self._lock:
|
|
135
|
+
self._outputs.append((after_event_id, stream, data))
|
|
136
|
+
|
|
137
|
+
def interned_files(self) -> list[str]:
|
|
138
|
+
"""Distinct source files this trace has interned code from.
|
|
139
|
+
|
|
140
|
+
Read from the in-memory intern table rather than the database
|
|
141
|
+
because callers need it during finalize, while codes may still be
|
|
142
|
+
sitting unflushed in `_new_codes`.
|
|
143
|
+
"""
|
|
144
|
+
with self._lock:
|
|
145
|
+
return sorted({file for file, _qual, _line in self._codes})
|
|
146
|
+
|
|
147
|
+
def set_meta(self, key, value) -> None:
|
|
148
|
+
with self._lock:
|
|
149
|
+
db.set_meta(self._conn, key, value)
|
|
150
|
+
self._commit()
|
|
151
|
+
|
|
152
|
+
def write_fingerprint(self, thread_id, hexdigest, count) -> None:
|
|
153
|
+
with self._lock:
|
|
154
|
+
self._conn.execute(
|
|
155
|
+
"INSERT OR REPLACE INTO fingerprints "
|
|
156
|
+
"(thread_id, hash, n_events) VALUES (?, ?, ?)",
|
|
157
|
+
(thread_id, hexdigest, count))
|
|
158
|
+
self._commit()
|
|
159
|
+
|
|
160
|
+
def write_task_fingerprints(self, rows) -> None:
|
|
161
|
+
"""Every task's row -- `(task_id, hexdigest, count)` -- in ONE
|
|
162
|
+
transaction.
|
|
163
|
+
|
|
164
|
+
One row per minted task serial; the name rides along from the
|
|
165
|
+
`tasks` row so the multiset comparison can read (name, hash) from
|
|
166
|
+
one table. A task whose name could not be read keeps NULL.
|
|
167
|
+
|
|
168
|
+
The buffer is flushed FIRST because that `tasks` row may still be
|
|
169
|
+
sitting in it: `INSERT ... SELECT` over a table that does not yet
|
|
170
|
+
hold the row inserts nothing at all, silently, and the caller (the
|
|
171
|
+
tracer's uninstall pass) has no way to tell. Under the CLI's
|
|
172
|
+
batch=512 that is the ordinary case for any async program short
|
|
173
|
+
enough not to have flushed on its own -- which is to say, most of
|
|
174
|
+
them: every task fingerprint was dropped and `diff` saw a run with
|
|
175
|
+
no tasks in it.
|
|
176
|
+
|
|
177
|
+
The flush does not commit, because it does not need to: a
|
|
178
|
+
connection sees its own uncommitted writes, so the SELECT below
|
|
179
|
+
reads the `tasks` rows this flush just inserted. What that buys is
|
|
180
|
+
the whole pass being one transaction -- one fsync for the run
|
|
181
|
+
rather than two per task. This is written from `uninstall`, after
|
|
182
|
+
the recorded program has already finished, so per-row commits
|
|
183
|
+
charged their cost to a process that looked done. Measured on this
|
|
184
|
+
box (ext4, 1,000 rows, best of five, three interleaved series):
|
|
185
|
+
1.3-3.2 ms per task the old way, 4.6-5.4 us per task this way. End
|
|
186
|
+
to end, recording a 2,000-task program went from 2.18 s to 0.35 s
|
|
187
|
+
of wall clock.
|
|
188
|
+
"""
|
|
189
|
+
rows = list(rows)
|
|
190
|
+
if not rows:
|
|
191
|
+
return
|
|
192
|
+
with self._lock:
|
|
193
|
+
self._flush_locked(commit=False)
|
|
194
|
+
self._conn.executemany(
|
|
195
|
+
"INSERT OR REPLACE INTO task_fingerprints "
|
|
196
|
+
"(task_id, name, hash, n_events) "
|
|
197
|
+
"SELECT ?, name, ?, ? FROM tasks WHERE id = ?",
|
|
198
|
+
[(task_id, hexdigest, count, task_id)
|
|
199
|
+
for task_id, hexdigest, count in rows])
|
|
200
|
+
self._commit()
|
|
201
|
+
|
|
202
|
+
def write_task_fingerprint(self, task_id, hexdigest, count) -> None:
|
|
203
|
+
"""One task's row. Kept as its own entry point -- callers with a
|
|
204
|
+
single task read better for it -- and routed through the bulk path
|
|
205
|
+
so there is one implementation of what a row means."""
|
|
206
|
+
self.write_task_fingerprints([(task_id, hexdigest, count)])
|
|
207
|
+
|
|
208
|
+
def _flush_locked(self, commit: bool = True) -> None:
|
|
209
|
+
c = self._conn
|
|
210
|
+
if self._string_rows:
|
|
211
|
+
if not self._strings_ready:
|
|
212
|
+
c.execute(
|
|
213
|
+
"CREATE TABLE IF NOT EXISTS strings ("
|
|
214
|
+
"id INTEGER PRIMARY KEY, body TEXT NOT NULL)")
|
|
215
|
+
self._strings_ready = True
|
|
216
|
+
c.executemany("INSERT INTO strings (id, body) VALUES (?, ?)",
|
|
217
|
+
self._string_rows)
|
|
218
|
+
self._string_rows.clear()
|
|
219
|
+
if self._new_codes:
|
|
220
|
+
c.executemany("INSERT INTO code_objects VALUES (?, ?, ?, ?)",
|
|
221
|
+
self._new_codes)
|
|
222
|
+
self._new_codes.clear()
|
|
223
|
+
if self._frames:
|
|
224
|
+
c.executemany(
|
|
225
|
+
"INSERT INTO frames (id, parent_id, code_id, call_event_id, "
|
|
226
|
+
"depth, thread_id, kind) VALUES (?, ?, ?, ?, ?, ?, ?)",
|
|
227
|
+
self._frames)
|
|
228
|
+
self._frames.clear()
|
|
229
|
+
if self._events:
|
|
230
|
+
c.executemany(
|
|
231
|
+
"INSERT INTO events (id, ts_ns, thread_id, kind, frame_id, "
|
|
232
|
+
"code_id, line, payload, task_id) VALUES "
|
|
233
|
+
"(?, ?, ?, ?, ?, ?, ?, ?, ?)", self._events)
|
|
234
|
+
self._events.clear()
|
|
235
|
+
if self._tasks:
|
|
236
|
+
c.executemany(
|
|
237
|
+
"INSERT INTO tasks (id, name, thread_id) VALUES (?, ?, ?)",
|
|
238
|
+
self._tasks)
|
|
239
|
+
self._tasks.clear()
|
|
240
|
+
if self._closes:
|
|
241
|
+
c.executemany(
|
|
242
|
+
"UPDATE frames SET return_event_id = ?, closed_by = ?, "
|
|
243
|
+
"unwind_exc = ? WHERE id = ?", self._closes)
|
|
244
|
+
self._closes.clear()
|
|
245
|
+
if self._outputs:
|
|
246
|
+
c.executemany(
|
|
247
|
+
"INSERT INTO output (after_event_id, stream, data) "
|
|
248
|
+
"VALUES (?, ?, ?)", self._outputs)
|
|
249
|
+
self._outputs.clear()
|
|
250
|
+
# `commit=False` is for a caller that has more to write in the SAME
|
|
251
|
+
# transaction (see `write_task_fingerprints`); every other caller
|
|
252
|
+
# takes the default, because an unflushed batch left uncommitted is
|
|
253
|
+
# a batch an unclean death loses -- in the durable mode, which is the
|
|
254
|
+
# mode where such a death leaves anything worth having. `_commit()`
|
|
255
|
+
# is where the two modes part.
|
|
256
|
+
if commit:
|
|
257
|
+
self._commit()
|
|
258
|
+
|
|
259
|
+
def flush(self) -> None:
|
|
260
|
+
with self._lock:
|
|
261
|
+
self._flush_locked()
|
|
262
|
+
|
|
263
|
+
def close(self) -> None:
|
|
264
|
+
with self._lock:
|
|
265
|
+
self._flush_locked()
|
|
266
|
+
# The one commit both modes make, and the only one that is not
|
|
267
|
+
# `_commit()`: in the non-durable mode this is the transaction.
|
|
268
|
+
self._conn.commit()
|
|
269
|
+
self._conn.close()
|
|
270
|
+
|
|
271
|
+
def discard(self) -> None:
|
|
272
|
+
"""Let go of a trace that will not be finished: roll back, close.
|
|
273
|
+
|
|
274
|
+
`close()`'s counterpart for a build that failed. In the non-durable
|
|
275
|
+
mode that commit IS the whole trace, and committing it would
|
|
276
|
+
checkpoint a full-size WAL into a file the caller is about to
|
|
277
|
+
unlink -- every byte of it written for nobody. The rollback drops
|
|
278
|
+
the open transaction instead; nothing buffered is flushed first,
|
|
279
|
+
because a row nobody will read is not worth writing.
|
|
280
|
+
"""
|
|
281
|
+
with self._lock:
|
|
282
|
+
self._conn.rollback()
|
|
283
|
+
self._conn.close()
|
sensorium/ts/__init__.py
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""Reading what the TypeScript recorder wrote.
|
|
2
|
+
|
|
3
|
+
The recorder itself is `typescript/` -- a transform, a runtime and the
|
|
4
|
+
harness wiring, all of it JavaScript. What it leaves behind is a directory
|
|
5
|
+
of JSONL spools, one per container, plus the driver's own record of the
|
|
6
|
+
invocation. This package is the other half: `spool` reads the wire,
|
|
7
|
+
`ingest` writes format-4 traces through the store's own `TraceWriter` so
|
|
8
|
+
the schema has exactly one home, and `cli` is the `sensorium ts`
|
|
9
|
+
subcommand.
|
|
10
|
+
"""
|