sensorium-trace 0.18.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sensorium/__init__.py +0 -0
- sensorium/__main__.py +4 -0
- sensorium/_c/rt/sensorium_c_rt.c +1446 -0
- sensorium/_typescript/package-lock.json +68 -0
- sensorium/_typescript/package.json +39 -0
- sensorium/_typescript/src/bindings.mjs +792 -0
- sensorium/_typescript/src/cap.mjs +227 -0
- sensorium/_typescript/src/capabilities.mjs +40 -0
- sensorium/_typescript/src/dbg.mjs +348 -0
- sensorium/_typescript/src/escape.mjs +351 -0
- sensorium/_typescript/src/focus.mjs +135 -0
- sensorium/_typescript/src/hook.mjs +127 -0
- sensorium/_typescript/src/index.mjs +1 -0
- sensorium/_typescript/src/naming.mjs +63 -0
- sensorium/_typescript/src/positions.mjs +37 -0
- sensorium/_typescript/src/probe.mjs +246 -0
- sensorium/_typescript/src/qualname.mjs +236 -0
- sensorium/_typescript/src/redact.mjs +711 -0
- sensorium/_typescript/src/register.mjs +15 -0
- sensorium/_typescript/src/resolve.mjs +260 -0
- sensorium/_typescript/src/rt.mjs +798 -0
- sensorium/_typescript/src/setup.mjs +11 -0
- sensorium/_typescript/src/tally.mjs +164 -0
- sensorium/_typescript/src/tasks.mjs +175 -0
- sensorium/_typescript/src/transform.mjs +800 -0
- sensorium/_typescript/src/vite.mjs +99 -0
- sensorium/c/__init__.py +8 -0
- sensorium/c/cc.py +612 -0
- sensorium/c/cli.py +60 -0
- sensorium/c/convert.py +669 -0
- sensorium/c/decode.py +300 -0
- sensorium/c/driver.py +365 -0
- sensorium/c/dwarf.py +706 -0
- sensorium/c/format.py +271 -0
- sensorium/c/pkg.py +187 -0
- sensorium/c/plan.py +277 -0
- sensorium/cli.py +140 -0
- sensorium/driver.py +47 -0
- sensorium/exit.py +33 -0
- sensorium/invocations.py +83 -0
- sensorium/limits.py +253 -0
- sensorium/mcp/__init__.py +0 -0
- sensorium/mcp/audit.py +162 -0
- sensorium/mcp/child.py +240 -0
- sensorium/mcp/cmd.py +101 -0
- sensorium/mcp/jsonrpc.py +172 -0
- sensorium/mcp/result.py +187 -0
- sensorium/mcp/schema.py +390 -0
- sensorium/mcp/server.py +493 -0
- sensorium/mcp/tools.py +299 -0
- sensorium/paths.py +80 -0
- sensorium/query/__init__.py +0 -0
- sensorium/query/caps.py +150 -0
- sensorium/query/dbg_dialects.py +64 -0
- sensorium/query/diff_cmd.py +748 -0
- sensorium/query/diff_notes.py +117 -0
- sensorium/query/exceptions_c.py +263 -0
- sensorium/query/exceptions_cmd.py +788 -0
- sensorium/query/exceptions_group.py +571 -0
- sensorium/query/exceptions_invocation.py +487 -0
- sensorium/query/exceptions_rust.py +683 -0
- sensorium/query/exceptions_typescript.py +688 -0
- sensorium/query/expr.py +606 -0
- sensorium/query/failures_cmd.py +306 -0
- sensorium/query/flow_cmd.py +778 -0
- sensorium/query/flow_report.py +128 -0
- sensorium/query/flow_values.py +195 -0
- sensorium/query/fmt.py +280 -0
- sensorium/query/frame_cmd.py +237 -0
- sensorium/query/grep_cmd.py +156 -0
- sensorium/query/help.py +18 -0
- sensorium/query/info_cmd.py +527 -0
- sensorium/query/info_rust.py +214 -0
- sensorium/query/info_typescript.py +261 -0
- sensorium/query/js_inspect.py +394 -0
- sensorium/query/moves.py +254 -0
- sensorium/query/redact_cmd.py +304 -0
- sensorium/query/refocus_cmd.py +661 -0
- sensorium/query/refocus_env.py +493 -0
- sensorium/query/refocus_facts.py +269 -0
- sensorium/query/refocus_licence.py +187 -0
- sensorium/query/refocus_report.py +327 -0
- sensorium/query/refocus_rust.py +537 -0
- sensorium/query/refocus_threads.py +197 -0
- sensorium/query/refocus_typescript.py +756 -0
- sensorium/query/refocus_world.py +666 -0
- sensorium/query/runs_cmd.py +213 -0
- sensorium/query/rust_debug.py +176 -0
- sensorium/query/sites.py +171 -0
- sensorium/query/tree_cmd.py +511 -0
- sensorium/query/tree_threads.py +168 -0
- sensorium/query/vocab.py +723 -0
- sensorium/query/watch_cmd.py +657 -0
- sensorium/record/__init__.py +0 -0
- sensorium/record/boot.py +797 -0
- sensorium/record/boot_io.py +141 -0
- sensorium/record/budget.py +135 -0
- sensorium/record/capture.py +433 -0
- sensorium/record/children.py +755 -0
- sensorium/record/children_merge.py +179 -0
- sensorium/record/elide.py +55 -0
- sensorium/record/fingerprint.py +26 -0
- sensorium/record/thread_starts.py +261 -0
- sensorium/record/tracer.py +798 -0
- sensorium/record/tracer_exc.py +199 -0
- sensorium/record/tracer_frames.py +618 -0
- sensorium/redact.py +304 -0
- sensorium/redact_content.py +156 -0
- sensorium/redact_key.py +317 -0
- sensorium/redact_store.py +502 -0
- sensorium/redact_values.py +433 -0
- sensorium/store/__init__.py +0 -0
- sensorium/store/db.py +284 -0
- sensorium/store/gc.py +360 -0
- sensorium/store/payloads.py +69 -0
- sensorium/store/reader.py +438 -0
- sensorium/store/writer.py +283 -0
- sensorium/ts/__init__.py +10 -0
- sensorium/ts/build.py +783 -0
- sensorium/ts/cli.py +130 -0
- sensorium/ts/driver.py +472 -0
- sensorium/ts/focus.py +146 -0
- sensorium/ts/harness.py +314 -0
- sensorium/ts/ingest.py +380 -0
- sensorium/ts/invocation.py +252 -0
- sensorium/ts/pkg.py +167 -0
- sensorium/ts/redaction.py +64 -0
- sensorium/ts/spool.py +168 -0
- sensorium/ts/truncation.py +81 -0
- sensorium/ts/wrapper.py +255 -0
- sensorium_trace-0.18.2.dist-info/METADATA +245 -0
- sensorium_trace-0.18.2.dist-info/RECORD +135 -0
- sensorium_trace-0.18.2.dist-info/WHEEL +4 -0
- sensorium_trace-0.18.2.dist-info/entry_points.txt +3 -0
- sensorium_trace-0.18.2.dist-info/licenses/LICENSE +21 -0
sensorium/ts/ingest.py
ADDED
|
@@ -0,0 +1,380 @@
|
|
|
1
|
+
"""A spool directory in, one trace per container out.
|
|
2
|
+
|
|
3
|
+
`ingest` is deterministic and re-runnable over a directory the driver left
|
|
4
|
+
behind, so a driver killed between the harness's exit and the conversion
|
|
5
|
+
leaves nothing that cannot be finished by hand. What makes that safe is the
|
|
6
|
+
marker it writes at the end (`ingested.json`): a second ingest of one
|
|
7
|
+
recording would mint a second set of run ids for the same containers, and
|
|
8
|
+
the store would hold each of them twice with nothing to say which was
|
|
9
|
+
which. So a directory that carries the marker is REFUSED, by name.
|
|
10
|
+
|
|
11
|
+
Spools convert in parallel, one worker per spool, over a `spawn` context.
|
|
12
|
+
`fork` would hand each worker the parent's threads, its open sqlite
|
|
13
|
+
handles and its signal dispositions; nothing here needs any of that, and a
|
|
14
|
+
converter that inherits a database handle it did not open is a converter
|
|
15
|
+
one bug away from writing into somebody else's file.
|
|
16
|
+
|
|
17
|
+
The pool is a `ProcessPoolExecutor` and not a `multiprocessing.Pool`,
|
|
18
|
+
because a worker can die of a signal it never gets to handle -- the OOM
|
|
19
|
+
killer is the ordinary way -- and `Pool.imap` waits for a result that is
|
|
20
|
+
never coming. The executor raises `BrokenProcessPool` instead, and a
|
|
21
|
+
refusal is a thing a caller can read.
|
|
22
|
+
"""
|
|
23
|
+
import concurrent.futures
|
|
24
|
+
import json
|
|
25
|
+
import multiprocessing
|
|
26
|
+
import os
|
|
27
|
+
import secrets
|
|
28
|
+
from dataclasses import dataclass, field
|
|
29
|
+
from pathlib import Path
|
|
30
|
+
|
|
31
|
+
from sensorium.ts import build, invocation, spool
|
|
32
|
+
|
|
33
|
+
#: The multiprocessing start method. Named so a test can hold it: see the
|
|
34
|
+
#: module docstring for why it is not `fork`.
|
|
35
|
+
CONTEXT = "spawn"
|
|
36
|
+
|
|
37
|
+
MARKER = "ingested.json"
|
|
38
|
+
MANIFEST_DIR = "manifests"
|
|
39
|
+
|
|
40
|
+
#: The invocation-wide tally, written by the Vite plugin. vitest transforms
|
|
41
|
+
#: in ONE process for the whole run, so its count is every container's.
|
|
42
|
+
TALLY = "_tally.json"
|
|
43
|
+
|
|
44
|
+
#: One container's own, `_tally-<pid>.json`, written by the `node --test`
|
|
45
|
+
#: loader hook. That harness runs one child process per test file, each with
|
|
46
|
+
#: its own hook thread and its own transform, so the count on such a trace is
|
|
47
|
+
#: that container's -- and one shared name would be N writers over one number.
|
|
48
|
+
TALLY_PREFIX = "_tally-"
|
|
49
|
+
|
|
50
|
+
#: How many times a run id is re-minted before the converter gives up. Two
|
|
51
|
+
#: containers of one invocation start in the same second, so the stamp
|
|
52
|
+
#: collides by design and only the token separates them; a collision is a
|
|
53
|
+
#: 1-in-16-million event and a second draw settles it.
|
|
54
|
+
MINT_TRIES = 8
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class IngestError(Exception):
|
|
58
|
+
"""The call cannot proceed: no such directory, no invocation record, or
|
|
59
|
+
a directory that has already been converted. Always names the file."""
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclass(frozen=True)
|
|
63
|
+
class Summary:
|
|
64
|
+
"""One spool's outcome.
|
|
65
|
+
|
|
66
|
+
A refusal is a Summary too, with `refused` naming why and the other
|
|
67
|
+
fields left at their empty values. One list of these is the whole
|
|
68
|
+
result of an ingest, so a caller never has to zip two sequences back
|
|
69
|
+
together to find out what happened to a given file.
|
|
70
|
+
"""
|
|
71
|
+
|
|
72
|
+
file: str
|
|
73
|
+
run_id: str | None = None
|
|
74
|
+
pid: int | None = None
|
|
75
|
+
test_file: str | None = None
|
|
76
|
+
events: int = 0
|
|
77
|
+
tasks: int = 0
|
|
78
|
+
incomplete: bool = False
|
|
79
|
+
refused: str | None = None
|
|
80
|
+
|
|
81
|
+
def line(self) -> str:
|
|
82
|
+
"""The one line an ingest prints per spool (P7). Two spaces between
|
|
83
|
+
fields, because `corpus.run_corpus.RUN_LINE` reads the first two of
|
|
84
|
+
them and the corpus learns what it just recorded from this string."""
|
|
85
|
+
if self.refused is not None:
|
|
86
|
+
return f"refused: {self.file} {self.refused}"
|
|
87
|
+
return (f"run: {self.run_id} pid: {self.pid} "
|
|
88
|
+
f"file: {self.test_file or '-'} events: {self.events} "
|
|
89
|
+
f"tasks: {self.tasks}")
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
@dataclass
|
|
93
|
+
class Ingested:
|
|
94
|
+
"""What one ingest did, as the marker records it."""
|
|
95
|
+
|
|
96
|
+
summaries: list[Summary] = field(default_factory=list)
|
|
97
|
+
|
|
98
|
+
@property
|
|
99
|
+
def refused(self) -> list[str]:
|
|
100
|
+
return [s.file for s in self.summaries if s.refused is not None]
|
|
101
|
+
|
|
102
|
+
@property
|
|
103
|
+
def run_ids(self) -> list[str]:
|
|
104
|
+
return [s.run_id for s in self.summaries if s.refused is None]
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
# --------------------------------------------------------------------------
|
|
108
|
+
# one spool
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def convert(spool_path, invocation_data: dict, harness_data: dict | None,
|
|
112
|
+
store_dir, tally: dict | None = None) -> Summary:
|
|
113
|
+
"""Convert one spool into one trace under `store_dir`.
|
|
114
|
+
|
|
115
|
+
`store_dir` is the trace ROOT -- the directory `$SENSORIUM_DIR` names --
|
|
116
|
+
and the trace lands in its `traces/` subdirectory, exactly where
|
|
117
|
+
`paths.find_trace` looks. The two records are passed as plain dicts
|
|
118
|
+
rather than as dataclasses because this function is the body of a
|
|
119
|
+
`spawn`ed worker and everything it is handed is pickled to get there.
|
|
120
|
+
|
|
121
|
+
The trace is written under a dotted temporary name and renamed into
|
|
122
|
+
place at the end, so a conversion that refuses half way through leaves
|
|
123
|
+
no `.db` behind for `runs` to list.
|
|
124
|
+
"""
|
|
125
|
+
spool_path = Path(spool_path)
|
|
126
|
+
inv = invocation.Invocation.from_json(invocation_data)
|
|
127
|
+
harness = (None if harness_data is None
|
|
128
|
+
else invocation.HarnessExit.from_json(harness_data))
|
|
129
|
+
sp = spool.read(spool_path)
|
|
130
|
+
|
|
131
|
+
traces = Path(store_dir) / "traces"
|
|
132
|
+
# `paths.traces_dir()`'s rule, spelled here because this worker is handed
|
|
133
|
+
# a `store_dir` and never reads the environment `paths` resolves: one
|
|
134
|
+
# level at a time, because `parents=True` gives every parent it creates
|
|
135
|
+
# the DEFAULT mode and the store root is where `redaction.key` lives.
|
|
136
|
+
Path(store_dir).mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
137
|
+
traces.mkdir(exist_ok=True, mode=0o700)
|
|
138
|
+
run_id, tmp = _reserve(traces, sp.boot["startTs"])
|
|
139
|
+
builder = build.Builder(sp, inv, harness, tally, tmp, run_id)
|
|
140
|
+
try:
|
|
141
|
+
meta = builder.build()
|
|
142
|
+
except BaseException:
|
|
143
|
+
builder.abort()
|
|
144
|
+
tmp.unlink(missing_ok=True)
|
|
145
|
+
raise
|
|
146
|
+
os.replace(tmp, traces / f"{run_id}.db")
|
|
147
|
+
return Summary(file=spool_path.name, run_id=run_id, pid=sp.boot["pid"],
|
|
148
|
+
test_file=meta.get("test_file"), events=builder.events,
|
|
149
|
+
tasks=len(builder.tasks),
|
|
150
|
+
incomplete=meta["incomplete"])
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _reserve(traces: Path, start_ts: float) -> tuple[str, Path]:
|
|
154
|
+
"""A run id no trace in this store holds, CLAIMED before it is returned.
|
|
155
|
+
|
|
156
|
+
The temporary file is created here, `O_EXCL`, and that creation IS the
|
|
157
|
+
reservation: two spawn workers converting two containers of one
|
|
158
|
+
invocation mint from the same second, so they collide on the stamp by
|
|
159
|
+
design and only the token separates them -- and a pair that looked with
|
|
160
|
+
`exists()` and then wrote could both have looked before either wrote.
|
|
161
|
+
`O_EXCL` is one atomic step and the loser simply draws again.
|
|
162
|
+
|
|
163
|
+
The empty file is handed straight to `TraceWriter`; sqlite opens a
|
|
164
|
+
zero-length file as a new database and lays the schema into it, which is
|
|
165
|
+
exactly what it would have done had it created the file itself.
|
|
166
|
+
"""
|
|
167
|
+
for _ in range(MINT_TRIES):
|
|
168
|
+
run_id = build.mint_run_id(start_ts, secrets.token_hex(3))
|
|
169
|
+
tmp = traces / f".{run_id}.db.tmp"
|
|
170
|
+
if (traces / f"{run_id}.db").exists():
|
|
171
|
+
continue
|
|
172
|
+
try:
|
|
173
|
+
os.close(os.open(tmp, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600))
|
|
174
|
+
except FileExistsError:
|
|
175
|
+
continue
|
|
176
|
+
return run_id, tmp
|
|
177
|
+
raise IngestError(f"could not mint a free run id in {traces} after "
|
|
178
|
+
f"{MINT_TRIES} tries")
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def _worker(job: tuple) -> Summary:
|
|
182
|
+
"""One spool, in its own process. Refusals come back as values, not as
|
|
183
|
+
exceptions: a spool this converter cannot read must not stop the ones
|
|
184
|
+
beside it, and the run's exit status is decided once, at the end, over
|
|
185
|
+
the whole list."""
|
|
186
|
+
path, inv, harness, store_dir, tally = job
|
|
187
|
+
try:
|
|
188
|
+
return convert(path, inv, harness, store_dir, tally)
|
|
189
|
+
except spool.SpoolError as e:
|
|
190
|
+
return Summary(file=Path(path).name, refused=str(e))
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
# --------------------------------------------------------------------------
|
|
194
|
+
# the directory
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def ingest_dir(spool_dir, store_dir, jobs: int | None = None) -> list[Summary]:
|
|
198
|
+
"""Every spool in `spool_dir`, converted into `store_dir`.
|
|
199
|
+
|
|
200
|
+
Spools are converted in NAME order and the summaries come back in that
|
|
201
|
+
order, so two ingests of one directory print their lines the same way
|
|
202
|
+
round however the pool scheduled them.
|
|
203
|
+
"""
|
|
204
|
+
spool_dir = Path(spool_dir)
|
|
205
|
+
if not spool_dir.is_dir():
|
|
206
|
+
raise IngestError(f"{spool_dir} is not a directory")
|
|
207
|
+
marker = spool_dir / MARKER
|
|
208
|
+
if marker.exists():
|
|
209
|
+
raise IngestError(
|
|
210
|
+
f"{marker} already exists: this spool directory has been "
|
|
211
|
+
"converted, and converting it again would mint a second run id "
|
|
212
|
+
"for every container in it")
|
|
213
|
+
|
|
214
|
+
inv = invocation.read_invocation(spool_dir)
|
|
215
|
+
harness = invocation.read_harness(spool_dir)
|
|
216
|
+
shared = _tally(spool_dir)
|
|
217
|
+
spools = sorted(spool_dir.glob("*.jsonl"))
|
|
218
|
+
if not spools:
|
|
219
|
+
raise IngestError(_no_spools(spool_dir))
|
|
220
|
+
|
|
221
|
+
jobs = max(1, min(jobs or os.cpu_count() or 1, len(spools)))
|
|
222
|
+
payload = (inv.to_json(),
|
|
223
|
+
None if harness is None else harness.to_json(),
|
|
224
|
+
str(store_dir))
|
|
225
|
+
work = [(str(p), *payload, _tally_for(spool_dir, p, shared))
|
|
226
|
+
for p in spools]
|
|
227
|
+
summaries: list[Summary] = []
|
|
228
|
+
try:
|
|
229
|
+
_map(jobs, work, summaries)
|
|
230
|
+
except Exception as e:
|
|
231
|
+
# Whatever this was, traces are already in the store and P6's marker
|
|
232
|
+
# is the only record of which ones. Writing it here is what keeps a
|
|
233
|
+
# re-ingest from minting a SECOND run id for a container that already
|
|
234
|
+
# converted -- the failure mode the marker exists to stop does not
|
|
235
|
+
# care why the run stopped.
|
|
236
|
+
_write_marker(marker, summaries, f"{type(e).__name__}: {e}")
|
|
237
|
+
raise IngestError(
|
|
238
|
+
f"{type(e).__name__}: {e} -- {len(summaries)} spool(s) converted "
|
|
239
|
+
f"before it; {marker} records them") from e
|
|
240
|
+
_write_marker(marker, summaries, None)
|
|
241
|
+
return summaries
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def _write_marker(marker: Path, summaries: list[Summary],
|
|
245
|
+
error: str | None) -> None:
|
|
246
|
+
"""P6's record of what this directory converted to.
|
|
247
|
+
|
|
248
|
+
`error` is present only on a run that did not finish, and names what
|
|
249
|
+
stopped it: a marker that said nothing about it would read as a complete
|
|
250
|
+
ingest of a directory only part of which was converted.
|
|
251
|
+
"""
|
|
252
|
+
result = Ingested(summaries=summaries)
|
|
253
|
+
body: dict = {"run_ids": result.run_ids, "refused": result.refused}
|
|
254
|
+
if error is not None:
|
|
255
|
+
body["error"] = error
|
|
256
|
+
invocation.write_record(marker, json.dumps(body, indent=2) + "\n")
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def _map(jobs: int, work: list[tuple], out: list[Summary]) -> None:
|
|
260
|
+
"""The pool, or no pool at all for a single job, appending to `out`.
|
|
261
|
+
|
|
262
|
+
The accumulator is the CALLER's, and results are taken one at a time
|
|
263
|
+
(`map`, in order), so a worker that dies of something this converter
|
|
264
|
+
did not anticipate does not also take the record of what had already
|
|
265
|
+
converted: a call that returned a list or nothing at all is what would
|
|
266
|
+
leave the marker unwritable.
|
|
267
|
+
|
|
268
|
+
`ProcessPoolExecutor` and not `multiprocessing.Pool` (R42). A worker
|
|
269
|
+
SIGKILLed -- by the OOM killer, by a `kill -9`, by any signal it
|
|
270
|
+
cannot catch -- never puts a result on the queue, and `Pool.imap`
|
|
271
|
+
waits for one FOREVER: the driver hung with no output and nothing to
|
|
272
|
+
interrupt but itself. The executor notices its child is gone and
|
|
273
|
+
raises `BrokenProcessPool`, which the caller's `except Exception`
|
|
274
|
+
turns into the marker plus a named refusal at exit 2. Measured: 60 s
|
|
275
|
+
of nothing under `imap`, immediate under the executor.
|
|
276
|
+
|
|
277
|
+
One spool through a process pool costs a whole interpreter start to
|
|
278
|
+
save nothing, and a caller who asked for one job usually wants one
|
|
279
|
+
process -- a debugger's stack, a profiler's numbers.
|
|
280
|
+
"""
|
|
281
|
+
if jobs == 1:
|
|
282
|
+
for job in work:
|
|
283
|
+
out.append(_worker(job))
|
|
284
|
+
return
|
|
285
|
+
ctx = multiprocessing.get_context(CONTEXT)
|
|
286
|
+
with concurrent.futures.ProcessPoolExecutor(jobs, mp_context=ctx) as ex:
|
|
287
|
+
for summary in ex.map(_worker, work):
|
|
288
|
+
out.append(summary)
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
#: What a directory with no spool in it says when its tallies explain
|
|
292
|
+
#: nothing: the run may have gone wrong anywhere between the harness and the
|
|
293
|
+
#: disk, and this converter does not guess which.
|
|
294
|
+
NOTHING_RECORDED = ("nothing was recorded, or the recorder wrote somewhere "
|
|
295
|
+
"else")
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def _no_spools(spool_dir: Path) -> str:
|
|
299
|
+
"""The refusal for a directory holding no spool at all (R45).
|
|
300
|
+
|
|
301
|
+
A CommonJS-only suite runs GREEN under `node --test` and records
|
|
302
|
+
nothing -- every file the hook met was excluded before it was parsed --
|
|
303
|
+
and `NOTHING_RECORDED` names neither that cause nor its fix. The
|
|
304
|
+
tallies beside the missing spools do: when every one of them counted
|
|
305
|
+
zero transforms and at least one exclusion, that IS what happened, and
|
|
306
|
+
the refusal says so by reason and by count.
|
|
307
|
+
|
|
308
|
+
Anything else keeps the old sentence. A run that transformed a file and
|
|
309
|
+
still wrote no spool went wrong somewhere no tally can see, and naming
|
|
310
|
+
CommonJS there would be a guess dressed as a diagnosis.
|
|
311
|
+
"""
|
|
312
|
+
tallies = [_read_tally(p) for p in _tally_files(spool_dir)]
|
|
313
|
+
counts: dict[str, int] = {}
|
|
314
|
+
for tally in tallies:
|
|
315
|
+
if tally.get("files_transformed") != 0 or not tally.get("excluded"):
|
|
316
|
+
counts = {}
|
|
317
|
+
break
|
|
318
|
+
for reason, n in tally["excluded"].items():
|
|
319
|
+
counts[reason] = counts.get(reason, 0) + n
|
|
320
|
+
if not counts:
|
|
321
|
+
return f"no spools in {spool_dir}: {NOTHING_RECORDED}"
|
|
322
|
+
reasons = ", ".join(f"{reason} x{n}" for reason, n in sorted(counts.items()))
|
|
323
|
+
line = (f"no spools in {spool_dir}: this run transformed 0 files and "
|
|
324
|
+
f"excluded {sum(counts.values())} ({reasons} across "
|
|
325
|
+
f"{len(tallies)} tallies)")
|
|
326
|
+
if "commonjs" in counts:
|
|
327
|
+
line += ("; this recorder instruments ES modules only, so a "
|
|
328
|
+
"CommonJS-only suite records nothing")
|
|
329
|
+
return line
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def _tally_files(spool_dir: Path) -> list[Path]:
|
|
333
|
+
"""Every tally an invocation left, the shared one first: what the two
|
|
334
|
+
harnesses wrote between them, whichever of them ran."""
|
|
335
|
+
manifests = spool_dir / MANIFEST_DIR
|
|
336
|
+
shared = [manifests / TALLY] if (manifests / TALLY).is_file() else []
|
|
337
|
+
return shared + sorted(manifests.glob(f"{TALLY_PREFIX}*.json"))
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def _tally(spool_dir: Path) -> dict:
|
|
341
|
+
"""The invocation's own count of what the transform covered, or `{}`.
|
|
342
|
+
|
|
343
|
+
Written by the Vite plugin on exit into `SENSORIUM_MANIFEST_DIR`, which
|
|
344
|
+
the driver points at `<spool>/manifests`. Absent when the harness never
|
|
345
|
+
transformed anything, or when the plugin could not write it -- and an
|
|
346
|
+
absent tally is written as no meta key at all, never as a zero, because
|
|
347
|
+
"nothing was transformed" and "nobody counted" are different facts.
|
|
348
|
+
"""
|
|
349
|
+
return _read_tally(spool_dir / MANIFEST_DIR / TALLY)
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def _tally_for(spool_dir: Path, spool: Path, shared: dict) -> dict:
|
|
353
|
+
"""This container's own count, falling back to the invocation's.
|
|
354
|
+
|
|
355
|
+
A spool is named `<pid>-<threadId>.jsonl`, and under `node --test` that
|
|
356
|
+
pid is also the pid of the process that did the transforming: the
|
|
357
|
+
harness runs one child per test file, so what a container was told
|
|
358
|
+
about is what that child transformed. vitest transforms once for the
|
|
359
|
+
whole run and writes the invocation-wide file instead, which every
|
|
360
|
+
container of it shares.
|
|
361
|
+
"""
|
|
362
|
+
pid = spool.name.split("-")[0]
|
|
363
|
+
if pid.isdigit():
|
|
364
|
+
own = _read_tally(spool_dir / MANIFEST_DIR
|
|
365
|
+
/ f"{TALLY_PREFIX}{pid}.json")
|
|
366
|
+
if own:
|
|
367
|
+
return own
|
|
368
|
+
return shared
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def _read_tally(path: Path) -> dict:
|
|
372
|
+
"""One tally file, or `{}`. Unreadable JSON is a refusal and not a
|
|
373
|
+
zero: a count nobody can read is not a count of nothing."""
|
|
374
|
+
if not path.is_file():
|
|
375
|
+
return {}
|
|
376
|
+
try:
|
|
377
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
378
|
+
except json.JSONDecodeError as e:
|
|
379
|
+
raise IngestError(f"{path} is not readable JSON: {e}") from None
|
|
380
|
+
return data if isinstance(data, dict) else {}
|
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
"""The driver's own record of one run, and the harness's ending.
|
|
2
|
+
|
|
3
|
+
Two small files sit beside the spools in a spool directory, and both are
|
|
4
|
+
written by the driver (`sensorium ts run`) and read by the converter
|
|
5
|
+
(`sensorium ts ingest`). They exist because a spool cannot say these
|
|
6
|
+
things: a container knows its own pid and its own environment, but not
|
|
7
|
+
which command produced it, not where the project root is, and not what the
|
|
8
|
+
harness exited with -- nobody inside the harness waits for the harness.
|
|
9
|
+
|
|
10
|
+
invocation.json written before the harness is spawned
|
|
11
|
+
harness.json written after it returns; ABSENT when the driver
|
|
12
|
+
died first, which is why every reader of it treats
|
|
13
|
+
absence as a fact and not as a zero
|
|
14
|
+
|
|
15
|
+
The dataclasses here are the contract between the two commands. They are
|
|
16
|
+
in this module rather than in either one so that neither owns it: `ingest`
|
|
17
|
+
imports `Invocation` to read, `run` imports it to write, and a field
|
|
18
|
+
either adds is a field the other sees.
|
|
19
|
+
"""
|
|
20
|
+
import hashlib
|
|
21
|
+
import json
|
|
22
|
+
import os
|
|
23
|
+
from collections.abc import Mapping
|
|
24
|
+
from dataclasses import MISSING, asdict, dataclass, field, fields
|
|
25
|
+
from pathlib import Path
|
|
26
|
+
|
|
27
|
+
#: The two harnesses this recorder wires. `vitest` runs the Vite plugin;
|
|
28
|
+
#: `node-test` runs the loader hook. Anything else is a driver refusal, not
|
|
29
|
+
#: something the converter has to have an opinion about.
|
|
30
|
+
HARNESSES = ("vitest", "node-test")
|
|
31
|
+
|
|
32
|
+
INVOCATION_FILE = "invocation.json"
|
|
33
|
+
HARNESS_FILE = "harness.json"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class InvocationError(Exception):
|
|
37
|
+
"""A spool directory's own record is missing or unreadable. Always
|
|
38
|
+
names the file: the caller's next move is to look at it."""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def write_record(path: Path, text: str) -> None:
|
|
42
|
+
"""Write one of a spool directory's records, 0600 AT CREATION.
|
|
43
|
+
|
|
44
|
+
`Path.write_text` opens 0666-under-the-umask, which is 0664 on a box
|
|
45
|
+
with the usual group-writable default. E16 part A measured what that
|
|
46
|
+
cost: `invocation.json`, `harness.json`, `ingested.json` and the
|
|
47
|
+
plugin's `manifests/*.json` were the only files beside a recording that
|
|
48
|
+
anybody on the box could read (H2, 2026-09-14). They hold the user's
|
|
49
|
+
argv, their project root, their target directory and the names of every
|
|
50
|
+
function in every file the transform touched.
|
|
51
|
+
|
|
52
|
+
At creation and never by a later `chmod`: a file created 0664 and
|
|
53
|
+
tightened a moment later is readable for that moment, and the moment is
|
|
54
|
+
exactly when it is being filled. `rust/cargo-sensorium/src/perms.rs`
|
|
55
|
+
takes the same position for the same store, and `store/db.py` and
|
|
56
|
+
`invocations.py` are this idiom's other two spellings.
|
|
57
|
+
|
|
58
|
+
The mode applies only to a file this CREATES. `O_TRUNC` re-writes an
|
|
59
|
+
existing record in place and leaves whatever mode its owner gave it --
|
|
60
|
+
`ingest` is re-runnable over a directory it has already converted, and
|
|
61
|
+
a person's own file is theirs.
|
|
62
|
+
|
|
63
|
+
HERE rather than in either caller: the driver writes two of these
|
|
64
|
+
records and the converter writes the third, and a mode spelled three
|
|
65
|
+
times is a mode that will be spelled two ways.
|
|
66
|
+
"""
|
|
67
|
+
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
|
68
|
+
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
|
69
|
+
f.write(text)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@dataclass(frozen=True)
|
|
73
|
+
class Invocation:
|
|
74
|
+
"""What the driver knew before it spawned anything.
|
|
75
|
+
|
|
76
|
+
`root` is the project directory every path in a trace is made relative
|
|
77
|
+
to; `cwd` is where the driver was standing, which is not necessarily
|
|
78
|
+
the same place. `recorder` is the fallback for a spool whose BOOT
|
|
79
|
+
record carries no `version` -- the driver read it out of the recorder's
|
|
80
|
+
own package, and a BOOT that names its writer wins over it (R5).
|
|
81
|
+
|
|
82
|
+
THREE COMMANDS, AND ONLY ONE OF THEM WAS TYPED (R26, 2026-09-09)
|
|
83
|
+
---------------------------------------------------------------
|
|
84
|
+
`command` is the tokens the user typed after `--`, before the driver
|
|
85
|
+
consumed anything from them and before it re-issued anything: `npx
|
|
86
|
+
vitest run src/fog --root packages/app`. `harness` is the KIND the
|
|
87
|
+
driver recognised (`vitest`, `node-test`) and `harness_args` is what
|
|
88
|
+
followed the harness token with `--root`/`--config` taken OUT, because
|
|
89
|
+
the driver re-issues those itself. The two are what the driver needs;
|
|
90
|
+
they are not a command, and joining them reconstructs one nobody typed
|
|
91
|
+
-- `node-test --test test/` names no program, and a `--root` the user
|
|
92
|
+
passed disappears from it. So the readers print `command`, and the
|
|
93
|
+
other two stay what they are for the machinery that uses them.
|
|
94
|
+
|
|
95
|
+
It holds the same tokens `argv` does. `argv` was written for the record
|
|
96
|
+
and is read by nothing; `command` is the field with a reader, and it is
|
|
97
|
+
LAST and defaulted because a spool directory written before it existed
|
|
98
|
+
is still a spool directory this converter must finish (`from_json`
|
|
99
|
+
reads its absence as `[]`, and `build` then writes no meta key at all).
|
|
100
|
+
|
|
101
|
+
`resolver_wall_s` is what resolving those specs cost. It is here
|
|
102
|
+
because `Resolution.wall` was measured and dropped, so an instrument
|
|
103
|
+
wanting the resolver's own time had to wrap the driver and time
|
|
104
|
+
everything around it too (`e12.sh` did). No reader prints it.
|
|
105
|
+
|
|
106
|
+
`focus` and `focus_matched` follow the same rule for the same reason.
|
|
107
|
+
`focus` is the specs AS TYPED, which is what a reader recognises;
|
|
108
|
+
`focus_matched` is `rel:qualname` for every function they selected,
|
|
109
|
+
sorted, which is what says whether the spec meant what its author
|
|
110
|
+
thought. The driver knows both before it spawns anything, because it
|
|
111
|
+
resolved them against the root to decide whether to spawn at all.
|
|
112
|
+
"""
|
|
113
|
+
|
|
114
|
+
invocation: str
|
|
115
|
+
harness: str
|
|
116
|
+
harness_args: list[str]
|
|
117
|
+
root: str
|
|
118
|
+
cwd: str
|
|
119
|
+
argv: list[str]
|
|
120
|
+
env_hash: str
|
|
121
|
+
start_ts: float
|
|
122
|
+
wrapper: str
|
|
123
|
+
vitest: str | None
|
|
124
|
+
node: str
|
|
125
|
+
driver_version: str
|
|
126
|
+
recorder: str
|
|
127
|
+
command: list[str] = field(default_factory=list)
|
|
128
|
+
focus: list[str] = field(default_factory=list)
|
|
129
|
+
focus_matched: list[str] = field(default_factory=list)
|
|
130
|
+
#: What resolving the focus COST, in seconds, or `None` where no focus
|
|
131
|
+
#: was given and the resolver never ran. Defaulted for the same reason
|
|
132
|
+
#: the two above are: a spool directory written before the field
|
|
133
|
+
#: existed is still one this converter must finish. `None` and not
|
|
134
|
+
#: `0.0` because an absent measurement is not a fast one.
|
|
135
|
+
resolver_wall_s: float | None = None
|
|
136
|
+
#: The run this recording re-runs one flag deeper, written by
|
|
137
|
+
#: `--refocus-of` and `None` where a person typed the command
|
|
138
|
+
#: themselves. Defaulted like its neighbours: a spool directory written
|
|
139
|
+
#: by a driver older than the flag lacks the key, and absence is what
|
|
140
|
+
#: the default is for.
|
|
141
|
+
refocus_of: str | None = None
|
|
142
|
+
|
|
143
|
+
def to_json(self) -> dict:
|
|
144
|
+
return asdict(self)
|
|
145
|
+
|
|
146
|
+
@classmethod
|
|
147
|
+
def from_json(cls, data: dict) -> "Invocation":
|
|
148
|
+
# A field with a DEFAULT is a field a spool directory written by an
|
|
149
|
+
# older driver can lack: absence is what the default is for. Every
|
|
150
|
+
# other one is required, and its absence names itself.
|
|
151
|
+
missing = [f.name for f in fields(cls)
|
|
152
|
+
if f.name not in data
|
|
153
|
+
and f.default is MISSING and f.default_factory is MISSING]
|
|
154
|
+
if missing:
|
|
155
|
+
raise InvocationError(
|
|
156
|
+
f"{INVOCATION_FILE} lacks {', '.join(missing)}")
|
|
157
|
+
return cls(**{f.name: data[f.name] for f in fields(cls)
|
|
158
|
+
if f.name in data})
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
@dataclass(frozen=True)
|
|
162
|
+
class HarnessExit:
|
|
163
|
+
"""What the driver witnessed when the harness returned.
|
|
164
|
+
|
|
165
|
+
`basis` is always `"waited"` -- the file exists only because somebody
|
|
166
|
+
waited -- and it is carried into the trace beside the status so no
|
|
167
|
+
reader has to remember which of the two exit facts it is holding
|
|
168
|
+
(TRACE-FORMAT section 4). `status` and `signal` are exclusive: a
|
|
169
|
+
process killed by a signal chose no status.
|
|
170
|
+
"""
|
|
171
|
+
|
|
172
|
+
status: int | None
|
|
173
|
+
signal: str | None
|
|
174
|
+
wall_start_ts: float
|
|
175
|
+
wall_end_ts: float
|
|
176
|
+
basis: str = "waited"
|
|
177
|
+
|
|
178
|
+
def to_json(self) -> dict:
|
|
179
|
+
return asdict(self)
|
|
180
|
+
|
|
181
|
+
@classmethod
|
|
182
|
+
def from_json(cls, data: dict) -> "HarnessExit":
|
|
183
|
+
missing = [f.name for f in fields(cls)
|
|
184
|
+
if f.name not in data and f.name != "basis"]
|
|
185
|
+
if missing:
|
|
186
|
+
raise InvocationError(f"{HARNESS_FILE} lacks {', '.join(missing)}")
|
|
187
|
+
return cls(**{f.name: data[f.name] for f in fields(cls)
|
|
188
|
+
if f.name in data})
|
|
189
|
+
|
|
190
|
+
def meta(self) -> dict:
|
|
191
|
+
"""The `harness_exit` value a trace carries: the ending and how it
|
|
192
|
+
was learned, and nothing about the driver's own clock -- the walls
|
|
193
|
+
are the driver's business, not a fact about the recorded program."""
|
|
194
|
+
return {"status": self.status, "signal": self.signal,
|
|
195
|
+
"basis": self.basis}
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def env_hash(env: Mapping[str, str]) -> str:
|
|
199
|
+
"""The TypeScript recorder's recipe: sorted `k=v` lines, sha256, first 16
|
|
200
|
+
hex characters. `typescript/src/redact.mjs`'s `envHash` is the same
|
|
201
|
+
recipe, and this is its only Python spelling.
|
|
202
|
+
|
|
203
|
+
In THIS module because neither of its two callers owns it: the driver
|
|
204
|
+
hashes its own environment into `invocation.json` below, and the
|
|
205
|
+
converter re-hashes the environment it redacted when it applies rule v1
|
|
206
|
+
to a BOOT written before that rule existed (`ts/build.py`). A second
|
|
207
|
+
spelling is a store whose traces stop comparing.
|
|
208
|
+
|
|
209
|
+
**Deliberately not the Python recorder's formula.** `record/boot.py`
|
|
210
|
+
hashes `json.dumps(env, sort_keys=True)`; this hashes the `"{k}={v}"`
|
|
211
|
+
join. Ruled 2026-09-02: `env_hash` is a per-recorder identity, compared
|
|
212
|
+
only between traces from the same recorder, and no command compares one
|
|
213
|
+
across languages.
|
|
214
|
+
|
|
215
|
+
`surrogateescape`, because a value out of `os.environ` can hold an
|
|
216
|
+
undecodable byte as a lone surrogate and a bare `.encode()` would raise
|
|
217
|
+
on it -- the whole invocation would go down over one variable the shell
|
|
218
|
+
happened to carry.
|
|
219
|
+
"""
|
|
220
|
+
body = "\n".join(f"{k}={v}" for k, v in sorted(env.items()))
|
|
221
|
+
return hashlib.sha256(body.encode("utf-8", "surrogateescape")).hexdigest()[:16]
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _load(path: Path) -> dict:
|
|
225
|
+
try:
|
|
226
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
227
|
+
except FileNotFoundError:
|
|
228
|
+
raise InvocationError(f"no {path.name} in {path.parent}: this is not "
|
|
229
|
+
"a spool directory a driver wrote") from None
|
|
230
|
+
except json.JSONDecodeError as e:
|
|
231
|
+
raise InvocationError(f"{path} is not readable JSON: {e}") from None
|
|
232
|
+
if not isinstance(data, dict):
|
|
233
|
+
raise InvocationError(f"{path} is not an object")
|
|
234
|
+
return data
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def read_invocation(spool_dir: Path) -> Invocation:
|
|
238
|
+
return Invocation.from_json(_load(Path(spool_dir) / INVOCATION_FILE))
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def read_harness(spool_dir: Path) -> HarnessExit | None:
|
|
242
|
+
"""The harness's ending, or None when nobody recorded one.
|
|
243
|
+
|
|
244
|
+
Absence is a fact and never an error: the driver writes this file after
|
|
245
|
+
the harness returns, so a driver that was killed first leaves a spool
|
|
246
|
+
directory that is complete in every other way. `ingest` is re-runnable
|
|
247
|
+
over exactly that directory.
|
|
248
|
+
"""
|
|
249
|
+
path = Path(spool_dir) / HARNESS_FILE
|
|
250
|
+
if not path.is_file():
|
|
251
|
+
return None
|
|
252
|
+
return HarnessExit.from_json(_load(path))
|