sensorium-trace 0.18.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sensorium/__init__.py +0 -0
- sensorium/__main__.py +4 -0
- sensorium/_c/rt/sensorium_c_rt.c +1446 -0
- sensorium/_typescript/package-lock.json +68 -0
- sensorium/_typescript/package.json +39 -0
- sensorium/_typescript/src/bindings.mjs +792 -0
- sensorium/_typescript/src/cap.mjs +227 -0
- sensorium/_typescript/src/capabilities.mjs +40 -0
- sensorium/_typescript/src/dbg.mjs +348 -0
- sensorium/_typescript/src/escape.mjs +351 -0
- sensorium/_typescript/src/focus.mjs +135 -0
- sensorium/_typescript/src/hook.mjs +127 -0
- sensorium/_typescript/src/index.mjs +1 -0
- sensorium/_typescript/src/naming.mjs +63 -0
- sensorium/_typescript/src/positions.mjs +37 -0
- sensorium/_typescript/src/probe.mjs +246 -0
- sensorium/_typescript/src/qualname.mjs +236 -0
- sensorium/_typescript/src/redact.mjs +711 -0
- sensorium/_typescript/src/register.mjs +15 -0
- sensorium/_typescript/src/resolve.mjs +260 -0
- sensorium/_typescript/src/rt.mjs +798 -0
- sensorium/_typescript/src/setup.mjs +11 -0
- sensorium/_typescript/src/tally.mjs +164 -0
- sensorium/_typescript/src/tasks.mjs +175 -0
- sensorium/_typescript/src/transform.mjs +800 -0
- sensorium/_typescript/src/vite.mjs +99 -0
- sensorium/c/__init__.py +8 -0
- sensorium/c/cc.py +612 -0
- sensorium/c/cli.py +60 -0
- sensorium/c/convert.py +669 -0
- sensorium/c/decode.py +300 -0
- sensorium/c/driver.py +365 -0
- sensorium/c/dwarf.py +706 -0
- sensorium/c/format.py +271 -0
- sensorium/c/pkg.py +187 -0
- sensorium/c/plan.py +277 -0
- sensorium/cli.py +140 -0
- sensorium/driver.py +47 -0
- sensorium/exit.py +33 -0
- sensorium/invocations.py +83 -0
- sensorium/limits.py +253 -0
- sensorium/mcp/__init__.py +0 -0
- sensorium/mcp/audit.py +162 -0
- sensorium/mcp/child.py +240 -0
- sensorium/mcp/cmd.py +101 -0
- sensorium/mcp/jsonrpc.py +172 -0
- sensorium/mcp/result.py +187 -0
- sensorium/mcp/schema.py +390 -0
- sensorium/mcp/server.py +493 -0
- sensorium/mcp/tools.py +299 -0
- sensorium/paths.py +80 -0
- sensorium/query/__init__.py +0 -0
- sensorium/query/caps.py +150 -0
- sensorium/query/dbg_dialects.py +64 -0
- sensorium/query/diff_cmd.py +748 -0
- sensorium/query/diff_notes.py +117 -0
- sensorium/query/exceptions_c.py +263 -0
- sensorium/query/exceptions_cmd.py +788 -0
- sensorium/query/exceptions_group.py +571 -0
- sensorium/query/exceptions_invocation.py +487 -0
- sensorium/query/exceptions_rust.py +683 -0
- sensorium/query/exceptions_typescript.py +688 -0
- sensorium/query/expr.py +606 -0
- sensorium/query/failures_cmd.py +306 -0
- sensorium/query/flow_cmd.py +778 -0
- sensorium/query/flow_report.py +128 -0
- sensorium/query/flow_values.py +195 -0
- sensorium/query/fmt.py +280 -0
- sensorium/query/frame_cmd.py +237 -0
- sensorium/query/grep_cmd.py +156 -0
- sensorium/query/help.py +18 -0
- sensorium/query/info_cmd.py +527 -0
- sensorium/query/info_rust.py +214 -0
- sensorium/query/info_typescript.py +261 -0
- sensorium/query/js_inspect.py +394 -0
- sensorium/query/moves.py +254 -0
- sensorium/query/redact_cmd.py +304 -0
- sensorium/query/refocus_cmd.py +661 -0
- sensorium/query/refocus_env.py +493 -0
- sensorium/query/refocus_facts.py +269 -0
- sensorium/query/refocus_licence.py +187 -0
- sensorium/query/refocus_report.py +327 -0
- sensorium/query/refocus_rust.py +537 -0
- sensorium/query/refocus_threads.py +197 -0
- sensorium/query/refocus_typescript.py +756 -0
- sensorium/query/refocus_world.py +666 -0
- sensorium/query/runs_cmd.py +213 -0
- sensorium/query/rust_debug.py +176 -0
- sensorium/query/sites.py +171 -0
- sensorium/query/tree_cmd.py +511 -0
- sensorium/query/tree_threads.py +168 -0
- sensorium/query/vocab.py +723 -0
- sensorium/query/watch_cmd.py +657 -0
- sensorium/record/__init__.py +0 -0
- sensorium/record/boot.py +797 -0
- sensorium/record/boot_io.py +141 -0
- sensorium/record/budget.py +135 -0
- sensorium/record/capture.py +433 -0
- sensorium/record/children.py +755 -0
- sensorium/record/children_merge.py +179 -0
- sensorium/record/elide.py +55 -0
- sensorium/record/fingerprint.py +26 -0
- sensorium/record/thread_starts.py +261 -0
- sensorium/record/tracer.py +798 -0
- sensorium/record/tracer_exc.py +199 -0
- sensorium/record/tracer_frames.py +618 -0
- sensorium/redact.py +304 -0
- sensorium/redact_content.py +156 -0
- sensorium/redact_key.py +317 -0
- sensorium/redact_store.py +502 -0
- sensorium/redact_values.py +433 -0
- sensorium/store/__init__.py +0 -0
- sensorium/store/db.py +284 -0
- sensorium/store/gc.py +360 -0
- sensorium/store/payloads.py +69 -0
- sensorium/store/reader.py +438 -0
- sensorium/store/writer.py +283 -0
- sensorium/ts/__init__.py +10 -0
- sensorium/ts/build.py +783 -0
- sensorium/ts/cli.py +130 -0
- sensorium/ts/driver.py +472 -0
- sensorium/ts/focus.py +146 -0
- sensorium/ts/harness.py +314 -0
- sensorium/ts/ingest.py +380 -0
- sensorium/ts/invocation.py +252 -0
- sensorium/ts/pkg.py +167 -0
- sensorium/ts/redaction.py +64 -0
- sensorium/ts/spool.py +168 -0
- sensorium/ts/truncation.py +81 -0
- sensorium/ts/wrapper.py +255 -0
- sensorium_trace-0.18.2.dist-info/METADATA +245 -0
- sensorium_trace-0.18.2.dist-info/RECORD +135 -0
- sensorium_trace-0.18.2.dist-info/WHEEL +4 -0
- sensorium_trace-0.18.2.dist-info/entry_points.txt +3 -0
- sensorium_trace-0.18.2.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,502 @@
|
|
|
1
|
+
"""`sensorium redact`'s JUDGEMENT: what a trace already on disk becomes.
|
|
2
|
+
|
|
3
|
+
Parts A and B put rule v1 at the three recorders' writers, so nothing NEW
|
|
4
|
+
reaches disk in plaintext; this is the other half. All 273 traces this box
|
|
5
|
+
holds predate the rule, each with a live token in `meta.env` at 0644. `plan()`
|
|
6
|
+
decides what one becomes -- the environment and its hash, every changed
|
|
7
|
+
payload row, `unwind_exc`, output row and `children` element, the stamp, the
|
|
8
|
+
counts (C2) -- and returns a `Plan`. It writes NOTHING (C14, P1): `apply()` is
|
|
9
|
+
the only writer, and `--dry-run` is this function alone.
|
|
10
|
+
|
|
11
|
+
The rule is not re-implemented: `redact.fires` and `redact_values` answer for
|
|
12
|
+
the retrofit what they answer for the recorders, which is what makes a second
|
|
13
|
+
pass idempotent (C5). **The environment pass is the NAME rule only** (C1) -- a
|
|
14
|
+
retrofit doing more to an old trace than a recorder does to a new one would be
|
|
15
|
+
two rules wearing one name -- and `redact.env` refuses by its docstring to
|
|
16
|
+
meet the already redacted environment a re-visit always is, so `_env_pass` is
|
|
17
|
+
its loop with two skips (C5, P4). **`env_hash` is reproduced BEFORE it is
|
|
18
|
+
recomputed** (C3): a retrofit that cannot get the stored hash out of the
|
|
19
|
+
stored environment cannot claim the new one is the same formula.
|
|
20
|
+
|
|
21
|
+
The stamp is always `by: "retrofit"` (C6) and its `values` ADDITIVE (P2): a
|
|
22
|
+
content hit leaves text rather than a marker, so what a trace HOLDS cannot be
|
|
23
|
+
recounted and the sum of the hands' counts is honest. It is written when
|
|
24
|
+
anything else changed or the trace was not already `mode: "on"` -- a pre-rule
|
|
25
|
+
or `mode: "off"` trace is stamped even when nothing fired (P5), an UNKEYED one
|
|
26
|
+
becomes keyed (P6), a settled one left alone. Rows go in id order, a RETURN
|
|
27
|
+
under the name `code_objects` holds (P7), an unchanged one unwritten (P8).
|
|
28
|
+
|
|
29
|
+
Nothing escapes as an exception: a newer format, a format-4 trace missing
|
|
30
|
+
required meta, an unknown lang (those three in `db.open_trace`'s words minus
|
|
31
|
+
its leading path), a non-reproducing hash, another key, a live tmp, a file
|
|
32
|
+
SQLite cannot open, a file another process removed, a meta row that is not
|
|
33
|
+
JSON -- each is a sentence on the `Plan` (C9), so one refusal does not end
|
|
34
|
+
an `--all` pass; an in-flight trace is skipped (C8). Of the judgement
|
|
35
|
+
only `live_tmp` touches the filesystem, to unlink a dead pid's leftover.
|
|
36
|
+
|
|
37
|
+
`apply()` is the other half and the only hand that writes: C7's backup ->
|
|
38
|
+
rewrite -> checkpoint -> fsync -> rename -> sidecars, on a private copy, so
|
|
39
|
+
the original is whole up to one atomic rename. A row the plan did not name
|
|
40
|
+
is a row it does not touch.
|
|
41
|
+
"""
|
|
42
|
+
import hashlib
|
|
43
|
+
import json
|
|
44
|
+
import os
|
|
45
|
+
import sqlite3
|
|
46
|
+
import stat
|
|
47
|
+
from collections.abc import Mapping
|
|
48
|
+
from dataclasses import dataclass, field, replace
|
|
49
|
+
from pathlib import Path
|
|
50
|
+
|
|
51
|
+
from sensorium import redact
|
|
52
|
+
from sensorium import redact_values as rv
|
|
53
|
+
from sensorium.redact_key import Key, _pid_alive
|
|
54
|
+
from sensorium.store import db, payloads
|
|
55
|
+
from sensorium.ts.invocation import env_hash as _kv_env_hash
|
|
56
|
+
|
|
57
|
+
#: The rewrite's tmp is `.<run>.db.redact.<pid>.tmp` beside the trace: a
|
|
58
|
+
#: dotfile, so every glob for `*.db` walks past it.
|
|
59
|
+
TMP_SUFFIX = ".redact"
|
|
60
|
+
|
|
61
|
+
#: What SQLite leaves beside a WAL database; a result carries neither.
|
|
62
|
+
_SIDECARS = ("-wal", "-shm")
|
|
63
|
+
|
|
64
|
+
#: The recorders whose `env_hash` is the sorted `k=v` join. An absent
|
|
65
|
+
#: `lang` predates the key and is the Python recorder's.
|
|
66
|
+
KV_LANGS = frozenset({"rust", "typescript"})
|
|
67
|
+
|
|
68
|
+
#: What a trace is left at, and what `tightens` measures against (C11).
|
|
69
|
+
TIGHT = 0o600
|
|
70
|
+
|
|
71
|
+
#: The writers' own separators (`store/writer.add_event`), so a row this
|
|
72
|
+
#: rewrites differs from the recorder's only where the rule fired.
|
|
73
|
+
_SEP = (",", ":")
|
|
74
|
+
|
|
75
|
+
_SKIPPED = "in flight (incomplete)"
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class SidecarLeft(OSError):
|
|
79
|
+
"""The rewrite LANDED and the OLD `-wal`/`-shm` are still on disk.
|
|
80
|
+
|
|
81
|
+
Raised only from `apply`'s post-rename cleanup, and distinct from every
|
|
82
|
+
other failure there because it is the one that is not a failure of the
|
|
83
|
+
rewrite (R19): the rename is past, the trace on disk is the redacted
|
|
84
|
+
one, and what could not be done is removing the displaced inode's log.
|
|
85
|
+
The command prints it as a NOTE and counts the trace as rewritten --
|
|
86
|
+
`REFUSED: the rewrite failed` over a trace that WAS rewritten would send
|
|
87
|
+
a reader looking for plaintext that is no longer in the database, and
|
|
88
|
+
say nothing about the log that still holds it.
|
|
89
|
+
"""
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
@dataclass(frozen=True)
|
|
93
|
+
class Plan:
|
|
94
|
+
"""One trace's whole future, in memory: `apply()` touches nothing this
|
|
95
|
+
does not name, and a plan that refuses or skips names nothing at all.
|
|
96
|
+
`env_names` is what the environment pass took and `values` what every
|
|
97
|
+
other site did; `meta` is the keys to (re)write (a subset of `env`,
|
|
98
|
+
`env_hash`, `redaction`, `children`); the row maps hold FINISHED text.
|
|
99
|
+
"""
|
|
100
|
+
path: Path
|
|
101
|
+
run: str
|
|
102
|
+
lang: str
|
|
103
|
+
refused: str | None = None
|
|
104
|
+
skipped: str | None = None
|
|
105
|
+
env_names: tuple[str, ...] = ()
|
|
106
|
+
values: int = 0
|
|
107
|
+
meta: dict = field(default_factory=dict)
|
|
108
|
+
payloads: dict[int, str] = field(default_factory=dict)
|
|
109
|
+
unwinds: dict[int, str] = field(default_factory=dict)
|
|
110
|
+
outputs: dict[int, str] = field(default_factory=dict)
|
|
111
|
+
mode_before: int = TIGHT
|
|
112
|
+
|
|
113
|
+
@property
|
|
114
|
+
def rewrites(self) -> bool:
|
|
115
|
+
return bool(self.meta or self.payloads or self.unwinds or self.outputs)
|
|
116
|
+
|
|
117
|
+
@property
|
|
118
|
+
def tightens(self) -> bool:
|
|
119
|
+
return self.mode_before != TIGHT
|
|
120
|
+
|
|
121
|
+
@property
|
|
122
|
+
def changes(self) -> bool:
|
|
123
|
+
return (not self.refused and not self.skipped
|
|
124
|
+
and (self.rewrites or self.tightens))
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def env_hash_for(lang: str, env: Mapping[str, str]) -> str:
|
|
128
|
+
"""The `env_hash` `lang`'s recorder would write for `env`: two formulas,
|
|
129
|
+
because it is a per-recorder identity (ruled 2026-09-02)."""
|
|
130
|
+
if lang in KV_LANGS:
|
|
131
|
+
return _kv_env_hash(env)
|
|
132
|
+
return hashlib.sha256(
|
|
133
|
+
json.dumps(env, sort_keys=True).encode()).hexdigest()[:16]
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def live_tmp(path: Path) -> str | None:
|
|
137
|
+
"""The refusal sentence if another `redact` holds this trace, else None.
|
|
138
|
+
|
|
139
|
+
A killed retrofit leaves the original whole and its tmp behind (C7), and
|
|
140
|
+
whose it was is in the name: a live pid holds the trace, a dead one left
|
|
141
|
+
litter, swept here because no other moment is sure to come; a middle
|
|
142
|
+
that is not a positive pid is nobody's to delete (`sweep_stale`)."""
|
|
143
|
+
path = Path(path)
|
|
144
|
+
prefix = f".{path.stem}.db{TMP_SUFFIX}."
|
|
145
|
+
for tmp in sorted(path.parent.glob(f"{prefix}*.tmp")):
|
|
146
|
+
try:
|
|
147
|
+
pid = int(tmp.name.removeprefix(prefix).removesuffix(".tmp"))
|
|
148
|
+
except ValueError:
|
|
149
|
+
continue
|
|
150
|
+
if pid <= 0:
|
|
151
|
+
continue
|
|
152
|
+
if _pid_alive(pid):
|
|
153
|
+
return f"another redact is rewriting it (pid {pid})"
|
|
154
|
+
try:
|
|
155
|
+
tmp.unlink()
|
|
156
|
+
except OSError:
|
|
157
|
+
pass # a leftover this cannot remove is not a refusal
|
|
158
|
+
return None
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def plan(path: Path, key: Key, knobs: redact.Knobs) -> Plan:
|
|
162
|
+
"""What `path` becomes under `key` and the caller's `knobs` -- the
|
|
163
|
+
command's own environment as `redact_cmd` read it (C4). Nothing here
|
|
164
|
+
reads `os.environ`, so a `SENSORIUM_NO_REDACT` in the shell running the
|
|
165
|
+
retrofit cannot quietly turn it off."""
|
|
166
|
+
# C4, forced at the one point the knobs enter: `redact.env` returns
|
|
167
|
+
# early on `off` but `redact.fires` deliberately ignores it, so an
|
|
168
|
+
# unforced `off` would take every firing value and then have
|
|
169
|
+
# `redact.meta` stamp the two-key `mode: "off"` shape over an
|
|
170
|
+
# environment whose plaintext is gone. Running the command IS the
|
|
171
|
+
# decision (§7), and `mode` is always "on".
|
|
172
|
+
knobs = replace(knobs, off=False)
|
|
173
|
+
path = Path(path)
|
|
174
|
+
run = path.stem
|
|
175
|
+
try:
|
|
176
|
+
mode_before = stat.S_IMODE(path.stat().st_mode)
|
|
177
|
+
except OSError:
|
|
178
|
+
mode_before = TIGHT # the open below refuses it, by name
|
|
179
|
+
busy = live_tmp(path)
|
|
180
|
+
if busy is not None:
|
|
181
|
+
return Plan(path, run, "", refused=busy, mode_before=mode_before)
|
|
182
|
+
try:
|
|
183
|
+
conn = db.open_trace(path)
|
|
184
|
+
except db.TraceFormatError as e:
|
|
185
|
+
# Its sentence opens with the path; the per-trace line names the run.
|
|
186
|
+
return Plan(path, run, "", mode_before=mode_before,
|
|
187
|
+
refused=str(e).removeprefix(f"{path} "))
|
|
188
|
+
except sqlite3.DatabaseError as e:
|
|
189
|
+
return Plan(path, run, "", mode_before=mode_before,
|
|
190
|
+
refused=f"not a database this sensorium can open: {e}")
|
|
191
|
+
except ValueError as e:
|
|
192
|
+
return Plan(path, run, "", mode_before=mode_before,
|
|
193
|
+
refused=f"meta is not JSON: {e}")
|
|
194
|
+
except OSError as e:
|
|
195
|
+
# A trace another process removed between the walk and this open:
|
|
196
|
+
# an `--all` pass loses that trace and not the rest of them.
|
|
197
|
+
return Plan(path, run, "", mode_before=mode_before,
|
|
198
|
+
refused=f"cannot open: {e}")
|
|
199
|
+
try:
|
|
200
|
+
return _judge(path, run, conn, key, knobs, mode_before)
|
|
201
|
+
finally:
|
|
202
|
+
conn.close()
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _judge(path: Path, run: str, conn: sqlite3.Connection, key: Key,
|
|
206
|
+
knobs: redact.Knobs, mode_before: int) -> Plan:
|
|
207
|
+
"""`plan`'s decision with the connection open, split off so one
|
|
208
|
+
`finally` closes it whichever way out is taken."""
|
|
209
|
+
try:
|
|
210
|
+
meta = db.all_meta(conn)
|
|
211
|
+
except ValueError as e:
|
|
212
|
+
return Plan(path, run, "", mode_before=mode_before,
|
|
213
|
+
refused=f"meta is not JSON: {e}")
|
|
214
|
+
lang = meta.get("lang") or "python"
|
|
215
|
+
if meta.get("incomplete") is True:
|
|
216
|
+
# The Python recorder writes in place: another process holds this
|
|
217
|
+
# open in WAL mode right now (C8).
|
|
218
|
+
return Plan(path, run, lang, skipped=_SKIPPED, mode_before=mode_before)
|
|
219
|
+
env = meta.get("env")
|
|
220
|
+
env = env if isinstance(env, dict) else {}
|
|
221
|
+
if env_hash_for(lang, env) != meta.get("env_hash"):
|
|
222
|
+
return Plan(path, run, lang, mode_before=mode_before,
|
|
223
|
+
refused=f"env_hash does not reproduce under the "
|
|
224
|
+
f"{lang} formula")
|
|
225
|
+
prev = meta.get("redaction")
|
|
226
|
+
already_on = isinstance(prev, dict) and prev.get("mode") == "on"
|
|
227
|
+
if already_on and prev.get("keyed") and prev.get("key_id") != key.key_id:
|
|
228
|
+
# C5: a stamp naming one key must be true of every digest under it.
|
|
229
|
+
return Plan(path, run, lang, mode_before=mode_before,
|
|
230
|
+
refused=f"digests under key {prev.get('key_id')}, the "
|
|
231
|
+
f"store's is {key.key_id or 'none'}; "
|
|
232
|
+
f"nothing rewritten")
|
|
233
|
+
table = dict(prev.get("env") or {}) if already_on else {}
|
|
234
|
+
stored, names = _env_pass(env, table, key, knobs)
|
|
235
|
+
writes: dict = {}
|
|
236
|
+
rv.install(key, knobs)
|
|
237
|
+
try:
|
|
238
|
+
payloads, unwinds, outputs, values = _rows(conn)
|
|
239
|
+
values += _children(meta, writes)
|
|
240
|
+
finally:
|
|
241
|
+
rv.reset()
|
|
242
|
+
if stored != env:
|
|
243
|
+
writes["env"] = stored
|
|
244
|
+
writes["env_hash"] = env_hash_for(lang, stored)
|
|
245
|
+
if writes or payloads or unwinds or outputs or not already_on:
|
|
246
|
+
before = prev.get("values", 0) if isinstance(prev, dict) else 0
|
|
247
|
+
writes["redaction"] = {**redact.meta(key, knobs, table, by="retrofit"),
|
|
248
|
+
"values": before + values}
|
|
249
|
+
return Plan(path, run, lang, env_names=tuple(names), values=values,
|
|
250
|
+
meta=writes, payloads=payloads, unwinds=unwinds,
|
|
251
|
+
outputs=outputs, mode_before=mode_before)
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def _env_pass(env: dict, table: dict, key: Key,
|
|
255
|
+
knobs: redact.Knobs) -> tuple[dict, list[str]]:
|
|
256
|
+
"""The environment to store and the names THIS pass took, sorted.
|
|
257
|
+
`table` is the previous stamp's map, extended in place, and the two
|
|
258
|
+
`continue`s are C5's and P4's."""
|
|
259
|
+
stored = {n: v for n, v in env.items() if n != redact.KEY_VAR}
|
|
260
|
+
names = []
|
|
261
|
+
for name in sorted(stored):
|
|
262
|
+
if name in table or stored[name] == redact.REDACTED:
|
|
263
|
+
continue
|
|
264
|
+
if not redact.fires(name, knobs):
|
|
265
|
+
continue
|
|
266
|
+
table[name] = key.digest(stored[name])
|
|
267
|
+
stored[name] = redact.REDACTED
|
|
268
|
+
names.append(name)
|
|
269
|
+
return stored, names
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def _rows(conn: sqlite3.Connection) -> tuple[dict, dict, dict, int]:
|
|
273
|
+
"""Every row the writers reach (C2) -- payloads, `unwind_exc`, output --
|
|
274
|
+
as three id -> text maps, and what they now withhold. `task_id` is not
|
|
275
|
+
named: the store's 18 oldest traces are format 1 and have no column."""
|
|
276
|
+
payloads_out: dict[int, str] = {}
|
|
277
|
+
unwinds: dict[int, str] = {}
|
|
278
|
+
outputs: dict[int, str] = {}
|
|
279
|
+
values = 0
|
|
280
|
+
strings = payloads.load_strings(conn)
|
|
281
|
+
for eid, kind, text, qualname in conn.execute(
|
|
282
|
+
"SELECT e.id, e.kind, e.payload, c.qualname FROM events e "
|
|
283
|
+
"LEFT JOIN code_objects c ON c.id = e.code_id "
|
|
284
|
+
"WHERE e.payload IS NOT NULL ORDER BY e.id").fetchall():
|
|
285
|
+
obj = json.loads(payloads.expand(text, strings))
|
|
286
|
+
new = _transform_payload(kind, obj, qualname)
|
|
287
|
+
if new != obj:
|
|
288
|
+
payloads_out[eid] = json.dumps(new, separators=_SEP)
|
|
289
|
+
values += rv.count(new) - rv.count(obj)
|
|
290
|
+
for fid, text in conn.execute(
|
|
291
|
+
"SELECT id, unwind_exc FROM frames "
|
|
292
|
+
"WHERE unwind_exc IS NOT NULL").fetchall():
|
|
293
|
+
obj = json.loads(text)
|
|
294
|
+
new = rv.exc(obj)
|
|
295
|
+
if new != obj:
|
|
296
|
+
unwinds[fid] = json.dumps(new, separators=_SEP)
|
|
297
|
+
values += rv.count_capture(new) - rv.count_capture(obj)
|
|
298
|
+
for oid, data in conn.execute(
|
|
299
|
+
"SELECT id, data FROM output ORDER BY id").fetchall():
|
|
300
|
+
after, hit = rv.text(data)
|
|
301
|
+
if hit:
|
|
302
|
+
outputs[oid] = after
|
|
303
|
+
values += 1
|
|
304
|
+
return payloads_out, unwinds, outputs, values
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def _transform_payload(kind: str, obj: dict, qualname: str | None) -> dict:
|
|
308
|
+
"""One payload under rule v1, as its writer would have stored it: a NEW
|
|
309
|
+
dict in the row's own key order, so one this leaves alone re-encodes
|
|
310
|
+
byte for byte. A key outside the capture vocabulary is copied across,
|
|
311
|
+
never guessed at; a RETURN with no `code_id` takes `value` (P7)."""
|
|
312
|
+
out = {}
|
|
313
|
+
for name, item in obj.items():
|
|
314
|
+
if name in ("args", "deltas") and isinstance(item, dict):
|
|
315
|
+
out[name] = {n: (rv.named(n, c) if isinstance(c, dict) else c)
|
|
316
|
+
for n, c in item.items()}
|
|
317
|
+
elif name == "value" and isinstance(item, dict):
|
|
318
|
+
out[name] = (rv.named_return(qualname, item)
|
|
319
|
+
if kind == "RETURN" and qualname else rv.value(item))
|
|
320
|
+
elif name in ("exc", "thrown") and isinstance(item, dict):
|
|
321
|
+
out[name] = rv.exc(item)
|
|
322
|
+
else:
|
|
323
|
+
out[name] = item
|
|
324
|
+
return out
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def _children(meta: dict, writes: dict) -> int:
|
|
328
|
+
"""`meta.children` under the CONTENT rule, element by element (R25), and
|
|
329
|
+
how many elements it took: a command line has positions rather than
|
|
330
|
+
bindings, so no name to read; `writes` gains the list only if it
|
|
331
|
+
changed."""
|
|
332
|
+
children = meta.get("children")
|
|
333
|
+
if not isinstance(children, list):
|
|
334
|
+
return 0
|
|
335
|
+
after, hits = [], 0
|
|
336
|
+
for command in children:
|
|
337
|
+
if not isinstance(command, list):
|
|
338
|
+
after.append(command)
|
|
339
|
+
continue
|
|
340
|
+
ruled = [rv.text(element) for element in command]
|
|
341
|
+
hits += sum(1 for _text, hit in ruled if hit)
|
|
342
|
+
after.append([text for text, _hit in ruled])
|
|
343
|
+
if hits:
|
|
344
|
+
writes["children"] = after
|
|
345
|
+
return hits
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def tmp_path_for(path: Path) -> Path:
|
|
350
|
+
"""The private copy `apply` builds this trace's replacement in:
|
|
351
|
+
`TMP_SUFFIX`'s name, BESIDE the trace because `os.replace` is atomic
|
|
352
|
+
only within one filesystem and a store can sit on any mount, and
|
|
353
|
+
carrying this pid because that is what `live_tmp` reads to tell a
|
|
354
|
+
live run's workspace from a killed one's litter."""
|
|
355
|
+
path = Path(path)
|
|
356
|
+
return path.with_name(f".{path.stem}.db{TMP_SUFFIX}.{os.getpid()}.tmp")
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
def _unlink_sidecars(path: Path) -> int:
|
|
360
|
+
"""`path`'s `-wal` and `-shm` removed, and how many there were.
|
|
361
|
+
Absent is the ordinary case and not an error; any other `OSError` is
|
|
362
|
+
raised, because a `-wal` this cannot remove is a file still holding
|
|
363
|
+
the plaintext pages the rewrite just took out."""
|
|
364
|
+
gone = 0
|
|
365
|
+
for suffix in _SIDECARS:
|
|
366
|
+
try:
|
|
367
|
+
path.with_name(path.name + suffix).unlink()
|
|
368
|
+
except FileNotFoundError:
|
|
369
|
+
continue
|
|
370
|
+
gone += 1
|
|
371
|
+
return gone
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def apply(p: Plan) -> None:
|
|
375
|
+
"""Put `p` on disk (C7, C11). The only writer in the retrofit.
|
|
376
|
+
|
|
377
|
+
In a private copy and never in place: a retrofit dying mid-rewrite
|
|
378
|
+
would leave a trace that is neither the old one nor the new. The copy
|
|
379
|
+
is `Connection.backup` and not a byte copy of the `.db` because a
|
|
380
|
+
committed row can be living in the `-wal` with nobody yet
|
|
381
|
+
checkpointing it -- in the DATABASE and not in that file -- and a copy
|
|
382
|
+
that drops it is a different trace (R37's lesson).
|
|
383
|
+
|
|
384
|
+
The TRUNCATE checkpoint comes BEFORE the close, so the copy is whole
|
|
385
|
+
in one file when it is renamed and the result carries no sidecars of
|
|
386
|
+
its own. The directory is fsynced after the rename because a rename
|
|
387
|
+
is durable only when its directory entry is. The ORIGINAL's
|
|
388
|
+
`-wal`/`-shm` go LAST because they belong to the inode the rename
|
|
389
|
+
displaced: unlinking them earlier would take pages the backup had not
|
|
390
|
+
read, and a reader holding that inode keeps them open regardless.
|
|
391
|
+
|
|
392
|
+
What a kill leaves: before the create, nothing; between it and the
|
|
393
|
+
rename, the original whole and a tmp this pid owns, swept by the next
|
|
394
|
+
`redact` to reach the trace once the pid is dead (C7); after it, the
|
|
395
|
+
redacted trace, the displaced inode's sidecars gone in a `finally` so
|
|
396
|
+
that only a signal ending the process outright can leave them. A
|
|
397
|
+
failure this process sees rather than dies from clears its own tmp
|
|
398
|
+
and is raised to the caller -- as `SidecarLeft` when the rename is
|
|
399
|
+
already past and only the old log could not be removed, which is a
|
|
400
|
+
note about a trace that WAS rewritten and not a failed rewrite (R19).
|
|
401
|
+
"""
|
|
402
|
+
if p.refused or p.skipped or not p.changes:
|
|
403
|
+
return
|
|
404
|
+
if not p.rewrites:
|
|
405
|
+
# C11: nothing to carry, so nothing is copied. A trace already
|
|
406
|
+
# redacted but sitting at 0644 is CHANGED, and the sidecars are
|
|
407
|
+
# tightened in place, not removed: nothing rewrote their pages.
|
|
408
|
+
os.chmod(p.path, TIGHT)
|
|
409
|
+
for suffix in _SIDECARS:
|
|
410
|
+
sidecar = p.path.with_name(p.path.name + suffix)
|
|
411
|
+
if sidecar.exists():
|
|
412
|
+
os.chmod(sidecar, TIGHT)
|
|
413
|
+
return
|
|
414
|
+
tmp = tmp_path_for(p.path)
|
|
415
|
+
try:
|
|
416
|
+
# O_EXCL, so a taken name is a failure and never a file this
|
|
417
|
+
# writes over blind; the pid in it makes that this process's own
|
|
418
|
+
# leftover. Another live process's tmp was `plan`'s refusal (C7).
|
|
419
|
+
os.close(os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, TIGHT))
|
|
420
|
+
_copy_and_rewrite(p, tmp)
|
|
421
|
+
_fsync(tmp)
|
|
422
|
+
os.chmod(tmp, TIGHT)
|
|
423
|
+
os.replace(tmp, p.path)
|
|
424
|
+
try:
|
|
425
|
+
_fsync(p.path.parent)
|
|
426
|
+
finally:
|
|
427
|
+
# In a `finally` and not after the `try`: past the rename the
|
|
428
|
+
# displaced inode's log is a file of plaintext pages beside a
|
|
429
|
+
# redacted database, and SQLite will recover it OVER that
|
|
430
|
+
# database -- serving the secrets back, or refusing the trace
|
|
431
|
+
# as malformed. A directory fsync that fails, or a Ctrl-C
|
|
432
|
+
# during an `--all` pass, must not be what leaves it there.
|
|
433
|
+
try:
|
|
434
|
+
_unlink_sidecars(p.path)
|
|
435
|
+
except OSError as e:
|
|
436
|
+
# Re-raised under its own class (R19), because past the
|
|
437
|
+
# rename this is the one failure here that is not a
|
|
438
|
+
# failure of the rewrite: the caller prints it as a note
|
|
439
|
+
# and counts the trace as done rather than refusing a
|
|
440
|
+
# trace that is on disk redacted.
|
|
441
|
+
raise SidecarLeft(f"old sidecar not removed: {e}") from e
|
|
442
|
+
except BaseException:
|
|
443
|
+
# The database first: `_unlink_sidecars` raises on an OSError that
|
|
444
|
+
# is not absence, and a tmp holding a whole plaintext copy must
|
|
445
|
+
# not survive because its `-wal` could not be removed.
|
|
446
|
+
tmp.unlink(missing_ok=True)
|
|
447
|
+
_unlink_sidecars(tmp)
|
|
448
|
+
raise
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
def _copy_and_rewrite(p: Plan, tmp: Path) -> None:
|
|
452
|
+
"""The copy, the rewrite and the checkpoint -- `apply`'s middle. One
|
|
453
|
+
transaction, so the copy is never a half-redacted database even for an
|
|
454
|
+
instant; the rows go in as the plan finished them, every judgement
|
|
455
|
+
having been made before anything was opened for writing. The original
|
|
456
|
+
is read and closed here and touched no further.
|
|
457
|
+
|
|
458
|
+
The checkpoint's FIRST column is read and not discarded (R19). `PRAGMA
|
|
459
|
+
wal_checkpoint(TRUNCATE)` returns `(busy, log, checkpointed)` and
|
|
460
|
+
REPORTS a failure rather than raising one: a busy checkpoint leaves
|
|
461
|
+
pages in the log, and the next line unlinks that log. On this tmp it
|
|
462
|
+
cannot happen -- one connection, just committed, nobody else holding
|
|
463
|
+
the file -- which is exactly why it is checked: the cost of reading a
|
|
464
|
+
column is nothing, and the cost of not reading it is a rewritten trace
|
|
465
|
+
that quietly lost the rows the checkpoint could not move."""
|
|
466
|
+
dst = sqlite3.connect(tmp)
|
|
467
|
+
try:
|
|
468
|
+
src = sqlite3.connect(p.path)
|
|
469
|
+
try:
|
|
470
|
+
src.backup(dst)
|
|
471
|
+
finally:
|
|
472
|
+
src.close()
|
|
473
|
+
dst.executemany("UPDATE events SET payload = ? WHERE id = ?",
|
|
474
|
+
[(t, i) for i, t in p.payloads.items()])
|
|
475
|
+
dst.executemany("UPDATE frames SET unwind_exc = ? WHERE id = ?",
|
|
476
|
+
[(t, i) for i, t in p.unwinds.items()])
|
|
477
|
+
dst.executemany("UPDATE output SET data = ? WHERE id = ?",
|
|
478
|
+
[(t, i) for i, t in p.outputs.items()])
|
|
479
|
+
for name, value in p.meta.items():
|
|
480
|
+
db.set_meta(dst, name, value)
|
|
481
|
+
payloads.drop_unreferenced(dst)
|
|
482
|
+
dst.commit()
|
|
483
|
+
busy, log, done = dst.execute(
|
|
484
|
+
"PRAGMA wal_checkpoint(TRUNCATE)").fetchone()
|
|
485
|
+
if busy:
|
|
486
|
+
raise sqlite3.OperationalError(
|
|
487
|
+
f"checkpoint busy: wal_checkpoint(TRUNCATE) returned "
|
|
488
|
+
f"({busy}, {log}, {done})")
|
|
489
|
+
finally:
|
|
490
|
+
dst.close()
|
|
491
|
+
# Past the checkpoint a surviving log is litter, not a reader's.
|
|
492
|
+
_unlink_sidecars(tmp)
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
def _fsync(path: Path) -> None:
|
|
496
|
+
"""`path` on the platter, file or directory: a rename is only as
|
|
497
|
+
durable as the two things it names."""
|
|
498
|
+
fd = os.open(path, os.O_RDONLY)
|
|
499
|
+
try:
|
|
500
|
+
os.fsync(fd)
|
|
501
|
+
finally:
|
|
502
|
+
os.close(fd)
|