sensorium-trace 0.18.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (135) hide show
  1. sensorium/__init__.py +0 -0
  2. sensorium/__main__.py +4 -0
  3. sensorium/_c/rt/sensorium_c_rt.c +1446 -0
  4. sensorium/_typescript/package-lock.json +68 -0
  5. sensorium/_typescript/package.json +39 -0
  6. sensorium/_typescript/src/bindings.mjs +792 -0
  7. sensorium/_typescript/src/cap.mjs +227 -0
  8. sensorium/_typescript/src/capabilities.mjs +40 -0
  9. sensorium/_typescript/src/dbg.mjs +348 -0
  10. sensorium/_typescript/src/escape.mjs +351 -0
  11. sensorium/_typescript/src/focus.mjs +135 -0
  12. sensorium/_typescript/src/hook.mjs +127 -0
  13. sensorium/_typescript/src/index.mjs +1 -0
  14. sensorium/_typescript/src/naming.mjs +63 -0
  15. sensorium/_typescript/src/positions.mjs +37 -0
  16. sensorium/_typescript/src/probe.mjs +246 -0
  17. sensorium/_typescript/src/qualname.mjs +236 -0
  18. sensorium/_typescript/src/redact.mjs +711 -0
  19. sensorium/_typescript/src/register.mjs +15 -0
  20. sensorium/_typescript/src/resolve.mjs +260 -0
  21. sensorium/_typescript/src/rt.mjs +798 -0
  22. sensorium/_typescript/src/setup.mjs +11 -0
  23. sensorium/_typescript/src/tally.mjs +164 -0
  24. sensorium/_typescript/src/tasks.mjs +175 -0
  25. sensorium/_typescript/src/transform.mjs +800 -0
  26. sensorium/_typescript/src/vite.mjs +99 -0
  27. sensorium/c/__init__.py +8 -0
  28. sensorium/c/cc.py +612 -0
  29. sensorium/c/cli.py +60 -0
  30. sensorium/c/convert.py +669 -0
  31. sensorium/c/decode.py +300 -0
  32. sensorium/c/driver.py +365 -0
  33. sensorium/c/dwarf.py +706 -0
  34. sensorium/c/format.py +271 -0
  35. sensorium/c/pkg.py +187 -0
  36. sensorium/c/plan.py +277 -0
  37. sensorium/cli.py +140 -0
  38. sensorium/driver.py +47 -0
  39. sensorium/exit.py +33 -0
  40. sensorium/invocations.py +83 -0
  41. sensorium/limits.py +253 -0
  42. sensorium/mcp/__init__.py +0 -0
  43. sensorium/mcp/audit.py +162 -0
  44. sensorium/mcp/child.py +240 -0
  45. sensorium/mcp/cmd.py +101 -0
  46. sensorium/mcp/jsonrpc.py +172 -0
  47. sensorium/mcp/result.py +187 -0
  48. sensorium/mcp/schema.py +390 -0
  49. sensorium/mcp/server.py +493 -0
  50. sensorium/mcp/tools.py +299 -0
  51. sensorium/paths.py +80 -0
  52. sensorium/query/__init__.py +0 -0
  53. sensorium/query/caps.py +150 -0
  54. sensorium/query/dbg_dialects.py +64 -0
  55. sensorium/query/diff_cmd.py +748 -0
  56. sensorium/query/diff_notes.py +117 -0
  57. sensorium/query/exceptions_c.py +263 -0
  58. sensorium/query/exceptions_cmd.py +788 -0
  59. sensorium/query/exceptions_group.py +571 -0
  60. sensorium/query/exceptions_invocation.py +487 -0
  61. sensorium/query/exceptions_rust.py +683 -0
  62. sensorium/query/exceptions_typescript.py +688 -0
  63. sensorium/query/expr.py +606 -0
  64. sensorium/query/failures_cmd.py +306 -0
  65. sensorium/query/flow_cmd.py +778 -0
  66. sensorium/query/flow_report.py +128 -0
  67. sensorium/query/flow_values.py +195 -0
  68. sensorium/query/fmt.py +280 -0
  69. sensorium/query/frame_cmd.py +237 -0
  70. sensorium/query/grep_cmd.py +156 -0
  71. sensorium/query/help.py +18 -0
  72. sensorium/query/info_cmd.py +527 -0
  73. sensorium/query/info_rust.py +214 -0
  74. sensorium/query/info_typescript.py +261 -0
  75. sensorium/query/js_inspect.py +394 -0
  76. sensorium/query/moves.py +254 -0
  77. sensorium/query/redact_cmd.py +304 -0
  78. sensorium/query/refocus_cmd.py +661 -0
  79. sensorium/query/refocus_env.py +493 -0
  80. sensorium/query/refocus_facts.py +269 -0
  81. sensorium/query/refocus_licence.py +187 -0
  82. sensorium/query/refocus_report.py +327 -0
  83. sensorium/query/refocus_rust.py +537 -0
  84. sensorium/query/refocus_threads.py +197 -0
  85. sensorium/query/refocus_typescript.py +756 -0
  86. sensorium/query/refocus_world.py +666 -0
  87. sensorium/query/runs_cmd.py +213 -0
  88. sensorium/query/rust_debug.py +176 -0
  89. sensorium/query/sites.py +171 -0
  90. sensorium/query/tree_cmd.py +511 -0
  91. sensorium/query/tree_threads.py +168 -0
  92. sensorium/query/vocab.py +723 -0
  93. sensorium/query/watch_cmd.py +657 -0
  94. sensorium/record/__init__.py +0 -0
  95. sensorium/record/boot.py +797 -0
  96. sensorium/record/boot_io.py +141 -0
  97. sensorium/record/budget.py +135 -0
  98. sensorium/record/capture.py +433 -0
  99. sensorium/record/children.py +755 -0
  100. sensorium/record/children_merge.py +179 -0
  101. sensorium/record/elide.py +55 -0
  102. sensorium/record/fingerprint.py +26 -0
  103. sensorium/record/thread_starts.py +261 -0
  104. sensorium/record/tracer.py +798 -0
  105. sensorium/record/tracer_exc.py +199 -0
  106. sensorium/record/tracer_frames.py +618 -0
  107. sensorium/redact.py +304 -0
  108. sensorium/redact_content.py +156 -0
  109. sensorium/redact_key.py +317 -0
  110. sensorium/redact_store.py +502 -0
  111. sensorium/redact_values.py +433 -0
  112. sensorium/store/__init__.py +0 -0
  113. sensorium/store/db.py +284 -0
  114. sensorium/store/gc.py +360 -0
  115. sensorium/store/payloads.py +69 -0
  116. sensorium/store/reader.py +438 -0
  117. sensorium/store/writer.py +283 -0
  118. sensorium/ts/__init__.py +10 -0
  119. sensorium/ts/build.py +783 -0
  120. sensorium/ts/cli.py +130 -0
  121. sensorium/ts/driver.py +472 -0
  122. sensorium/ts/focus.py +146 -0
  123. sensorium/ts/harness.py +314 -0
  124. sensorium/ts/ingest.py +380 -0
  125. sensorium/ts/invocation.py +252 -0
  126. sensorium/ts/pkg.py +167 -0
  127. sensorium/ts/redaction.py +64 -0
  128. sensorium/ts/spool.py +168 -0
  129. sensorium/ts/truncation.py +81 -0
  130. sensorium/ts/wrapper.py +255 -0
  131. sensorium_trace-0.18.2.dist-info/METADATA +245 -0
  132. sensorium_trace-0.18.2.dist-info/RECORD +135 -0
  133. sensorium_trace-0.18.2.dist-info/WHEEL +4 -0
  134. sensorium_trace-0.18.2.dist-info/entry_points.txt +3 -0
  135. sensorium_trace-0.18.2.dist-info/licenses/LICENSE +21 -0
sensorium/limits.py ADDED
@@ -0,0 +1,253 @@
1
+ """Per-run recording caps and the store retention policy.
2
+
3
+ Resolution order, first value wins:
4
+
5
+ 1. A CLI flag (``None`` means the flag was not passed).
6
+ 2. An environment variable, when it is set and non-empty.
7
+ 3. A key in ``$SENSORIUM_DIR/config.json`` (or ``~/.sensorium/config.json``
8
+ when ``SENSORIUM_DIR`` is unset).
9
+ 4. The built-in default.
10
+
11
+ A value of ``0`` means unlimited for that knob. ``--no-cap`` is unlimited
12
+ for every knob the command did not set explicitly.
13
+
14
+ The config file is a JSON object. Unknown keys are ignored. The keys this
15
+ module reads:
16
+
17
+ - ``max_events``, ``max_bytes``, ``max_calls``: the per-run cap.
18
+ - ``store_budget``: optional total size of the trace store, in bytes.
19
+ - ``gc_keep``: optional maximum number of traces to retain.
20
+ - ``gc_older_than_days``: optional age after which a trace may be removed.
21
+
22
+ ``sensorium gc`` applies whichever of the three retention knobs it was
23
+ given. A run applies them automatically only when at least one is set in
24
+ the environment or the config file.
25
+ """
26
+ from __future__ import annotations
27
+
28
+ import json
29
+ import os
30
+ from dataclasses import dataclass
31
+ from pathlib import Path
32
+
33
+ DEFAULT_MAX_EVENTS = 100_000
34
+ DEFAULT_MAX_BYTES = 8 * 1024 * 1024
35
+ # Above the largest same-function burst the suite records (300 tasks) and
36
+ # far below a hot loop. A lower default would mark those traces truncated.
37
+ DEFAULT_MAX_CALLS = 1024
38
+
39
+ ENV_EVENTS = "SENSORIUM_MAX_EVENTS"
40
+ ENV_BYTES = "SENSORIUM_MAX_BYTES"
41
+ ENV_CALLS = "SENSORIUM_MAX_CALLS"
42
+ ENV_BUDGET = "SENSORIUM_STORE_BUDGET"
43
+ ENV_KEEP = "SENSORIUM_GC_KEEP"
44
+ ENV_OLDER = "SENSORIUM_GC_OLDER_THAN"
45
+
46
+ _BYTES = {
47
+ "kib": 1024, "kb": 1024, "k": 1024,
48
+ "mib": 1024 ** 2, "mb": 1024 ** 2, "m": 1024 ** 2,
49
+ "gib": 1024 ** 3, "gb": 1024 ** 3, "g": 1024 ** 3,
50
+ }
51
+ _DURATION = {"w": 7 * 86400, "d": 86400, "h": 3600, "m": 60, "s": 1}
52
+
53
+
54
+ class LimitsError(ValueError):
55
+ """A cap, a size, or a duration that cannot be read."""
56
+
57
+
58
+ @dataclass(frozen=True)
59
+ class Limits:
60
+ """The cap one recording is made under. ``0`` on a field is unlimited."""
61
+
62
+ max_events: int
63
+ max_bytes: int
64
+ max_calls: int
65
+
66
+ @property
67
+ def engaged(self) -> bool:
68
+ return bool(self.max_events or self.max_bytes or self.max_calls)
69
+
70
+
71
+ @dataclass(frozen=True)
72
+ class StorePolicy:
73
+ """What ``gc`` is allowed to delete. ``None`` means that knob is off."""
74
+
75
+ max_bytes: int | None = None
76
+ keep: int | None = None
77
+ older_than: float | None = None
78
+
79
+ def is_empty(self) -> bool:
80
+ return (self.max_bytes is None and self.keep is None
81
+ and self.older_than is None)
82
+
83
+
84
+ def parse_size(text) -> int:
85
+ """Bytes or a count. ``0`` is unlimited. ``K``/``M``/``G`` are powers of 1024.
86
+
87
+ ``KB``/``MB``/``GB`` and ``KiB``/``MiB``/``GiB`` are the same powers.
88
+ A fractional magnitude (``1.5M``) is accepted and truncated toward zero.
89
+ """
90
+ if isinstance(text, bool) or text is None:
91
+ raise LimitsError(f"not a size: {text!r}")
92
+ if isinstance(text, int):
93
+ if text < 0:
94
+ raise LimitsError(f"size must be >= 0 (got {text})")
95
+ return text
96
+ if isinstance(text, float):
97
+ if text < 0:
98
+ raise LimitsError(f"size must be >= 0 (got {text})")
99
+ return int(text)
100
+ raw = str(text).strip().lower().replace("_", "")
101
+ if not raw:
102
+ raise LimitsError("empty size")
103
+ mult = 1
104
+ # Longest suffix first, so "1mb" is mega and not the "m" prefix of it.
105
+ for suffix, factor in sorted(_BYTES.items(), key=lambda kv: len(kv[0]),
106
+ reverse=True):
107
+ if raw.endswith(suffix) and raw[:-len(suffix)]:
108
+ mult = factor
109
+ raw = raw[:-len(suffix)].strip()
110
+ break
111
+ try:
112
+ number = float(raw) if "." in raw else int(raw, 10)
113
+ except ValueError as e:
114
+ raise LimitsError(f"not a size: {text!r}") from e
115
+ if number < 0:
116
+ raise LimitsError(f"size must be >= 0 (got {text!r})")
117
+ return int(number * mult)
118
+
119
+
120
+ def parse_duration(text) -> float:
121
+ """Seconds. A bare number is seconds. ``30d``, ``12h``, ``2w``, ``45m``, ``90s``."""
122
+ if isinstance(text, bool) or text is None:
123
+ raise LimitsError(f"not a duration: {text!r}")
124
+ if isinstance(text, (int, float)):
125
+ if text < 0:
126
+ raise LimitsError(f"duration must be >= 0 (got {text})")
127
+ return float(text)
128
+ raw = str(text).strip().lower().replace("_", "")
129
+ if not raw:
130
+ raise LimitsError("empty duration")
131
+ mult = 1.0
132
+ if raw[-1] in _DURATION and not raw[-1].isdigit():
133
+ mult = float(_DURATION[raw[-1]])
134
+ raw = raw[:-1].strip()
135
+ if not raw:
136
+ raise LimitsError(f"not a duration: {text!r}")
137
+ try:
138
+ number = float(raw)
139
+ except ValueError as e:
140
+ raise LimitsError(f"not a duration: {text!r}") from e
141
+ if number < 0:
142
+ raise LimitsError(f"duration must be >= 0 (got {text!r})")
143
+ return number * mult
144
+
145
+
146
+ def config_path(environ: dict | None = None) -> Path:
147
+ env = os.environ if environ is None else environ
148
+ root = env.get("SENSORIUM_DIR")
149
+ base = Path(root) if root else Path.home() / ".sensorium"
150
+ return base / "config.json"
151
+
152
+
153
+ def load_config(environ: dict | None = None) -> dict:
154
+ """The store config object, or ``{}`` when there is no file.
155
+
156
+ A file that exists and is not a JSON object is an error: ignoring it
157
+ would apply the defaults and delete, or record, on a policy the user
158
+ thought they had written.
159
+ """
160
+ path = config_path(environ)
161
+ if not path.is_file():
162
+ return {}
163
+ try:
164
+ data = json.loads(path.read_text(encoding="utf-8"))
165
+ except (OSError, json.JSONDecodeError) as e:
166
+ raise LimitsError(f"cannot read {path}: {e}") from e
167
+ if not isinstance(data, dict):
168
+ raise LimitsError(f"{path} must be a JSON object")
169
+ return data
170
+
171
+
172
+ def _from_env(environ: dict, name: str):
173
+ if name not in environ:
174
+ return None
175
+ text = environ[name]
176
+ if text is None or str(text).strip() == "":
177
+ return None
178
+ return text
179
+
180
+
181
+ def _pick(cli, env_text, config_value, default: int) -> int:
182
+ if cli is not None:
183
+ return parse_size(cli)
184
+ if env_text is not None:
185
+ return parse_size(env_text)
186
+ if config_value is not None:
187
+ return parse_size(config_value)
188
+ return default
189
+
190
+
191
+ def resolve(*, max_events=None, max_bytes=None, max_calls=None,
192
+ no_cap: bool = False, environ: dict | None = None,
193
+ config: dict | None = None) -> Limits:
194
+ """The cap for one recording.
195
+
196
+ ``config`` may be passed by a test. Otherwise the store file is read.
197
+ ``no_cap`` fills only the knobs the call did not set.
198
+ """
199
+ env = os.environ if environ is None else environ
200
+ cfg = load_config(env) if config is None else config
201
+ if no_cap:
202
+ if max_events is None:
203
+ max_events = 0
204
+ if max_bytes is None:
205
+ max_bytes = 0
206
+ if max_calls is None:
207
+ max_calls = 0
208
+ return Limits(
209
+ max_events=_pick(max_events, _from_env(env, ENV_EVENTS),
210
+ cfg.get("max_events"), DEFAULT_MAX_EVENTS),
211
+ max_bytes=_pick(max_bytes, _from_env(env, ENV_BYTES),
212
+ cfg.get("max_bytes"), DEFAULT_MAX_BYTES),
213
+ max_calls=_pick(max_calls, _from_env(env, ENV_CALLS),
214
+ cfg.get("max_calls"), DEFAULT_MAX_CALLS),
215
+ )
216
+
217
+
218
+ def _optional_int(cli, env_text, config_value) -> int | None:
219
+ if cli is not None:
220
+ return parse_size(cli)
221
+ if env_text is not None:
222
+ return parse_size(env_text)
223
+ if config_value is not None:
224
+ return parse_size(config_value)
225
+ return None
226
+
227
+
228
+ def store_policy(*, max_bytes=None, keep=None, older_than=None,
229
+ environ: dict | None = None,
230
+ config: dict | None = None) -> StorePolicy:
231
+ """Retention for ``gc``. ``older_than`` is seconds when passed as a number."""
232
+ env = os.environ if environ is None else environ
233
+ cfg = load_config(env) if config is None else config
234
+ if older_than is None:
235
+ env_age = _from_env(env, ENV_OLDER)
236
+ if env_age is not None:
237
+ older = parse_duration(env_age)
238
+ elif cfg.get("gc_older_than_days") is not None:
239
+ older = float(parse_size(cfg["gc_older_than_days"])) * 86400
240
+ else:
241
+ older = None
242
+ else:
243
+ older = float(older_than)
244
+ resolved_keep = _optional_int(keep, _from_env(env, ENV_KEEP),
245
+ cfg.get("gc_keep"))
246
+ if resolved_keep is not None and resolved_keep < 1:
247
+ raise LimitsError("keep must be >= 1")
248
+ return StorePolicy(
249
+ max_bytes=_optional_int(max_bytes, _from_env(env, ENV_BUDGET),
250
+ cfg.get("store_budget")),
251
+ keep=resolved_keep,
252
+ older_than=older,
253
+ )
File without changes
sensorium/mcp/audit.py ADDED
@@ -0,0 +1,162 @@
1
+ """`mcp.jsonl`: what was asked of this server, and how it ended.
2
+
3
+ One JSON line per `tools/call`, per rejection and per cancel, in
4
+ `<store root>/mcp.jsonl` -- a sibling of `invocations.jsonl`, invisible
5
+ to every trace lookup for its reason: `runs` and `find_trace` glob
6
+ `traces/*.db`, and this file is not under `traces/`.
7
+
8
+ WHAT IT IS FOR. The census a future policy argument has to stand on
9
+ (§6.3) -- which tools a model reaches for, how often a call is refused
10
+ and why, how much output it takes to answer. So a rejection line carries
11
+ `reason` and `fields` as DATA (P16): one prose message cannot be
12
+ counted, and a census parsing English stops matching the first time the
13
+ wording improves.
14
+
15
+ WHAT IT IS NOT. Never the environment, never `cwd`, never the result
16
+ text. `bytes` says how much was shown and `truncated` whether that was
17
+ all of it; what was shown in full is the trace, which the tenant already
18
+ holds. Arguments pass the content rule first (D28): a model that types a
19
+ secret into `grep`'s pattern leaves the marker here, not the value. The
20
+ invocation log the child writes does NOT apply the rule to its argv --
21
+ an asymmetry older than this file, carried as debt, not fixed in passing.
22
+
23
+ ONE KNOB (D29). `SENSORIUM_NO_INVOCATION_LOG` disables this file too,
24
+ read through `invocations._log_disabled` rather than re-implemented: the
25
+ user has one word for "log nothing about my calls" and two readings of
26
+ it would eventually disagree. The server says so once on stderr at boot.
27
+ And nothing here raises: an unwritable location prints one line and
28
+ returns, exactly as `invocations.record` does -- a tool call's answer is
29
+ never the audit file's business.
30
+
31
+ This module records. `result.py` renders; the two never import each
32
+ other, and the server is what calls both.
33
+ """
34
+ from __future__ import annotations
35
+
36
+ import json
37
+ import os
38
+ import sys
39
+ import threading
40
+ from pathlib import Path
41
+
42
+ from sensorium import invocations, paths, redact_content
43
+ from sensorium.mcp.child import Outcome
44
+
45
+ #: One writer at a time, because there are two: `record_cancel` runs on
46
+ #: the server's READER thread while `record_call` runs on the worker,
47
+ #: and each opens an fd of its own. `O_APPEND` makes ONE `write(2)`
48
+ #: atomic, which covers a short line on a regular file and nothing else
49
+ #: -- a line past the 8 KiB text buffer, or a store on a filesystem that
50
+ #: writes short, leaves in several. This lock is this PROCESS's: two
51
+ #: servers on one store still rely on `O_APPEND` alone.
52
+ _WRITING = threading.Lock()
53
+
54
+
55
+ def path() -> Path:
56
+ return paths.trace_root() / "mcp.jsonl"
57
+
58
+
59
+ def disabled() -> bool:
60
+ """The invocation log's knob, not a second one of our own (D29)."""
61
+ return invocations._log_disabled()
62
+
63
+
64
+ def scrub(value):
65
+ """`value` with the content rule applied to every string in it.
66
+
67
+ All the way down, not the top level only: `focus`, `include` and
68
+ `exclude` are arrays, and a secret typed into one of them would
69
+ otherwise reach the file verbatim. Tuples come back as lists, which
70
+ is what JSON has; anything that is not a string, a sequence or a
71
+ mapping passes through untouched.
72
+ """
73
+ if isinstance(value, str):
74
+ return redact_content.content(value)[0]
75
+ if isinstance(value, (list, tuple)):
76
+ return [scrub(v) for v in value]
77
+ if isinstance(value, dict):
78
+ return {k: scrub(v) for k, v in value.items()}
79
+ return value
80
+
81
+
82
+ def _write(line: dict) -> None:
83
+ """Append one line. Never raises; writes nothing when disabled."""
84
+ if disabled():
85
+ return
86
+ try:
87
+ p = path()
88
+ # 0700 on the root and 0600 on the file: `invocations.record`'s
89
+ # two-half rule, for its reason -- this file names every call and
90
+ # its arguments, and `p.open("a")` would create it 0666-under-the-
91
+ # umask. An EXISTING DIRECTORY keeps its own mode.
92
+ p.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
93
+ with _WRITING:
94
+ fd = os.open(p, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600)
95
+ # ...but the file does NOT. `os.open`'s mode applies on
96
+ # CREATE only, so an `mcp.jsonl` left 0644 by an older build,
97
+ # a restore or a copy under another umask would keep it while
98
+ # `docs/mcp.md` promised 0600. Re-tightened on the fd we just
99
+ # opened, which is the one file we know we are about to write
100
+ # -- never by path, which is a different file by then.
101
+ if hasattr(os, "fchmod") and os.fstat(fd).st_mode & 0o077:
102
+ os.fchmod(fd, 0o600)
103
+ with os.fdopen(fd, "a", encoding="utf-8") as f:
104
+ f.write(json.dumps(line, separators=(",", ":"),
105
+ ensure_ascii=False) + "\n")
106
+ except (OSError, RuntimeError, ValueError) as e:
107
+ # OSError: the usual "can't write there". RuntimeError: `path()`
108
+ # -> `trace_root()` -> `Path.home()` raises THAT, not OSError,
109
+ # when no home can be determined and SENSORIUM_DIR is unset.
110
+ # ValueError: `UnicodeEncodeError` is one, and a LONE SURROGATE
111
+ # in a field name, a tool name or an argument -- all model-typed
112
+ # text -- cannot be encoded onto this UTF-8 file. That one used
113
+ # to walk out of here into the server's `_work`, which answered
114
+ # `-32603` for an id that already held a result: two responses
115
+ # to one request, out of a logger that promises to raise none.
116
+ # The census loses the line and says so on stderr, which is the
117
+ # bargain this module already makes for every unwritable case.
118
+ print(f"sensorium mcp: audit: {e}", file=sys.stderr)
119
+
120
+
121
+ def record_call(tool: str, arguments: dict, o: Outcome, nbytes: int,
122
+ truncated: bool) -> None:
123
+ """One answered call. `bytes` is the length of the FINAL text."""
124
+ line = {"utc": invocations._utc_now(), "tool": tool,
125
+ "arguments": scrub(arguments), "exit": o.exit,
126
+ "bytes": nbytes, "truncated": truncated, "ms": o.ms}
127
+ if o.timed_out:
128
+ line["timeout"] = True
129
+ if o.exit is None and not o.cancelled:
130
+ # Why there was no status -- the timeout's sentence or the spawn
131
+ # error. A cancel needs none: `record_cancel` says so in a field.
132
+ line["cause"] = o.cause
133
+ _write(line)
134
+
135
+
136
+ def record_rejection(tool: str | None, code: int, reason: str, message: str,
137
+ fields: tuple[str, ...] = ()) -> None:
138
+ """A call that was refused (P16). `tool` is None for a malformed
139
+ request that named none; `fields` is written only when there are
140
+ fields to name, an empty list reading to a census as "some field was
141
+ named", which is not what happened.
142
+
143
+ Tool name, field names and message are model-typed text like any
144
+ argument, so they pass the content rule too: a secret misspelt into a
145
+ field name must not survive here because the call was REFUSED.
146
+ """
147
+ rejected: dict = {"code": code, "reason": reason}
148
+ if fields:
149
+ rejected["fields"] = scrub(list(fields))
150
+ rejected["message"] = scrub(message)
151
+ _write({"utc": invocations._utc_now(), "tool": scrub(tool),
152
+ "rejected": rejected})
153
+
154
+
155
+ def record_cancel(tool: str, arguments: dict, queued: bool,
156
+ ms: int | None) -> None:
157
+ """A call the client cancelled. `queued` says whether it had started:
158
+ a cancel that landed while the request was still in the queue never
159
+ spawned anything, and `ms` is None for it."""
160
+ _write({"utc": invocations._utc_now(), "tool": tool,
161
+ "arguments": scrub(arguments), "cancelled": True,
162
+ "queued": queued, "ms": ms})
sensorium/mcp/child.py ADDED
@@ -0,0 +1,240 @@
1
+ """One tool call, one child process -- in its own session, killed by group.
2
+
3
+ WHY A CHILD (spec S4.1). `sensorium run` executes the target IN the
4
+ recording process under `sys.monitoring`, so an in-process `record`
5
+ would import the client's program into the server; a query that raised
6
+ would take the server down with it; `cwd` and `SENSORIUM_DIR` are
7
+ process state, not arguments; and a timeout on an in-process call has no
8
+ clean kill. The cost is one interpreter start per call, which is
9
+ reported and never gated. The interpreter is `sys.executable` (or the
10
+ call's own `python`), never a `sensorium` found on PATH -- the
11
+ stale-driver lesson applies to the server's own interpreter too.
12
+
13
+ WHY STDIN IS /dev/null. The server's stdin IS the JSON-RPC stream. A
14
+ child that inherited it would read the client's next request and the
15
+ session would stall with neither side at fault, so `stdin` is
16
+ `DEVNULL` for every call without exception.
17
+
18
+ THE KILL ORDER, and why SIGKILL is unconditional. `start_new_session`
19
+ puts the child in a session and a process group of its own, whose id is
20
+ the child's pid, so one `killpg` reaches everything the call spawned --
21
+ never `pkill -f`, which self-matches and leaves the survivors. The
22
+ sequence is SIGTERM, then up to `GRACE` seconds of polling for the GROUP
23
+ to empty (the leader is reaped as we go: a zombie leader keeps its group
24
+ alive and would read as a survivor), then SIGKILL -- sent whether or not
25
+ anything answered, because a leader that exited politely can leave a
26
+ grandchild that ignored SIGTERM holding the inherited pipes, and a kill
27
+ that asked "is the leader still alive?" would never send the signal that
28
+ ends it. `communicate(timeout=REAP)` bounds the read: a descendant that
29
+ escaped the session with its own `setsid` can still hold a pipe.
30
+
31
+ ONE READER (R9). Only `run`'s thread touches the pipes. Two threads
32
+ inside one `Popen.communicate` are not two readers of one buffer: the
33
+ second rebinds `_fileobj2output`, so each sees half the output, and the
34
+ first to reach EOF closes a file object the other is about to read from
35
+ BY NUMBER, which a concurrent spawn may since have reused. So `cancel`
36
+ signals with `reap=False` and waits for `run` to come out on its own.
37
+
38
+ CANCEL IS NEVER LOST. `run` spawns under the Child's one lock and, still
39
+ holding it, re-reads the cancelled flag and kills the group it has just
40
+ created. That closes the window the server cannot close from outside:
41
+ the reader thread can call `cancel` after the worker popped the request
42
+ but before `Popen` returned, and a cancel that only killed `_proc` would
43
+ find it `None` and do nothing. `cancel` takes the same lock, so the two
44
+ can happen in one order only; the worker's `communicate` then returns
45
+ because the pipes closed, reads the flag, and reports `cancelled` with
46
+ no exit status. A cancel after `run` has finished is a no-op -- the pid
47
+ is reaped, and may already be somebody else's.
48
+ """
49
+ from __future__ import annotations
50
+
51
+ import os
52
+ import signal
53
+ import subprocess
54
+ import sys
55
+ import threading
56
+ import time
57
+ from dataclasses import dataclass
58
+
59
+ #: Seconds between SIGTERM and the unconditional SIGKILL.
60
+ GRACE = 2.0
61
+
62
+ #: How often the grace period asks whether the group has emptied.
63
+ POLL = 0.05
64
+
65
+ #: The bound on the final read: a pipe held from outside the session.
66
+ REAP = 2.0
67
+
68
+
69
+ @dataclass(frozen=True)
70
+ class Outcome:
71
+ """What one call produced. `exit` is `Popen.returncode` (negative
72
+ for a signal) and is None whenever no exit was PRODUCED -- a timeout,
73
+ a cancel, a spawn that never happened -- which the server renders as
74
+ `no answer` rather than as a status."""
75
+
76
+ exit: int | None
77
+ stdout: str
78
+ stderr: str
79
+ cause: str | None
80
+ ms: int
81
+ timed_out: bool = False
82
+ cancelled: bool = False
83
+ pgid: int | None = None
84
+
85
+
86
+ def _signal_group(pgid: int, sig: int) -> None:
87
+ try:
88
+ os.killpg(pgid, sig)
89
+ except ProcessLookupError:
90
+ pass # already gone: what the signal was for
91
+
92
+
93
+ def _shut(proc: subprocess.Popen) -> None:
94
+ """Close what we will not finish reading, so no fd leaks."""
95
+ for pipe in (proc.stdout, proc.stderr):
96
+ if pipe is not None:
97
+ try:
98
+ pipe.close()
99
+ except OSError:
100
+ pass
101
+
102
+
103
+ def _reap(proc: subprocess.Popen) -> tuple[str, str]:
104
+ """Collect the child and its streams, without hanging on either.
105
+ `Popen` keeps what each `communicate` read, so a second call after
106
+ `kill_group` adds no wait and hands back the whole of it."""
107
+ try:
108
+ out, err = proc.communicate(timeout=REAP)
109
+ except subprocess.TimeoutExpired:
110
+ _shut(proc) # a descendant outside the session has it
111
+ except (OSError, ValueError):
112
+ pass # already closed by an earlier kill
113
+ else:
114
+ return out or "", err or ""
115
+ return "", ""
116
+
117
+
118
+ def kill_group(proc: subprocess.Popen, grace: float = GRACE,
119
+ reap: bool = True) -> None:
120
+ """SIGTERM the child's group, wait `grace` for it to empty, SIGKILL
121
+ it anyway, then reap. See the module docstring for why every step is
122
+ where it is. `ProcessLookupError` at any step means "already gone"
123
+ and is never an error.
124
+
125
+ `reap=False` signals ONLY: no `communicate`, no `close`, nothing
126
+ that touches a pipe -- what a caller on a second thread passes (R9)."""
127
+ if not hasattr(os, "killpg"): # pragma: no cover - Windows
128
+ proc.terminate() # advisory, untested here
129
+ try:
130
+ proc.wait(timeout=grace)
131
+ except subprocess.TimeoutExpired:
132
+ proc.kill()
133
+ if reap:
134
+ _reap(proc)
135
+ return
136
+ pgid = proc.pid # start_new_session: the child leads
137
+ _signal_group(pgid, signal.SIGTERM)
138
+ deadline = time.monotonic() + grace
139
+ while time.monotonic() < deadline:
140
+ proc.poll() # reap: a zombie leader holds the group
141
+ try:
142
+ os.killpg(pgid, 0)
143
+ except ProcessLookupError:
144
+ break # the whole group has gone
145
+ time.sleep(POLL)
146
+ _signal_group(pgid, signal.SIGKILL)
147
+ if reap:
148
+ _reap(proc)
149
+
150
+
151
+ class Child:
152
+ """The CLI, as a child process, for exactly one call.
153
+
154
+ `name` is the tool the server was asked for, which is not always
155
+ `argv[0]` (`record` is the CLI's `run`); it appears in the one log
156
+ line and nowhere else. `cwd` and `python` arrive from a JSON object
157
+ where "" is a valid string, and an empty one means what an absent
158
+ one means -- `Popen(cwd="")` raises and `[""]` is no interpreter."""
159
+
160
+ def __init__(self, argv: list[str], cwd: str | None, python: str | None,
161
+ env: dict, timeout: float, log=None,
162
+ name: str | None = None):
163
+ self._argv = list(argv)
164
+ self._cwd = cwd or None
165
+ self._python = python or None
166
+ self._env = env
167
+ self._timeout = timeout
168
+ self._log = log
169
+ self._name = name or (self._argv[0] if self._argv else "?")
170
+ self._lock = threading.Lock()
171
+ self._proc: subprocess.Popen | None = None
172
+ self._cancelled = False
173
+ self._done = threading.Event() # `run` has left `communicate`
174
+
175
+ def run(self) -> Outcome:
176
+ started = time.monotonic()
177
+ with self._lock:
178
+ try:
179
+ proc = subprocess.Popen(
180
+ [self._python or sys.executable, "-m", "sensorium",
181
+ *self._argv],
182
+ cwd=self._cwd, env=self._env,
183
+ stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
184
+ stderr=subprocess.PIPE, text=True, encoding="utf-8",
185
+ errors="replace", start_new_session=True)
186
+ except OSError as err:
187
+ return self._outcome(started, None, "", "", str(err))
188
+ self._proc = proc
189
+ if self._log is not None:
190
+ self._log(f"child {proc.pid} pgid {proc.pid} "
191
+ f"tool {self._name}")
192
+ if self._cancelled:
193
+ kill_group(proc) # a cancel that landed before the spawn
194
+ timed_out, broke, out, err = False, None, "", ""
195
+ try:
196
+ out, err = proc.communicate(timeout=self._timeout)
197
+ except subprocess.TimeoutExpired:
198
+ timed_out = True
199
+ except (OSError, ValueError) as exc:
200
+ broke = str(exc) # a cancel closed the pipes to unblock us
201
+ finally:
202
+ self._done.set() # from here the pipes are nobody's
203
+ if timed_out:
204
+ kill_group(proc) # the worker IS the single reader here
205
+ out, err = _reap(proc)
206
+ elif broke is not None:
207
+ try:
208
+ proc.wait(timeout=REAP) # never leave the leader unreaped
209
+ except subprocess.TimeoutExpired:
210
+ pass
211
+ with self._lock:
212
+ cancelled = self._cancelled
213
+ if cancelled:
214
+ return self._outcome(started, None, out, err, "cancelled",
215
+ cancelled=True, pgid=proc.pid)
216
+ if timed_out:
217
+ cause = (f"timed out after {self._timeout:g} s "
218
+ f"(server flag --timeout / --run-timeout)")
219
+ return self._outcome(started, None, out, err, cause,
220
+ timed_out=True, pgid=proc.pid)
221
+ return self._outcome(started, proc.returncode, out, err, broke,
222
+ pgid=proc.pid)
223
+
224
+ def cancel(self) -> None:
225
+ """Kill the call, from whichever thread noticed -- by SIGNAL
226
+ only, never a read (R9, and the module docstring). Idempotent,
227
+ and a no-op once `run` has finished."""
228
+ with self._lock:
229
+ self._cancelled = True
230
+ if self._done.is_set() or self._proc is None:
231
+ return # nothing running, or nothing yet to kill
232
+ proc = self._proc
233
+ kill_group(proc, reap=False)
234
+ if not self._done.wait(REAP):
235
+ _shut(proc) # a pipe held from outside the session
236
+
237
+ def _outcome(self, started: float, exit_: int | None, out: str, err: str,
238
+ cause: str | None, **flags) -> Outcome:
239
+ return Outcome(exit=exit_, stdout=out, stderr=err, cause=cause,
240
+ ms=int((time.monotonic() - started) * 1000), **flags)