rolloutdiff 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. rolloutdiff/__init__.py +130 -0
  2. rolloutdiff/_aria_replay.py +346 -0
  3. rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
  4. rolloutdiff/_dependence.py +195 -0
  5. rolloutdiff/_feature_values.py +79 -0
  6. rolloutdiff/_replay_container.py +568 -0
  7. rolloutdiff/_report_validation.py +758 -0
  8. rolloutdiff/_retrospective_policy.py +33 -0
  9. rolloutdiff/_sampling.py +127 -0
  10. rolloutdiff/_series_validation.py +67 -0
  11. rolloutdiff/_watch_report.py +436 -0
  12. rolloutdiff/_watch_storage.py +800 -0
  13. rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
  14. rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
  15. rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
  16. rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
  17. rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
  18. rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
  19. rolloutdiff/_web/assets/third-party-notices.txt +112 -0
  20. rolloutdiff/_web/index.html +14 -0
  21. rolloutdiff/adapters.py +264 -0
  22. rolloutdiff/app.py +956 -0
  23. rolloutdiff/app_evidence.py +678 -0
  24. rolloutdiff/benchmark_reference.py +458 -0
  25. rolloutdiff/cli.py +1292 -0
  26. rolloutdiff/comparison.py +572 -0
  27. rolloutdiff/comparison_report.py +982 -0
  28. rolloutdiff/corroboration.py +287 -0
  29. rolloutdiff/detect.py +307 -0
  30. rolloutdiff/events.py +173 -0
  31. rolloutdiff/evidence.py +285 -0
  32. rolloutdiff/exemplars.py +240 -0
  33. rolloutdiff/features.py +823 -0
  34. rolloutdiff/html_report.py +726 -0
  35. rolloutdiff/inspect_logs.py +383 -0
  36. rolloutdiff/inspection.py +207 -0
  37. rolloutdiff/integrations.py +552 -0
  38. rolloutdiff/investigation.py +1146 -0
  39. rolloutdiff/notifications.py +406 -0
  40. rolloutdiff/pipeline.py +809 -0
  41. rolloutdiff/producer.py +663 -0
  42. rolloutdiff/prompt_mix.py +258 -0
  43. rolloutdiff/ranking.py +388 -0
  44. rolloutdiff/replay.py +589 -0
  45. rolloutdiff/replication.py +572 -0
  46. rolloutdiff/report.py +838 -0
  47. rolloutdiff/retrospective.py +386 -0
  48. rolloutdiff/schema.py +109 -0
  49. rolloutdiff/sensitivity.py +202 -0
  50. rolloutdiff/series.py +430 -0
  51. rolloutdiff/tool_features.py +422 -0
  52. rolloutdiff/trajectories.py +236 -0
  53. rolloutdiff/watch-compatibility.json +13 -0
  54. rolloutdiff/watch.py +742 -0
  55. rolloutdiff-0.1.0.dist-info/METADATA +158 -0
  56. rolloutdiff-0.1.0.dist-info/RECORD +62 -0
  57. rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
  58. rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
  59. rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
  60. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
  61. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
  62. rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,823 @@
1
+ """Lexical and structural response features.
2
+
3
+ Complete think/thinking sections are excluded; resp_chars retains original length.
4
+ AST indicators omit alias resolution, dataflow, and outcomes. Non-Python fences are
5
+ excluded from prose.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import ast
11
+ import io
12
+ import re
13
+ import tokenize
14
+ import warnings
15
+ from collections import Counter
16
+ from dataclasses import dataclass, field
17
+
18
+ import pandas as pd
19
+
20
+ # File tags cannot pair across fence boundaries.
21
+ _FENCE_RE = re.compile(
22
+ r"^[ \t]*(?P<fence>(?P<marker>`|~)(?P=marker){2,})(?P<language>[^\n]*)\n"
23
+ r"(?P<fence_body>.*?)(?:^[ \t]*(?P=fence)(?P=marker)*[ \t]*(?:\n|$)|\Z)",
24
+ re.DOTALL | re.MULTILINE | re.IGNORECASE,
25
+ )
26
+ _FILE_OPEN_RE = re.compile(
27
+ r"<file\b(?P<attrs>(?:[^>'\"]|'[^']*'|\"[^\"]*\")*)>", re.IGNORECASE
28
+ )
29
+ _FILE_CLOSE_RE = re.compile(r"</file\s*>", re.IGNORECASE)
30
+ _FILE_ATTR_RE = re.compile(r"(?:^|\s)([\w:.-]+)\s*=\s*(['\"])(.*?)\2", re.DOTALL)
31
+ _FILE_HINT_RE = re.compile(r"</?file\b", re.IGNORECASE)
32
+ _MAX_MARKUP_DEPTH = 64
33
+ _FSTRING_PREFIX_RE = re.compile(r"(?:fr|rf|f)(?=['\"])", re.IGNORECASE)
34
+ _FILENAME_RE = re.compile(
35
+ r"^[ \t]*#[ \t]*filename[ \t]*:[ \t]*(.+?)[ \t]*$", re.MULTILINE | re.IGNORECASE
36
+ )
37
+ _THINK_RE = re.compile(r"<(think|thinking)\b[^>]*>.*?</\1>", re.DOTALL | re.IGNORECASE)
38
+ _WORD_RE = re.compile(r"\b[a-zA-Z]{3,}\b")
39
+ _STOP = frozenset(
40
+ [
41
+ "the",
42
+ "and",
43
+ "for",
44
+ "that",
45
+ "with",
46
+ "you",
47
+ "this",
48
+ "are",
49
+ "can",
50
+ "not",
51
+ "your",
52
+ "from",
53
+ "have",
54
+ "but",
55
+ "will",
56
+ "what",
57
+ "how",
58
+ "all",
59
+ "out",
60
+ "use",
61
+ "any",
62
+ "has",
63
+ "also",
64
+ "more",
65
+ "make",
66
+ "which",
67
+ "then",
68
+ "there",
69
+ "their",
70
+ "about",
71
+ "when",
72
+ "some",
73
+ "into",
74
+ "other",
75
+ "just",
76
+ "its",
77
+ "only",
78
+ "these",
79
+ "could",
80
+ "would",
81
+ "should",
82
+ "been",
83
+ "very",
84
+ "who",
85
+ "where",
86
+ "does",
87
+ "did",
88
+ "they",
89
+ "them",
90
+ "than",
91
+ "because",
92
+ "while",
93
+ "after",
94
+ "such",
95
+ "through",
96
+ "between",
97
+ "being",
98
+ "most",
99
+ "here",
100
+ "much",
101
+ "those",
102
+ "well",
103
+ "one",
104
+ ]
105
+ )
106
+ _EXIT_CALLS = {"sys.exit", "os._exit", "exit", "quit", "os.abort"}
107
+ _EXEC_CALLS = {
108
+ "exec",
109
+ "eval",
110
+ "compile",
111
+ "subprocess.run",
112
+ "subprocess.call",
113
+ "subprocess.Popen",
114
+ "os.system",
115
+ "os.popen",
116
+ }
117
+ _CMP_DUNDERS = {
118
+ "__eq__",
119
+ "__ne__",
120
+ "__lt__",
121
+ "__le__",
122
+ "__gt__",
123
+ "__ge__",
124
+ "__hash__",
125
+ "__bool__",
126
+ "__contains__",
127
+ }
128
+ _FILE_EXT_RE = re.compile(r"^[\w\-./]+\.(py|txt|json|cfg|ini|toml|yaml|yml|sh|csv)$")
129
+
130
+
131
+ _AST_FLAGS = (
132
+ "calls_exit",
133
+ "calls_exec_or_shell",
134
+ "file_write",
135
+ "file_read",
136
+ "defines_cmp_dunder",
137
+ "defines_any_dunder",
138
+ "defines_test_token",
139
+ "module_attr_assign",
140
+ "has_main_guard",
141
+ "top_level_call",
142
+ "imports_sys",
143
+ "imports_os",
144
+ "raises",
145
+ "catches_all",
146
+ "defines_class",
147
+ "overrides_builtin_name",
148
+ )
149
+ _BUILTINS_LIKE = {
150
+ "print",
151
+ "input",
152
+ "open",
153
+ "len",
154
+ "range",
155
+ "exit",
156
+ "quit",
157
+ "any",
158
+ "all",
159
+ "sorted",
160
+ "map",
161
+ "filter",
162
+ }
163
+
164
+
165
+ @dataclass
166
+ class RolloutFeatures:
167
+ dense: dict[str, float] = field(default_factory=dict)
168
+ sparse: set[str] = field(default_factory=set)
169
+
170
+
171
+ def _callee_name(node: ast.AST) -> str | None:
172
+ if isinstance(node, ast.Name):
173
+ return node.id
174
+ if isinstance(node, ast.Attribute):
175
+ base = _callee_name(node.value)
176
+ return f"{base}.{node.attr}" if base else node.attr
177
+ return None
178
+
179
+
180
+ def _parse(code: str) -> ast.Module | None:
181
+ try:
182
+ with warnings.catch_warnings():
183
+ warnings.simplefilter("ignore", SyntaxWarning)
184
+ warnings.simplefilter("ignore", DeprecationWarning)
185
+ return ast.parse(code)
186
+ except (SyntaxError, ValueError, RecursionError):
187
+ return None
188
+
189
+
190
+ @dataclass(frozen=True)
191
+ class _Region:
192
+ start: int
193
+ end: int
194
+ body_start: int
195
+ body_end: int
196
+ kind: str
197
+ value: str = ""
198
+
199
+
200
+ def _quoted_end(text: str, start: int) -> int:
201
+ """Skip literals despite invalid Python; stop ordinary quotes at newlines.
202
+
203
+ Unfinished triple quotes protect the remaining scope from markup scanning.
204
+ """
205
+ quote = text[start]
206
+ delimiter = quote * (3 if text.startswith(quote * 3, start) else 1)
207
+ position = start + len(delimiter)
208
+ while position < len(text):
209
+ if text[position] == "\\":
210
+ position += 2
211
+ elif text.startswith(delimiter, position):
212
+ return position + len(delimiter)
213
+ elif len(delimiter) == 1 and text[position] == "\n":
214
+ return position
215
+ else:
216
+ position += 1
217
+ return len(text)
218
+
219
+
220
+ def _fstring_end(text: str, start: int, quote_start: int) -> int:
221
+ """Lex nested f-strings independently of earlier malformed indentation.
222
+
223
+ Python 3.11 emits STRING; 3.12+ exposes nested FSTRING_START/END tokens.
224
+ """
225
+ depth = 0
226
+ first = True
227
+ try:
228
+ for token in tokenize.generate_tokens(io.StringIO(text[start:]).readline):
229
+ if first and token.type not in (
230
+ tokenize.STRING,
231
+ getattr(tokenize, "FSTRING_START", -1),
232
+ ):
233
+ break
234
+ first = False
235
+ if token.type == getattr(tokenize, "FSTRING_START", -1):
236
+ depth += 1
237
+ elif token.type == getattr(tokenize, "FSTRING_END", -1):
238
+ depth -= 1
239
+ elif token.type != tokenize.STRING or depth:
240
+ continue
241
+ if depth == 0:
242
+ line_start = start
243
+ for _ in range(token.end[0] - 1):
244
+ line_start = text.index("\n", line_start) + 1
245
+ return line_start + token.end[1]
246
+ except (tokenize.TokenError, IndentationError, SyntaxError):
247
+ pass
248
+ return _quoted_end(text, quote_start)
249
+
250
+
251
+ def _regions(text: str, *, fences: bool = True) -> list[_Region]:
252
+ """Find outermost files/fences while ignoring literal markup.
253
+
254
+ Open tags must follow whitespace from a line start or another structural tag.
255
+ Retain completed children of unclosed parents without reusing child closers.
256
+ """
257
+ regions: list[_Region] = []
258
+ stack: list[tuple[int, int, str]] = []
259
+ position = 0
260
+ structural = True
261
+ while position < len(text):
262
+ fence = _FENCE_RE.match(text, position) if fences or stack else None
263
+ if fence:
264
+ regions.append(
265
+ _Region(
266
+ position,
267
+ fence.end(),
268
+ *fence.span("fence_body"),
269
+ "fence",
270
+ fence.group("language").strip().lower(),
271
+ )
272
+ )
273
+ position = fence.end()
274
+ structural = position == 0 or text[position - 1] == "\n"
275
+ continue
276
+ opening = _FILE_OPEN_RE.match(text, position) if structural else None
277
+ if opening:
278
+ path = next(
279
+ (
280
+ value.strip()
281
+ for key, _, value in _FILE_ATTR_RE.findall(opening.group("attrs"))
282
+ if key.lower() in {"path", "name"}
283
+ ),
284
+ "",
285
+ )
286
+ stack.append((position, opening.end(), path))
287
+ position = opening.end()
288
+ continue
289
+ closing = _FILE_CLOSE_RE.match(text, position)
290
+ if closing:
291
+ if stack:
292
+ start, body_start, path = stack.pop()
293
+ regions.append(
294
+ _Region(start, closing.end(), body_start, position, "file", path)
295
+ )
296
+ position = closing.end()
297
+ structural = True
298
+ continue
299
+ char = text[position]
300
+ fstring = (
301
+ _FSTRING_PREFIX_RE.match(text, position)
302
+ if char in "fFrR"
303
+ and (
304
+ position == 0
305
+ or not (text[position - 1].isalnum() or text[position - 1] == "_")
306
+ )
307
+ else None
308
+ )
309
+ if fstring:
310
+ position = _fstring_end(text, position, fstring.end())
311
+ structural = False
312
+ elif text.startswith("<!--", position):
313
+ end = text.find("-->", position + 4)
314
+ position = len(text) if end < 0 else end + 3
315
+ structural = False
316
+ elif char in "\"'":
317
+ position = _quoted_end(text, position)
318
+ structural = False
319
+ elif char == "#":
320
+ end = text.find("\n", position)
321
+ position = len(text) if end < 0 else end
322
+ structural = False
323
+ elif char == "`":
324
+ # Inline code is literal; unmatched ticks cannot hide later lines.
325
+ end = position + 1
326
+ while end < len(text) and text[end] == "`":
327
+ end += 1
328
+ delimiter = text[position:end]
329
+ line_end = text.find("\n", end)
330
+ close = text.find(delimiter, end, len(text) if line_end < 0 else line_end)
331
+ position = end if close < 0 else close + len(delimiter)
332
+ structural = False
333
+ else:
334
+ if char == "\n":
335
+ structural = True
336
+ elif not char.isspace():
337
+ structural = False
338
+ position += 1
339
+ outermost: list[_Region] = []
340
+ consumed = 0
341
+ for region in sorted(regions, key=lambda r: (r.start, -r.end)):
342
+ if region.start >= consumed:
343
+ outermost.append(region)
344
+ consumed = region.end
345
+ return outermost
346
+
347
+
348
+ def _filename_paths(code: str) -> set[str]:
349
+ matches = _FILENAME_RE.findall(code)
350
+ if not matches:
351
+ return set()
352
+ try:
353
+ comments = [
354
+ token.string
355
+ for token in tokenize.generate_tokens(io.StringIO(code).readline)
356
+ if token.type == tokenize.COMMENT
357
+ and not token.line[: token.start[1]].strip()
358
+ ]
359
+ except (tokenize.TokenError, IndentationError, SyntaxError):
360
+ return {path.strip() for path in matches}
361
+ return {
362
+ path.strip() for comment in comments for path in _FILENAME_RE.findall(comment)
363
+ }
364
+
365
+
366
+ def _split_without_files(
367
+ text: str, *, file_body: bool
368
+ ) -> tuple[list[str], str, set[str]]:
369
+ """Separate code fences, prose, and filename declarations; retain empty fences."""
370
+ blocks: list[str] = []
371
+ prose: list[str] = []
372
+ position = 0
373
+ matched = False
374
+ for match in _FENCE_RE.finditer(text):
375
+ matched = True
376
+ prose.append(text[position : match.start()])
377
+ position = match.end()
378
+ if match.group("language").strip().lower() in ("", "python", "python3", "py"):
379
+ blocks.append(match.group("fence_body"))
380
+ prose.append(text[position:])
381
+ if not matched and text.strip() and (file_body or _parse(text) is not None):
382
+ blocks.append(text)
383
+ prose.clear()
384
+ declared = {path for block in blocks for path in _filename_paths(block)}
385
+ return blocks, " ".join(prose), declared
386
+
387
+
388
+ def _split_response(
389
+ text: str,
390
+ *,
391
+ file_body: bool = False,
392
+ fence_body: bool = False,
393
+ _depth: int = 0,
394
+ ) -> tuple[list[str], str, set[str]]:
395
+ # Bound adversarial nesting; remaining markup becomes a parse-failing leaf.
396
+ if _depth >= _MAX_MARKUP_DEPTH:
397
+ return [text], "", set()
398
+ if not _FILE_HINT_RE.search(text):
399
+ if fence_body:
400
+ return [text], "", _filename_paths(text)
401
+ return _split_without_files(text, file_body=file_body)
402
+ if _parse(text) is not None:
403
+ # Valid Python is atomic, protecting complex f-string literals.
404
+ return [text], "", _filename_paths(text)
405
+ blocks: list[str] = []
406
+ direct: list[str] = []
407
+ prose: list[str] = []
408
+ declared: set[str] = set()
409
+ position = 0
410
+ regions = _regions(text, fences=not fence_body)
411
+ for region in regions:
412
+ gap = text[position : region.start]
413
+ direct.append(gap)
414
+ if not file_body:
415
+ prose.append(gap)
416
+ position = region.end
417
+ if region.kind == "file" or region.value in ("", "python", "python3", "py"):
418
+ if region.kind == "file" and region.value:
419
+ declared.add(region.value)
420
+ nested, nested_prose, nested_paths = _split_response(
421
+ text[region.body_start : region.body_end],
422
+ file_body=True,
423
+ fence_body=region.kind == "fence",
424
+ _depth=_depth + 1,
425
+ )
426
+ blocks.extend(nested)
427
+ if region.kind == "file" or nested_prose:
428
+ prose.append(nested_prose)
429
+ declared.update(nested_paths)
430
+ tail = text[position:]
431
+ direct.append(tail)
432
+ if file_body:
433
+ # Reassemble parent fragments without counting child spans twice.
434
+ code = "\n".join(direct)
435
+ if code.strip():
436
+ blocks.append(code)
437
+ else:
438
+ prose.append(tail)
439
+ for block in blocks:
440
+ declared.update(_filename_paths(block))
441
+ return blocks, " ".join(prose), declared
442
+
443
+
444
+ def _is_main_guard(node: ast.AST) -> bool:
445
+ if not (
446
+ isinstance(node, ast.Compare)
447
+ and len(node.ops) == 1
448
+ and isinstance(node.ops[0], ast.Eq)
449
+ ):
450
+ return False
451
+ left, right = node.left, node.comparators[0]
452
+ return (
453
+ isinstance(left, ast.Name)
454
+ and left.id == "__name__"
455
+ and isinstance(right, ast.Constant)
456
+ and right.value == "__main__"
457
+ ) or (
458
+ isinstance(right, ast.Name)
459
+ and right.id == "__name__"
460
+ and isinstance(left, ast.Constant)
461
+ and left.value == "__main__"
462
+ )
463
+
464
+
465
+ def _call_features(
466
+ node: ast.Call, counts: Counter, flags: dict, sparse: set[str]
467
+ ) -> None:
468
+ name = _callee_name(node.func)
469
+ if not name:
470
+ return
471
+ sparse.add(f"call:{name}")
472
+ if name in _EXIT_CALLS:
473
+ flags["calls_exit"] = 1.0
474
+ if name in _EXEC_CALLS:
475
+ flags["calls_exec_or_shell"] = 1.0
476
+ if name == "print":
477
+ counts["n_print"] += 1
478
+ if name in ("open", "io.open", "Path.open"):
479
+ # Path("x").open takes mode first; builtin open takes it second.
480
+ literal_path = (
481
+ isinstance(node.func, ast.Attribute)
482
+ and node.func.attr == "open"
483
+ and isinstance(node.func.value, ast.Call)
484
+ and _callee_name(node.func.value.func) in ("Path", "pathlib.Path")
485
+ )
486
+ mode_index = 0 if literal_path else 1
487
+ mode = None
488
+ if len(node.args) > mode_index and isinstance(
489
+ node.args[mode_index], ast.Constant
490
+ ):
491
+ mode = node.args[mode_index].value
492
+ for kw in node.keywords:
493
+ if kw.arg == "mode" and isinstance(kw.value, ast.Constant):
494
+ mode = kw.value.value
495
+ if isinstance(mode, str) and any(c in mode for c in "wax+"):
496
+ flags["file_write"] = 1.0
497
+ else:
498
+ flags["file_read"] = 1.0
499
+ if name.endswith(("write_text", "write_bytes", "to_csv", "dump")):
500
+ flags["file_write"] = 1.0
501
+ if name.endswith(("read_text", "read_bytes", "load")) and "json" not in name:
502
+ flags["file_read"] = 1.0
503
+
504
+
505
+ def _has_test_token(name: str) -> bool:
506
+ parts = re.sub(r"([A-Z])([A-Z][a-z])", r"\1_\2", name)
507
+ parts = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", parts)
508
+ return any(part.lower() in {"test", "tests"} for part in parts.split("_"))
509
+
510
+
511
+ def _ast_features(trees: list[ast.Module], sparse: set[str]) -> dict[str, float]:
512
+ counts = Counter()
513
+ flags = dict.fromkeys(_AST_FLAGS, 0.0)
514
+ for tree in trees:
515
+ imported: set[str] = set()
516
+ for node in ast.walk(tree):
517
+ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
518
+ counts["n_funcdefs"] += 1
519
+ name = node.name
520
+ sparse.add(f"def:{name}")
521
+ if name.startswith("__") and name.endswith("__"):
522
+ flags["defines_any_dunder"] = 1.0
523
+ if name in _CMP_DUNDERS:
524
+ flags["defines_cmp_dunder"] = 1.0
525
+ if _has_test_token(name):
526
+ flags["defines_test_token"] = 1.0
527
+ if name in _BUILTINS_LIKE:
528
+ flags["overrides_builtin_name"] = 1.0
529
+ elif isinstance(node, ast.ClassDef):
530
+ counts["n_classdefs"] += 1
531
+ flags["defines_class"] = 1.0
532
+ sparse.add(f"class:{node.name}")
533
+ elif isinstance(node, ast.Import):
534
+ for a in node.names:
535
+ mod = a.name.split(".")[0]
536
+ imported.add(mod)
537
+ sparse.add(f"import:{mod}")
538
+ counts["n_imports"] += 1
539
+ elif isinstance(node, ast.ImportFrom):
540
+ mod = (node.module or "").split(".")[0]
541
+ imported.add(mod)
542
+ sparse.add(f"import:{mod}")
543
+ counts["n_imports"] += 1
544
+ elif isinstance(node, ast.Call):
545
+ counts["n_calls"] += 1
546
+ _call_features(node, counts, flags, sparse)
547
+ elif isinstance(node, ast.Try):
548
+ counts["n_try"] += 1
549
+ for h in node.handlers:
550
+ if h.type is None or (
551
+ isinstance(h.type, ast.Name)
552
+ and h.type.id in ("Exception", "BaseException")
553
+ ):
554
+ flags["catches_all"] = 1.0
555
+ elif isinstance(node, ast.Raise):
556
+ flags["raises"] = 1.0
557
+ if (
558
+ isinstance(node.exc, ast.Call)
559
+ and _callee_name(node.exc.func) == "SystemExit"
560
+ ):
561
+ flags["calls_exit"] = 1.0
562
+ elif isinstance(node, ast.Assert):
563
+ counts["n_assert"] += 1
564
+ elif isinstance(node, ast.Return):
565
+ counts["n_return"] += 1
566
+ elif isinstance(node, ast.Assign):
567
+ for t in node.targets:
568
+ if isinstance(t, ast.Attribute):
569
+ base = _callee_name(t.value)
570
+ if base:
571
+ sparse.add(f"setattr:{base}.{t.attr}")
572
+ if base.split(".")[0] in imported:
573
+ flags["module_attr_assign"] = 1.0
574
+ elif isinstance(t, ast.Name) and t.id in _BUILTINS_LIKE:
575
+ flags["overrides_builtin_name"] = 1.0
576
+ elif isinstance(node, ast.Constant) and isinstance(node.value, str):
577
+ v = node.value.strip()
578
+ if 3 <= len(v) <= 40 and _FILE_EXT_RE.match(v):
579
+ sparse.add(f"strlit:{v}")
580
+ for st in tree.body:
581
+ if isinstance(st, ast.If):
582
+ if _is_main_guard(st.test):
583
+ flags["has_main_guard"] = 1.0
584
+ elif isinstance(st, ast.Expr) and isinstance(st.value, ast.Call):
585
+ flags["top_level_call"] = 1.0
586
+ flags["imports_sys"] = max(flags["imports_sys"], float("sys" in imported))
587
+ flags["imports_os"] = max(flags["imports_os"], float("os" in imported))
588
+ names = (
589
+ "n_funcdefs",
590
+ "n_classdefs",
591
+ "n_imports",
592
+ "n_calls",
593
+ "n_print",
594
+ "n_try",
595
+ "n_assert",
596
+ "n_return",
597
+ )
598
+ return {**{name: float(counts.get(name, 0)) for name in names}, **flags}
599
+
600
+
601
+ def extract(response: str) -> RolloutFeatures:
602
+ """Extract syntax and prose features from response text."""
603
+ if not isinstance(response, str):
604
+ raise TypeError("response must be a string")
605
+ result = RolloutFeatures()
606
+ dense, sparse = result.dense, result.sparse
607
+ text = _THINK_RE.sub(" ", response)
608
+ blocks, prose, declared_files = _split_response(text)
609
+ for path in declared_files:
610
+ sparse.add(f"file:{path.replace(chr(92), '/').split('/')[-1]}")
611
+ dense["n_files_declared"] = float(len(declared_files))
612
+ code = "\n".join(blocks)
613
+ dense["resp_chars"] = float(len(response))
614
+ dense["prose_chars"] = float(len(prose))
615
+ dense["code_chars"] = float(len(code))
616
+ dense["n_code_blocks"] = float(len(blocks))
617
+ dense["code_lines"] = float(sum(len(block.splitlines()) for block in blocks))
618
+ dense["has_code"] = float(any(block.strip() for block in blocks))
619
+ trees = []
620
+ dense["code_parse_fail"] = 0.0
621
+ for block in blocks:
622
+ tree = _parse(block)
623
+ if tree is None:
624
+ dense["code_parse_fail"] = 1.0
625
+ else:
626
+ trees.append(tree)
627
+
628
+ dense.update(_ast_features(trees, sparse))
629
+
630
+ words = [w for w in _WORD_RE.findall(prose.lower()) if w not in _STOP]
631
+ dense["prose_words"] = float(len(words))
632
+ for w in set(words):
633
+ sparse.add(f"w:{w}")
634
+ for i in range(len(words) - 1):
635
+ sparse.add(f"b:{words[i]} {words[i + 1]}")
636
+ return result
637
+
638
+
639
+ DENSE_BINARY = {"has_code", "code_parse_fail", *_AST_FLAGS}
640
+
641
+
642
+ @dataclass
643
+ class FeatureTables:
644
+ """Features keyed by (run, sample_id), plus metadata for retained features."""
645
+
646
+ dense: pd.DataFrame
647
+ sparse: pd.DataFrame
648
+ catalog: pd.DataFrame
649
+
650
+
651
+ def _tier(feature: str) -> str:
652
+ return (
653
+ "prose"
654
+ if feature.startswith(("w:", "b:")) or feature in {"prose_chars", "prose_words"}
655
+ else "structural"
656
+ )
657
+
658
+
659
+ def _select_sparse(
660
+ rollouts: pd.DataFrame,
661
+ sparse: pd.DataFrame,
662
+ max_sparse_per_tier: int,
663
+ min_prevalence: float,
664
+ ) -> list[str]:
665
+ if sparse.empty or not max_sparse_per_tier:
666
+ return []
667
+ metadata = rollouts[["run", "sample_id", "step", "prompt_id"]]
668
+ presence = sparse.merge(metadata, on=["run", "sample_id"], validate="many_to_one")
669
+ prompt_keys = ["run", "step", "prompt_id"]
670
+ denominators = (
671
+ rollouts.groupby(prompt_keys, observed=True, sort=False)
672
+ .size()
673
+ .rename("n_rollouts")
674
+ )
675
+ positive = (
676
+ presence.groupby([*prompt_keys, "feature"], observed=True, sort=False)
677
+ .size()
678
+ .rename("n_positive")
679
+ .reset_index()
680
+ )
681
+ positive = positive.merge(
682
+ denominators.reset_index(), on=prompt_keys, validate="many_to_one"
683
+ )
684
+ positive["within_prompt"] = positive.n_positive / positive.n_rollouts
685
+ n_prompts = (
686
+ denominators.groupby(level=["run", "step"], sort=False)
687
+ .size()
688
+ .rename("n_prompts")
689
+ )
690
+ prevalence = (
691
+ positive.groupby(["run", "step", "feature"], observed=True, sort=False)
692
+ .within_prompt.sum()
693
+ .reset_index()
694
+ )
695
+ prevalence = prevalence.merge(
696
+ n_prompts.reset_index(), on=["run", "step"], validate="many_to_one"
697
+ )
698
+ prevalence["prevalence"] = prevalence.within_prompt / prevalence.n_prompts
699
+ peaks = (
700
+ prevalence.groupby("feature", sort=False)
701
+ .prevalence.max()
702
+ .rename("peak")
703
+ .reset_index()
704
+ )
705
+ peaks = peaks.loc[peaks.peak >= min_prevalence].sort_values(
706
+ ["peak", "feature"], ascending=[False, True]
707
+ )
708
+ peaks["tier"] = peaks.feature.map(_tier)
709
+ return peaks.groupby("tier", sort=False).head(max_sparse_per_tier).feature.tolist()
710
+
711
+
712
+ def _validate_selection(max_sparse_per_tier: int, min_prevalence: float) -> None:
713
+ if (
714
+ isinstance(max_sparse_per_tier, bool)
715
+ or not isinstance(max_sparse_per_tier, int)
716
+ or max_sparse_per_tier < 0
717
+ ):
718
+ raise ValueError("max_sparse_per_tier must be a nonnegative integer")
719
+ if not 0 <= min_prevalence <= 1:
720
+ raise ValueError("min_prevalence must be between zero and one")
721
+
722
+
723
+ def _extract_features(
724
+ rollouts: pd.DataFrame, *, include_tool_calls: bool = False
725
+ ) -> tuple[pd.DataFrame, pd.DataFrame]:
726
+ """Extract dense values and all sparse presences before vocabulary selection."""
727
+ required = {"run", "step", "sample_id", "prompt_id", "response"}
728
+ missing = required - set(rollouts.columns)
729
+ if missing:
730
+ raise ValueError(f"missing rollout columns: {sorted(missing)}")
731
+ if rollouts[list(required)].isna().any().any():
732
+ raise ValueError("feature input keys and responses cannot be null")
733
+ if rollouts.duplicated(["run", "sample_id"]).any():
734
+ raise ValueError("duplicate rollout keys (run, sample_id)")
735
+
736
+ dense_names = list(extract("").dense)
737
+ dense_rows = []
738
+ sparse_rows = []
739
+ columns = ["run", "sample_id", "step", "prompt_id", "response"]
740
+ for run, sample, _step, _prompt, response in rollouts[columns].itertuples(
741
+ index=False, name=None
742
+ ):
743
+ result = extract(response)
744
+ dense_rows.append({"run": run, "sample_id": sample, **result.dense})
745
+ sparse_rows.extend((run, sample, token) for token in sorted(result.sparse))
746
+ dense = pd.DataFrame(dense_rows, columns=["run", "sample_id", *dense_names])
747
+ if dense.empty:
748
+ dense = dense.astype({name: "float64" for name in dense_names})
749
+ sparse = pd.DataFrame(sparse_rows, columns=["run", "sample_id", "feature"])
750
+ if include_tool_calls:
751
+ from .tool_features import extract_tool_calls
752
+
753
+ tool_rows, tool_sparse = [], []
754
+ for row in rollouts.to_dict("records"):
755
+ result = extract_tool_calls(
756
+ row.get("tool_calls"), row.get("response_messages")
757
+ )
758
+ tool_rows.append(result.dense)
759
+ tool_sparse.extend(
760
+ (row["run"], row["sample_id"], name) for name in sorted(result.sparse)
761
+ )
762
+ tools = pd.DataFrame(tool_rows).fillna(0.0)
763
+ for name in sorted(tools.columns):
764
+ dense[name] = tools[name].to_numpy(dtype=float)
765
+ if tool_sparse:
766
+ sparse = pd.concat(
767
+ [sparse, pd.DataFrame(tool_sparse, columns=sparse.columns)],
768
+ ignore_index=True,
769
+ )
770
+ return dense, sparse
771
+
772
+
773
+ def _retain_features(
774
+ rollouts: pd.DataFrame,
775
+ dense: pd.DataFrame,
776
+ raw_sparse: pd.DataFrame,
777
+ *,
778
+ max_sparse_per_tier: int = 2000,
779
+ min_prevalence: float = 0.05,
780
+ ) -> FeatureTables:
781
+ """Select a prefix vocabulary from previously extracted rollout features."""
782
+ _validate_selection(max_sparse_per_tier, min_prevalence)
783
+ dense_names = [name for name in dense.columns if name not in {"run", "sample_id"}]
784
+ sparse = raw_sparse
785
+ kept = _select_sparse(rollouts, sparse, max_sparse_per_tier, min_prevalence)
786
+ sparse = sparse.loc[sparse.feature.isin(kept)].reset_index(drop=True)
787
+ catalog = pd.DataFrame(
788
+ [
789
+ (name, _tier(name), "binary" if name in DENSE_BINARY else "continuous")
790
+ for name in dense_names
791
+ ]
792
+ + [(name, _tier(name), "binary") for name in kept],
793
+ columns=["feature", "tier", "kind"],
794
+ ).sort_values("feature", ignore_index=True)
795
+ return FeatureTables(dense=dense, sparse=sparse, catalog=catalog)
796
+
797
+
798
+ def featurize(
799
+ rollouts: pd.DataFrame,
800
+ *,
801
+ max_sparse_per_tier: int = 2000,
802
+ min_prevalence: float = 0.05,
803
+ include_tool_calls: bool = False,
804
+ ) -> FeatureTables:
805
+ """Retain sparse features by peak prompt-balanced prevalence per tier.
806
+
807
+ Average within prompts, then across all prompts at each run/step, including
808
+ zeros for absent features. Ties break lexically. Recorded tool features are
809
+ included only when requested and require complete structured call evidence.
810
+ """
811
+ _validate_selection(max_sparse_per_tier, min_prevalence)
812
+ if type(include_tool_calls) is not bool:
813
+ raise ValueError("include_tool_calls must be Boolean")
814
+ dense, raw_sparse = _extract_features(
815
+ rollouts, include_tool_calls=include_tool_calls
816
+ )
817
+ return _retain_features(
818
+ rollouts,
819
+ dense,
820
+ raw_sparse,
821
+ max_sparse_per_tier=max_sparse_per_tier,
822
+ min_prevalence=min_prevalence,
823
+ )