rolloutdiff 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rolloutdiff/__init__.py +130 -0
- rolloutdiff/_aria_replay.py +346 -0
- rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
- rolloutdiff/_dependence.py +195 -0
- rolloutdiff/_feature_values.py +79 -0
- rolloutdiff/_replay_container.py +568 -0
- rolloutdiff/_report_validation.py +758 -0
- rolloutdiff/_retrospective_policy.py +33 -0
- rolloutdiff/_sampling.py +127 -0
- rolloutdiff/_series_validation.py +67 -0
- rolloutdiff/_watch_report.py +436 -0
- rolloutdiff/_watch_storage.py +800 -0
- rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
- rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
- rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
- rolloutdiff/_web/assets/third-party-notices.txt +112 -0
- rolloutdiff/_web/index.html +14 -0
- rolloutdiff/adapters.py +264 -0
- rolloutdiff/app.py +956 -0
- rolloutdiff/app_evidence.py +678 -0
- rolloutdiff/benchmark_reference.py +458 -0
- rolloutdiff/cli.py +1292 -0
- rolloutdiff/comparison.py +572 -0
- rolloutdiff/comparison_report.py +982 -0
- rolloutdiff/corroboration.py +287 -0
- rolloutdiff/detect.py +307 -0
- rolloutdiff/events.py +173 -0
- rolloutdiff/evidence.py +285 -0
- rolloutdiff/exemplars.py +240 -0
- rolloutdiff/features.py +823 -0
- rolloutdiff/html_report.py +726 -0
- rolloutdiff/inspect_logs.py +383 -0
- rolloutdiff/inspection.py +207 -0
- rolloutdiff/integrations.py +552 -0
- rolloutdiff/investigation.py +1146 -0
- rolloutdiff/notifications.py +406 -0
- rolloutdiff/pipeline.py +809 -0
- rolloutdiff/producer.py +663 -0
- rolloutdiff/prompt_mix.py +258 -0
- rolloutdiff/ranking.py +388 -0
- rolloutdiff/replay.py +589 -0
- rolloutdiff/replication.py +572 -0
- rolloutdiff/report.py +838 -0
- rolloutdiff/retrospective.py +386 -0
- rolloutdiff/schema.py +109 -0
- rolloutdiff/sensitivity.py +202 -0
- rolloutdiff/series.py +430 -0
- rolloutdiff/tool_features.py +422 -0
- rolloutdiff/trajectories.py +236 -0
- rolloutdiff/watch-compatibility.json +13 -0
- rolloutdiff/watch.py +742 -0
- rolloutdiff-0.1.0.dist-info/METADATA +158 -0
- rolloutdiff-0.1.0.dist-info/RECORD +62 -0
- rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
- rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
- rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
- rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
rolloutdiff/features.py
ADDED
|
@@ -0,0 +1,823 @@
|
|
|
1
|
+
"""Lexical and structural response features.
|
|
2
|
+
|
|
3
|
+
Complete think/thinking sections are excluded; resp_chars retains original length.
|
|
4
|
+
AST indicators omit alias resolution, dataflow, and outcomes. Non-Python fences are
|
|
5
|
+
excluded from prose.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import ast
|
|
11
|
+
import io
|
|
12
|
+
import re
|
|
13
|
+
import tokenize
|
|
14
|
+
import warnings
|
|
15
|
+
from collections import Counter
|
|
16
|
+
from dataclasses import dataclass, field
|
|
17
|
+
|
|
18
|
+
import pandas as pd
|
|
19
|
+
|
|
20
|
+
# File tags cannot pair across fence boundaries.
|
|
21
|
+
_FENCE_RE = re.compile(
|
|
22
|
+
r"^[ \t]*(?P<fence>(?P<marker>`|~)(?P=marker){2,})(?P<language>[^\n]*)\n"
|
|
23
|
+
r"(?P<fence_body>.*?)(?:^[ \t]*(?P=fence)(?P=marker)*[ \t]*(?:\n|$)|\Z)",
|
|
24
|
+
re.DOTALL | re.MULTILINE | re.IGNORECASE,
|
|
25
|
+
)
|
|
26
|
+
_FILE_OPEN_RE = re.compile(
|
|
27
|
+
r"<file\b(?P<attrs>(?:[^>'\"]|'[^']*'|\"[^\"]*\")*)>", re.IGNORECASE
|
|
28
|
+
)
|
|
29
|
+
_FILE_CLOSE_RE = re.compile(r"</file\s*>", re.IGNORECASE)
|
|
30
|
+
_FILE_ATTR_RE = re.compile(r"(?:^|\s)([\w:.-]+)\s*=\s*(['\"])(.*?)\2", re.DOTALL)
|
|
31
|
+
_FILE_HINT_RE = re.compile(r"</?file\b", re.IGNORECASE)
|
|
32
|
+
_MAX_MARKUP_DEPTH = 64
|
|
33
|
+
_FSTRING_PREFIX_RE = re.compile(r"(?:fr|rf|f)(?=['\"])", re.IGNORECASE)
|
|
34
|
+
_FILENAME_RE = re.compile(
|
|
35
|
+
r"^[ \t]*#[ \t]*filename[ \t]*:[ \t]*(.+?)[ \t]*$", re.MULTILINE | re.IGNORECASE
|
|
36
|
+
)
|
|
37
|
+
_THINK_RE = re.compile(r"<(think|thinking)\b[^>]*>.*?</\1>", re.DOTALL | re.IGNORECASE)
|
|
38
|
+
_WORD_RE = re.compile(r"\b[a-zA-Z]{3,}\b")
|
|
39
|
+
_STOP = frozenset(
|
|
40
|
+
[
|
|
41
|
+
"the",
|
|
42
|
+
"and",
|
|
43
|
+
"for",
|
|
44
|
+
"that",
|
|
45
|
+
"with",
|
|
46
|
+
"you",
|
|
47
|
+
"this",
|
|
48
|
+
"are",
|
|
49
|
+
"can",
|
|
50
|
+
"not",
|
|
51
|
+
"your",
|
|
52
|
+
"from",
|
|
53
|
+
"have",
|
|
54
|
+
"but",
|
|
55
|
+
"will",
|
|
56
|
+
"what",
|
|
57
|
+
"how",
|
|
58
|
+
"all",
|
|
59
|
+
"out",
|
|
60
|
+
"use",
|
|
61
|
+
"any",
|
|
62
|
+
"has",
|
|
63
|
+
"also",
|
|
64
|
+
"more",
|
|
65
|
+
"make",
|
|
66
|
+
"which",
|
|
67
|
+
"then",
|
|
68
|
+
"there",
|
|
69
|
+
"their",
|
|
70
|
+
"about",
|
|
71
|
+
"when",
|
|
72
|
+
"some",
|
|
73
|
+
"into",
|
|
74
|
+
"other",
|
|
75
|
+
"just",
|
|
76
|
+
"its",
|
|
77
|
+
"only",
|
|
78
|
+
"these",
|
|
79
|
+
"could",
|
|
80
|
+
"would",
|
|
81
|
+
"should",
|
|
82
|
+
"been",
|
|
83
|
+
"very",
|
|
84
|
+
"who",
|
|
85
|
+
"where",
|
|
86
|
+
"does",
|
|
87
|
+
"did",
|
|
88
|
+
"they",
|
|
89
|
+
"them",
|
|
90
|
+
"than",
|
|
91
|
+
"because",
|
|
92
|
+
"while",
|
|
93
|
+
"after",
|
|
94
|
+
"such",
|
|
95
|
+
"through",
|
|
96
|
+
"between",
|
|
97
|
+
"being",
|
|
98
|
+
"most",
|
|
99
|
+
"here",
|
|
100
|
+
"much",
|
|
101
|
+
"those",
|
|
102
|
+
"well",
|
|
103
|
+
"one",
|
|
104
|
+
]
|
|
105
|
+
)
|
|
106
|
+
_EXIT_CALLS = {"sys.exit", "os._exit", "exit", "quit", "os.abort"}
|
|
107
|
+
_EXEC_CALLS = {
|
|
108
|
+
"exec",
|
|
109
|
+
"eval",
|
|
110
|
+
"compile",
|
|
111
|
+
"subprocess.run",
|
|
112
|
+
"subprocess.call",
|
|
113
|
+
"subprocess.Popen",
|
|
114
|
+
"os.system",
|
|
115
|
+
"os.popen",
|
|
116
|
+
}
|
|
117
|
+
_CMP_DUNDERS = {
|
|
118
|
+
"__eq__",
|
|
119
|
+
"__ne__",
|
|
120
|
+
"__lt__",
|
|
121
|
+
"__le__",
|
|
122
|
+
"__gt__",
|
|
123
|
+
"__ge__",
|
|
124
|
+
"__hash__",
|
|
125
|
+
"__bool__",
|
|
126
|
+
"__contains__",
|
|
127
|
+
}
|
|
128
|
+
_FILE_EXT_RE = re.compile(r"^[\w\-./]+\.(py|txt|json|cfg|ini|toml|yaml|yml|sh|csv)$")
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
_AST_FLAGS = (
|
|
132
|
+
"calls_exit",
|
|
133
|
+
"calls_exec_or_shell",
|
|
134
|
+
"file_write",
|
|
135
|
+
"file_read",
|
|
136
|
+
"defines_cmp_dunder",
|
|
137
|
+
"defines_any_dunder",
|
|
138
|
+
"defines_test_token",
|
|
139
|
+
"module_attr_assign",
|
|
140
|
+
"has_main_guard",
|
|
141
|
+
"top_level_call",
|
|
142
|
+
"imports_sys",
|
|
143
|
+
"imports_os",
|
|
144
|
+
"raises",
|
|
145
|
+
"catches_all",
|
|
146
|
+
"defines_class",
|
|
147
|
+
"overrides_builtin_name",
|
|
148
|
+
)
|
|
149
|
+
_BUILTINS_LIKE = {
|
|
150
|
+
"print",
|
|
151
|
+
"input",
|
|
152
|
+
"open",
|
|
153
|
+
"len",
|
|
154
|
+
"range",
|
|
155
|
+
"exit",
|
|
156
|
+
"quit",
|
|
157
|
+
"any",
|
|
158
|
+
"all",
|
|
159
|
+
"sorted",
|
|
160
|
+
"map",
|
|
161
|
+
"filter",
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
@dataclass
|
|
166
|
+
class RolloutFeatures:
|
|
167
|
+
dense: dict[str, float] = field(default_factory=dict)
|
|
168
|
+
sparse: set[str] = field(default_factory=set)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _callee_name(node: ast.AST) -> str | None:
|
|
172
|
+
if isinstance(node, ast.Name):
|
|
173
|
+
return node.id
|
|
174
|
+
if isinstance(node, ast.Attribute):
|
|
175
|
+
base = _callee_name(node.value)
|
|
176
|
+
return f"{base}.{node.attr}" if base else node.attr
|
|
177
|
+
return None
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _parse(code: str) -> ast.Module | None:
|
|
181
|
+
try:
|
|
182
|
+
with warnings.catch_warnings():
|
|
183
|
+
warnings.simplefilter("ignore", SyntaxWarning)
|
|
184
|
+
warnings.simplefilter("ignore", DeprecationWarning)
|
|
185
|
+
return ast.parse(code)
|
|
186
|
+
except (SyntaxError, ValueError, RecursionError):
|
|
187
|
+
return None
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
@dataclass(frozen=True)
|
|
191
|
+
class _Region:
|
|
192
|
+
start: int
|
|
193
|
+
end: int
|
|
194
|
+
body_start: int
|
|
195
|
+
body_end: int
|
|
196
|
+
kind: str
|
|
197
|
+
value: str = ""
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _quoted_end(text: str, start: int) -> int:
|
|
201
|
+
"""Skip literals despite invalid Python; stop ordinary quotes at newlines.
|
|
202
|
+
|
|
203
|
+
Unfinished triple quotes protect the remaining scope from markup scanning.
|
|
204
|
+
"""
|
|
205
|
+
quote = text[start]
|
|
206
|
+
delimiter = quote * (3 if text.startswith(quote * 3, start) else 1)
|
|
207
|
+
position = start + len(delimiter)
|
|
208
|
+
while position < len(text):
|
|
209
|
+
if text[position] == "\\":
|
|
210
|
+
position += 2
|
|
211
|
+
elif text.startswith(delimiter, position):
|
|
212
|
+
return position + len(delimiter)
|
|
213
|
+
elif len(delimiter) == 1 and text[position] == "\n":
|
|
214
|
+
return position
|
|
215
|
+
else:
|
|
216
|
+
position += 1
|
|
217
|
+
return len(text)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def _fstring_end(text: str, start: int, quote_start: int) -> int:
|
|
221
|
+
"""Lex nested f-strings independently of earlier malformed indentation.
|
|
222
|
+
|
|
223
|
+
Python 3.11 emits STRING; 3.12+ exposes nested FSTRING_START/END tokens.
|
|
224
|
+
"""
|
|
225
|
+
depth = 0
|
|
226
|
+
first = True
|
|
227
|
+
try:
|
|
228
|
+
for token in tokenize.generate_tokens(io.StringIO(text[start:]).readline):
|
|
229
|
+
if first and token.type not in (
|
|
230
|
+
tokenize.STRING,
|
|
231
|
+
getattr(tokenize, "FSTRING_START", -1),
|
|
232
|
+
):
|
|
233
|
+
break
|
|
234
|
+
first = False
|
|
235
|
+
if token.type == getattr(tokenize, "FSTRING_START", -1):
|
|
236
|
+
depth += 1
|
|
237
|
+
elif token.type == getattr(tokenize, "FSTRING_END", -1):
|
|
238
|
+
depth -= 1
|
|
239
|
+
elif token.type != tokenize.STRING or depth:
|
|
240
|
+
continue
|
|
241
|
+
if depth == 0:
|
|
242
|
+
line_start = start
|
|
243
|
+
for _ in range(token.end[0] - 1):
|
|
244
|
+
line_start = text.index("\n", line_start) + 1
|
|
245
|
+
return line_start + token.end[1]
|
|
246
|
+
except (tokenize.TokenError, IndentationError, SyntaxError):
|
|
247
|
+
pass
|
|
248
|
+
return _quoted_end(text, quote_start)
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def _regions(text: str, *, fences: bool = True) -> list[_Region]:
|
|
252
|
+
"""Find outermost files/fences while ignoring literal markup.
|
|
253
|
+
|
|
254
|
+
Open tags must follow whitespace from a line start or another structural tag.
|
|
255
|
+
Retain completed children of unclosed parents without reusing child closers.
|
|
256
|
+
"""
|
|
257
|
+
regions: list[_Region] = []
|
|
258
|
+
stack: list[tuple[int, int, str]] = []
|
|
259
|
+
position = 0
|
|
260
|
+
structural = True
|
|
261
|
+
while position < len(text):
|
|
262
|
+
fence = _FENCE_RE.match(text, position) if fences or stack else None
|
|
263
|
+
if fence:
|
|
264
|
+
regions.append(
|
|
265
|
+
_Region(
|
|
266
|
+
position,
|
|
267
|
+
fence.end(),
|
|
268
|
+
*fence.span("fence_body"),
|
|
269
|
+
"fence",
|
|
270
|
+
fence.group("language").strip().lower(),
|
|
271
|
+
)
|
|
272
|
+
)
|
|
273
|
+
position = fence.end()
|
|
274
|
+
structural = position == 0 or text[position - 1] == "\n"
|
|
275
|
+
continue
|
|
276
|
+
opening = _FILE_OPEN_RE.match(text, position) if structural else None
|
|
277
|
+
if opening:
|
|
278
|
+
path = next(
|
|
279
|
+
(
|
|
280
|
+
value.strip()
|
|
281
|
+
for key, _, value in _FILE_ATTR_RE.findall(opening.group("attrs"))
|
|
282
|
+
if key.lower() in {"path", "name"}
|
|
283
|
+
),
|
|
284
|
+
"",
|
|
285
|
+
)
|
|
286
|
+
stack.append((position, opening.end(), path))
|
|
287
|
+
position = opening.end()
|
|
288
|
+
continue
|
|
289
|
+
closing = _FILE_CLOSE_RE.match(text, position)
|
|
290
|
+
if closing:
|
|
291
|
+
if stack:
|
|
292
|
+
start, body_start, path = stack.pop()
|
|
293
|
+
regions.append(
|
|
294
|
+
_Region(start, closing.end(), body_start, position, "file", path)
|
|
295
|
+
)
|
|
296
|
+
position = closing.end()
|
|
297
|
+
structural = True
|
|
298
|
+
continue
|
|
299
|
+
char = text[position]
|
|
300
|
+
fstring = (
|
|
301
|
+
_FSTRING_PREFIX_RE.match(text, position)
|
|
302
|
+
if char in "fFrR"
|
|
303
|
+
and (
|
|
304
|
+
position == 0
|
|
305
|
+
or not (text[position - 1].isalnum() or text[position - 1] == "_")
|
|
306
|
+
)
|
|
307
|
+
else None
|
|
308
|
+
)
|
|
309
|
+
if fstring:
|
|
310
|
+
position = _fstring_end(text, position, fstring.end())
|
|
311
|
+
structural = False
|
|
312
|
+
elif text.startswith("<!--", position):
|
|
313
|
+
end = text.find("-->", position + 4)
|
|
314
|
+
position = len(text) if end < 0 else end + 3
|
|
315
|
+
structural = False
|
|
316
|
+
elif char in "\"'":
|
|
317
|
+
position = _quoted_end(text, position)
|
|
318
|
+
structural = False
|
|
319
|
+
elif char == "#":
|
|
320
|
+
end = text.find("\n", position)
|
|
321
|
+
position = len(text) if end < 0 else end
|
|
322
|
+
structural = False
|
|
323
|
+
elif char == "`":
|
|
324
|
+
# Inline code is literal; unmatched ticks cannot hide later lines.
|
|
325
|
+
end = position + 1
|
|
326
|
+
while end < len(text) and text[end] == "`":
|
|
327
|
+
end += 1
|
|
328
|
+
delimiter = text[position:end]
|
|
329
|
+
line_end = text.find("\n", end)
|
|
330
|
+
close = text.find(delimiter, end, len(text) if line_end < 0 else line_end)
|
|
331
|
+
position = end if close < 0 else close + len(delimiter)
|
|
332
|
+
structural = False
|
|
333
|
+
else:
|
|
334
|
+
if char == "\n":
|
|
335
|
+
structural = True
|
|
336
|
+
elif not char.isspace():
|
|
337
|
+
structural = False
|
|
338
|
+
position += 1
|
|
339
|
+
outermost: list[_Region] = []
|
|
340
|
+
consumed = 0
|
|
341
|
+
for region in sorted(regions, key=lambda r: (r.start, -r.end)):
|
|
342
|
+
if region.start >= consumed:
|
|
343
|
+
outermost.append(region)
|
|
344
|
+
consumed = region.end
|
|
345
|
+
return outermost
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def _filename_paths(code: str) -> set[str]:
|
|
349
|
+
matches = _FILENAME_RE.findall(code)
|
|
350
|
+
if not matches:
|
|
351
|
+
return set()
|
|
352
|
+
try:
|
|
353
|
+
comments = [
|
|
354
|
+
token.string
|
|
355
|
+
for token in tokenize.generate_tokens(io.StringIO(code).readline)
|
|
356
|
+
if token.type == tokenize.COMMENT
|
|
357
|
+
and not token.line[: token.start[1]].strip()
|
|
358
|
+
]
|
|
359
|
+
except (tokenize.TokenError, IndentationError, SyntaxError):
|
|
360
|
+
return {path.strip() for path in matches}
|
|
361
|
+
return {
|
|
362
|
+
path.strip() for comment in comments for path in _FILENAME_RE.findall(comment)
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def _split_without_files(
|
|
367
|
+
text: str, *, file_body: bool
|
|
368
|
+
) -> tuple[list[str], str, set[str]]:
|
|
369
|
+
"""Separate code fences, prose, and filename declarations; retain empty fences."""
|
|
370
|
+
blocks: list[str] = []
|
|
371
|
+
prose: list[str] = []
|
|
372
|
+
position = 0
|
|
373
|
+
matched = False
|
|
374
|
+
for match in _FENCE_RE.finditer(text):
|
|
375
|
+
matched = True
|
|
376
|
+
prose.append(text[position : match.start()])
|
|
377
|
+
position = match.end()
|
|
378
|
+
if match.group("language").strip().lower() in ("", "python", "python3", "py"):
|
|
379
|
+
blocks.append(match.group("fence_body"))
|
|
380
|
+
prose.append(text[position:])
|
|
381
|
+
if not matched and text.strip() and (file_body or _parse(text) is not None):
|
|
382
|
+
blocks.append(text)
|
|
383
|
+
prose.clear()
|
|
384
|
+
declared = {path for block in blocks for path in _filename_paths(block)}
|
|
385
|
+
return blocks, " ".join(prose), declared
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
def _split_response(
|
|
389
|
+
text: str,
|
|
390
|
+
*,
|
|
391
|
+
file_body: bool = False,
|
|
392
|
+
fence_body: bool = False,
|
|
393
|
+
_depth: int = 0,
|
|
394
|
+
) -> tuple[list[str], str, set[str]]:
|
|
395
|
+
# Bound adversarial nesting; remaining markup becomes a parse-failing leaf.
|
|
396
|
+
if _depth >= _MAX_MARKUP_DEPTH:
|
|
397
|
+
return [text], "", set()
|
|
398
|
+
if not _FILE_HINT_RE.search(text):
|
|
399
|
+
if fence_body:
|
|
400
|
+
return [text], "", _filename_paths(text)
|
|
401
|
+
return _split_without_files(text, file_body=file_body)
|
|
402
|
+
if _parse(text) is not None:
|
|
403
|
+
# Valid Python is atomic, protecting complex f-string literals.
|
|
404
|
+
return [text], "", _filename_paths(text)
|
|
405
|
+
blocks: list[str] = []
|
|
406
|
+
direct: list[str] = []
|
|
407
|
+
prose: list[str] = []
|
|
408
|
+
declared: set[str] = set()
|
|
409
|
+
position = 0
|
|
410
|
+
regions = _regions(text, fences=not fence_body)
|
|
411
|
+
for region in regions:
|
|
412
|
+
gap = text[position : region.start]
|
|
413
|
+
direct.append(gap)
|
|
414
|
+
if not file_body:
|
|
415
|
+
prose.append(gap)
|
|
416
|
+
position = region.end
|
|
417
|
+
if region.kind == "file" or region.value in ("", "python", "python3", "py"):
|
|
418
|
+
if region.kind == "file" and region.value:
|
|
419
|
+
declared.add(region.value)
|
|
420
|
+
nested, nested_prose, nested_paths = _split_response(
|
|
421
|
+
text[region.body_start : region.body_end],
|
|
422
|
+
file_body=True,
|
|
423
|
+
fence_body=region.kind == "fence",
|
|
424
|
+
_depth=_depth + 1,
|
|
425
|
+
)
|
|
426
|
+
blocks.extend(nested)
|
|
427
|
+
if region.kind == "file" or nested_prose:
|
|
428
|
+
prose.append(nested_prose)
|
|
429
|
+
declared.update(nested_paths)
|
|
430
|
+
tail = text[position:]
|
|
431
|
+
direct.append(tail)
|
|
432
|
+
if file_body:
|
|
433
|
+
# Reassemble parent fragments without counting child spans twice.
|
|
434
|
+
code = "\n".join(direct)
|
|
435
|
+
if code.strip():
|
|
436
|
+
blocks.append(code)
|
|
437
|
+
else:
|
|
438
|
+
prose.append(tail)
|
|
439
|
+
for block in blocks:
|
|
440
|
+
declared.update(_filename_paths(block))
|
|
441
|
+
return blocks, " ".join(prose), declared
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
def _is_main_guard(node: ast.AST) -> bool:
|
|
445
|
+
if not (
|
|
446
|
+
isinstance(node, ast.Compare)
|
|
447
|
+
and len(node.ops) == 1
|
|
448
|
+
and isinstance(node.ops[0], ast.Eq)
|
|
449
|
+
):
|
|
450
|
+
return False
|
|
451
|
+
left, right = node.left, node.comparators[0]
|
|
452
|
+
return (
|
|
453
|
+
isinstance(left, ast.Name)
|
|
454
|
+
and left.id == "__name__"
|
|
455
|
+
and isinstance(right, ast.Constant)
|
|
456
|
+
and right.value == "__main__"
|
|
457
|
+
) or (
|
|
458
|
+
isinstance(right, ast.Name)
|
|
459
|
+
and right.id == "__name__"
|
|
460
|
+
and isinstance(left, ast.Constant)
|
|
461
|
+
and left.value == "__main__"
|
|
462
|
+
)
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
def _call_features(
|
|
466
|
+
node: ast.Call, counts: Counter, flags: dict, sparse: set[str]
|
|
467
|
+
) -> None:
|
|
468
|
+
name = _callee_name(node.func)
|
|
469
|
+
if not name:
|
|
470
|
+
return
|
|
471
|
+
sparse.add(f"call:{name}")
|
|
472
|
+
if name in _EXIT_CALLS:
|
|
473
|
+
flags["calls_exit"] = 1.0
|
|
474
|
+
if name in _EXEC_CALLS:
|
|
475
|
+
flags["calls_exec_or_shell"] = 1.0
|
|
476
|
+
if name == "print":
|
|
477
|
+
counts["n_print"] += 1
|
|
478
|
+
if name in ("open", "io.open", "Path.open"):
|
|
479
|
+
# Path("x").open takes mode first; builtin open takes it second.
|
|
480
|
+
literal_path = (
|
|
481
|
+
isinstance(node.func, ast.Attribute)
|
|
482
|
+
and node.func.attr == "open"
|
|
483
|
+
and isinstance(node.func.value, ast.Call)
|
|
484
|
+
and _callee_name(node.func.value.func) in ("Path", "pathlib.Path")
|
|
485
|
+
)
|
|
486
|
+
mode_index = 0 if literal_path else 1
|
|
487
|
+
mode = None
|
|
488
|
+
if len(node.args) > mode_index and isinstance(
|
|
489
|
+
node.args[mode_index], ast.Constant
|
|
490
|
+
):
|
|
491
|
+
mode = node.args[mode_index].value
|
|
492
|
+
for kw in node.keywords:
|
|
493
|
+
if kw.arg == "mode" and isinstance(kw.value, ast.Constant):
|
|
494
|
+
mode = kw.value.value
|
|
495
|
+
if isinstance(mode, str) and any(c in mode for c in "wax+"):
|
|
496
|
+
flags["file_write"] = 1.0
|
|
497
|
+
else:
|
|
498
|
+
flags["file_read"] = 1.0
|
|
499
|
+
if name.endswith(("write_text", "write_bytes", "to_csv", "dump")):
|
|
500
|
+
flags["file_write"] = 1.0
|
|
501
|
+
if name.endswith(("read_text", "read_bytes", "load")) and "json" not in name:
|
|
502
|
+
flags["file_read"] = 1.0
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
def _has_test_token(name: str) -> bool:
|
|
506
|
+
parts = re.sub(r"([A-Z])([A-Z][a-z])", r"\1_\2", name)
|
|
507
|
+
parts = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", parts)
|
|
508
|
+
return any(part.lower() in {"test", "tests"} for part in parts.split("_"))
|
|
509
|
+
|
|
510
|
+
|
|
511
|
+
def _ast_features(trees: list[ast.Module], sparse: set[str]) -> dict[str, float]:
|
|
512
|
+
counts = Counter()
|
|
513
|
+
flags = dict.fromkeys(_AST_FLAGS, 0.0)
|
|
514
|
+
for tree in trees:
|
|
515
|
+
imported: set[str] = set()
|
|
516
|
+
for node in ast.walk(tree):
|
|
517
|
+
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
518
|
+
counts["n_funcdefs"] += 1
|
|
519
|
+
name = node.name
|
|
520
|
+
sparse.add(f"def:{name}")
|
|
521
|
+
if name.startswith("__") and name.endswith("__"):
|
|
522
|
+
flags["defines_any_dunder"] = 1.0
|
|
523
|
+
if name in _CMP_DUNDERS:
|
|
524
|
+
flags["defines_cmp_dunder"] = 1.0
|
|
525
|
+
if _has_test_token(name):
|
|
526
|
+
flags["defines_test_token"] = 1.0
|
|
527
|
+
if name in _BUILTINS_LIKE:
|
|
528
|
+
flags["overrides_builtin_name"] = 1.0
|
|
529
|
+
elif isinstance(node, ast.ClassDef):
|
|
530
|
+
counts["n_classdefs"] += 1
|
|
531
|
+
flags["defines_class"] = 1.0
|
|
532
|
+
sparse.add(f"class:{node.name}")
|
|
533
|
+
elif isinstance(node, ast.Import):
|
|
534
|
+
for a in node.names:
|
|
535
|
+
mod = a.name.split(".")[0]
|
|
536
|
+
imported.add(mod)
|
|
537
|
+
sparse.add(f"import:{mod}")
|
|
538
|
+
counts["n_imports"] += 1
|
|
539
|
+
elif isinstance(node, ast.ImportFrom):
|
|
540
|
+
mod = (node.module or "").split(".")[0]
|
|
541
|
+
imported.add(mod)
|
|
542
|
+
sparse.add(f"import:{mod}")
|
|
543
|
+
counts["n_imports"] += 1
|
|
544
|
+
elif isinstance(node, ast.Call):
|
|
545
|
+
counts["n_calls"] += 1
|
|
546
|
+
_call_features(node, counts, flags, sparse)
|
|
547
|
+
elif isinstance(node, ast.Try):
|
|
548
|
+
counts["n_try"] += 1
|
|
549
|
+
for h in node.handlers:
|
|
550
|
+
if h.type is None or (
|
|
551
|
+
isinstance(h.type, ast.Name)
|
|
552
|
+
and h.type.id in ("Exception", "BaseException")
|
|
553
|
+
):
|
|
554
|
+
flags["catches_all"] = 1.0
|
|
555
|
+
elif isinstance(node, ast.Raise):
|
|
556
|
+
flags["raises"] = 1.0
|
|
557
|
+
if (
|
|
558
|
+
isinstance(node.exc, ast.Call)
|
|
559
|
+
and _callee_name(node.exc.func) == "SystemExit"
|
|
560
|
+
):
|
|
561
|
+
flags["calls_exit"] = 1.0
|
|
562
|
+
elif isinstance(node, ast.Assert):
|
|
563
|
+
counts["n_assert"] += 1
|
|
564
|
+
elif isinstance(node, ast.Return):
|
|
565
|
+
counts["n_return"] += 1
|
|
566
|
+
elif isinstance(node, ast.Assign):
|
|
567
|
+
for t in node.targets:
|
|
568
|
+
if isinstance(t, ast.Attribute):
|
|
569
|
+
base = _callee_name(t.value)
|
|
570
|
+
if base:
|
|
571
|
+
sparse.add(f"setattr:{base}.{t.attr}")
|
|
572
|
+
if base.split(".")[0] in imported:
|
|
573
|
+
flags["module_attr_assign"] = 1.0
|
|
574
|
+
elif isinstance(t, ast.Name) and t.id in _BUILTINS_LIKE:
|
|
575
|
+
flags["overrides_builtin_name"] = 1.0
|
|
576
|
+
elif isinstance(node, ast.Constant) and isinstance(node.value, str):
|
|
577
|
+
v = node.value.strip()
|
|
578
|
+
if 3 <= len(v) <= 40 and _FILE_EXT_RE.match(v):
|
|
579
|
+
sparse.add(f"strlit:{v}")
|
|
580
|
+
for st in tree.body:
|
|
581
|
+
if isinstance(st, ast.If):
|
|
582
|
+
if _is_main_guard(st.test):
|
|
583
|
+
flags["has_main_guard"] = 1.0
|
|
584
|
+
elif isinstance(st, ast.Expr) and isinstance(st.value, ast.Call):
|
|
585
|
+
flags["top_level_call"] = 1.0
|
|
586
|
+
flags["imports_sys"] = max(flags["imports_sys"], float("sys" in imported))
|
|
587
|
+
flags["imports_os"] = max(flags["imports_os"], float("os" in imported))
|
|
588
|
+
names = (
|
|
589
|
+
"n_funcdefs",
|
|
590
|
+
"n_classdefs",
|
|
591
|
+
"n_imports",
|
|
592
|
+
"n_calls",
|
|
593
|
+
"n_print",
|
|
594
|
+
"n_try",
|
|
595
|
+
"n_assert",
|
|
596
|
+
"n_return",
|
|
597
|
+
)
|
|
598
|
+
return {**{name: float(counts.get(name, 0)) for name in names}, **flags}
|
|
599
|
+
|
|
600
|
+
|
|
601
|
+
def extract(response: str) -> RolloutFeatures:
|
|
602
|
+
"""Extract syntax and prose features from response text."""
|
|
603
|
+
if not isinstance(response, str):
|
|
604
|
+
raise TypeError("response must be a string")
|
|
605
|
+
result = RolloutFeatures()
|
|
606
|
+
dense, sparse = result.dense, result.sparse
|
|
607
|
+
text = _THINK_RE.sub(" ", response)
|
|
608
|
+
blocks, prose, declared_files = _split_response(text)
|
|
609
|
+
for path in declared_files:
|
|
610
|
+
sparse.add(f"file:{path.replace(chr(92), '/').split('/')[-1]}")
|
|
611
|
+
dense["n_files_declared"] = float(len(declared_files))
|
|
612
|
+
code = "\n".join(blocks)
|
|
613
|
+
dense["resp_chars"] = float(len(response))
|
|
614
|
+
dense["prose_chars"] = float(len(prose))
|
|
615
|
+
dense["code_chars"] = float(len(code))
|
|
616
|
+
dense["n_code_blocks"] = float(len(blocks))
|
|
617
|
+
dense["code_lines"] = float(sum(len(block.splitlines()) for block in blocks))
|
|
618
|
+
dense["has_code"] = float(any(block.strip() for block in blocks))
|
|
619
|
+
trees = []
|
|
620
|
+
dense["code_parse_fail"] = 0.0
|
|
621
|
+
for block in blocks:
|
|
622
|
+
tree = _parse(block)
|
|
623
|
+
if tree is None:
|
|
624
|
+
dense["code_parse_fail"] = 1.0
|
|
625
|
+
else:
|
|
626
|
+
trees.append(tree)
|
|
627
|
+
|
|
628
|
+
dense.update(_ast_features(trees, sparse))
|
|
629
|
+
|
|
630
|
+
words = [w for w in _WORD_RE.findall(prose.lower()) if w not in _STOP]
|
|
631
|
+
dense["prose_words"] = float(len(words))
|
|
632
|
+
for w in set(words):
|
|
633
|
+
sparse.add(f"w:{w}")
|
|
634
|
+
for i in range(len(words) - 1):
|
|
635
|
+
sparse.add(f"b:{words[i]} {words[i + 1]}")
|
|
636
|
+
return result
|
|
637
|
+
|
|
638
|
+
|
|
639
|
+
DENSE_BINARY = {"has_code", "code_parse_fail", *_AST_FLAGS}
|
|
640
|
+
|
|
641
|
+
|
|
642
|
+
@dataclass
|
|
643
|
+
class FeatureTables:
|
|
644
|
+
"""Features keyed by (run, sample_id), plus metadata for retained features."""
|
|
645
|
+
|
|
646
|
+
dense: pd.DataFrame
|
|
647
|
+
sparse: pd.DataFrame
|
|
648
|
+
catalog: pd.DataFrame
|
|
649
|
+
|
|
650
|
+
|
|
651
|
+
def _tier(feature: str) -> str:
|
|
652
|
+
return (
|
|
653
|
+
"prose"
|
|
654
|
+
if feature.startswith(("w:", "b:")) or feature in {"prose_chars", "prose_words"}
|
|
655
|
+
else "structural"
|
|
656
|
+
)
|
|
657
|
+
|
|
658
|
+
|
|
659
|
+
def _select_sparse(
|
|
660
|
+
rollouts: pd.DataFrame,
|
|
661
|
+
sparse: pd.DataFrame,
|
|
662
|
+
max_sparse_per_tier: int,
|
|
663
|
+
min_prevalence: float,
|
|
664
|
+
) -> list[str]:
|
|
665
|
+
if sparse.empty or not max_sparse_per_tier:
|
|
666
|
+
return []
|
|
667
|
+
metadata = rollouts[["run", "sample_id", "step", "prompt_id"]]
|
|
668
|
+
presence = sparse.merge(metadata, on=["run", "sample_id"], validate="many_to_one")
|
|
669
|
+
prompt_keys = ["run", "step", "prompt_id"]
|
|
670
|
+
denominators = (
|
|
671
|
+
rollouts.groupby(prompt_keys, observed=True, sort=False)
|
|
672
|
+
.size()
|
|
673
|
+
.rename("n_rollouts")
|
|
674
|
+
)
|
|
675
|
+
positive = (
|
|
676
|
+
presence.groupby([*prompt_keys, "feature"], observed=True, sort=False)
|
|
677
|
+
.size()
|
|
678
|
+
.rename("n_positive")
|
|
679
|
+
.reset_index()
|
|
680
|
+
)
|
|
681
|
+
positive = positive.merge(
|
|
682
|
+
denominators.reset_index(), on=prompt_keys, validate="many_to_one"
|
|
683
|
+
)
|
|
684
|
+
positive["within_prompt"] = positive.n_positive / positive.n_rollouts
|
|
685
|
+
n_prompts = (
|
|
686
|
+
denominators.groupby(level=["run", "step"], sort=False)
|
|
687
|
+
.size()
|
|
688
|
+
.rename("n_prompts")
|
|
689
|
+
)
|
|
690
|
+
prevalence = (
|
|
691
|
+
positive.groupby(["run", "step", "feature"], observed=True, sort=False)
|
|
692
|
+
.within_prompt.sum()
|
|
693
|
+
.reset_index()
|
|
694
|
+
)
|
|
695
|
+
prevalence = prevalence.merge(
|
|
696
|
+
n_prompts.reset_index(), on=["run", "step"], validate="many_to_one"
|
|
697
|
+
)
|
|
698
|
+
prevalence["prevalence"] = prevalence.within_prompt / prevalence.n_prompts
|
|
699
|
+
peaks = (
|
|
700
|
+
prevalence.groupby("feature", sort=False)
|
|
701
|
+
.prevalence.max()
|
|
702
|
+
.rename("peak")
|
|
703
|
+
.reset_index()
|
|
704
|
+
)
|
|
705
|
+
peaks = peaks.loc[peaks.peak >= min_prevalence].sort_values(
|
|
706
|
+
["peak", "feature"], ascending=[False, True]
|
|
707
|
+
)
|
|
708
|
+
peaks["tier"] = peaks.feature.map(_tier)
|
|
709
|
+
return peaks.groupby("tier", sort=False).head(max_sparse_per_tier).feature.tolist()
|
|
710
|
+
|
|
711
|
+
|
|
712
|
+
def _validate_selection(max_sparse_per_tier: int, min_prevalence: float) -> None:
|
|
713
|
+
if (
|
|
714
|
+
isinstance(max_sparse_per_tier, bool)
|
|
715
|
+
or not isinstance(max_sparse_per_tier, int)
|
|
716
|
+
or max_sparse_per_tier < 0
|
|
717
|
+
):
|
|
718
|
+
raise ValueError("max_sparse_per_tier must be a nonnegative integer")
|
|
719
|
+
if not 0 <= min_prevalence <= 1:
|
|
720
|
+
raise ValueError("min_prevalence must be between zero and one")
|
|
721
|
+
|
|
722
|
+
|
|
723
|
+
def _extract_features(
|
|
724
|
+
rollouts: pd.DataFrame, *, include_tool_calls: bool = False
|
|
725
|
+
) -> tuple[pd.DataFrame, pd.DataFrame]:
|
|
726
|
+
"""Extract dense values and all sparse presences before vocabulary selection."""
|
|
727
|
+
required = {"run", "step", "sample_id", "prompt_id", "response"}
|
|
728
|
+
missing = required - set(rollouts.columns)
|
|
729
|
+
if missing:
|
|
730
|
+
raise ValueError(f"missing rollout columns: {sorted(missing)}")
|
|
731
|
+
if rollouts[list(required)].isna().any().any():
|
|
732
|
+
raise ValueError("feature input keys and responses cannot be null")
|
|
733
|
+
if rollouts.duplicated(["run", "sample_id"]).any():
|
|
734
|
+
raise ValueError("duplicate rollout keys (run, sample_id)")
|
|
735
|
+
|
|
736
|
+
dense_names = list(extract("").dense)
|
|
737
|
+
dense_rows = []
|
|
738
|
+
sparse_rows = []
|
|
739
|
+
columns = ["run", "sample_id", "step", "prompt_id", "response"]
|
|
740
|
+
for run, sample, _step, _prompt, response in rollouts[columns].itertuples(
|
|
741
|
+
index=False, name=None
|
|
742
|
+
):
|
|
743
|
+
result = extract(response)
|
|
744
|
+
dense_rows.append({"run": run, "sample_id": sample, **result.dense})
|
|
745
|
+
sparse_rows.extend((run, sample, token) for token in sorted(result.sparse))
|
|
746
|
+
dense = pd.DataFrame(dense_rows, columns=["run", "sample_id", *dense_names])
|
|
747
|
+
if dense.empty:
|
|
748
|
+
dense = dense.astype({name: "float64" for name in dense_names})
|
|
749
|
+
sparse = pd.DataFrame(sparse_rows, columns=["run", "sample_id", "feature"])
|
|
750
|
+
if include_tool_calls:
|
|
751
|
+
from .tool_features import extract_tool_calls
|
|
752
|
+
|
|
753
|
+
tool_rows, tool_sparse = [], []
|
|
754
|
+
for row in rollouts.to_dict("records"):
|
|
755
|
+
result = extract_tool_calls(
|
|
756
|
+
row.get("tool_calls"), row.get("response_messages")
|
|
757
|
+
)
|
|
758
|
+
tool_rows.append(result.dense)
|
|
759
|
+
tool_sparse.extend(
|
|
760
|
+
(row["run"], row["sample_id"], name) for name in sorted(result.sparse)
|
|
761
|
+
)
|
|
762
|
+
tools = pd.DataFrame(tool_rows).fillna(0.0)
|
|
763
|
+
for name in sorted(tools.columns):
|
|
764
|
+
dense[name] = tools[name].to_numpy(dtype=float)
|
|
765
|
+
if tool_sparse:
|
|
766
|
+
sparse = pd.concat(
|
|
767
|
+
[sparse, pd.DataFrame(tool_sparse, columns=sparse.columns)],
|
|
768
|
+
ignore_index=True,
|
|
769
|
+
)
|
|
770
|
+
return dense, sparse
|
|
771
|
+
|
|
772
|
+
|
|
773
|
+
def _retain_features(
|
|
774
|
+
rollouts: pd.DataFrame,
|
|
775
|
+
dense: pd.DataFrame,
|
|
776
|
+
raw_sparse: pd.DataFrame,
|
|
777
|
+
*,
|
|
778
|
+
max_sparse_per_tier: int = 2000,
|
|
779
|
+
min_prevalence: float = 0.05,
|
|
780
|
+
) -> FeatureTables:
|
|
781
|
+
"""Select a prefix vocabulary from previously extracted rollout features."""
|
|
782
|
+
_validate_selection(max_sparse_per_tier, min_prevalence)
|
|
783
|
+
dense_names = [name for name in dense.columns if name not in {"run", "sample_id"}]
|
|
784
|
+
sparse = raw_sparse
|
|
785
|
+
kept = _select_sparse(rollouts, sparse, max_sparse_per_tier, min_prevalence)
|
|
786
|
+
sparse = sparse.loc[sparse.feature.isin(kept)].reset_index(drop=True)
|
|
787
|
+
catalog = pd.DataFrame(
|
|
788
|
+
[
|
|
789
|
+
(name, _tier(name), "binary" if name in DENSE_BINARY else "continuous")
|
|
790
|
+
for name in dense_names
|
|
791
|
+
]
|
|
792
|
+
+ [(name, _tier(name), "binary") for name in kept],
|
|
793
|
+
columns=["feature", "tier", "kind"],
|
|
794
|
+
).sort_values("feature", ignore_index=True)
|
|
795
|
+
return FeatureTables(dense=dense, sparse=sparse, catalog=catalog)
|
|
796
|
+
|
|
797
|
+
|
|
798
|
+
def featurize(
|
|
799
|
+
rollouts: pd.DataFrame,
|
|
800
|
+
*,
|
|
801
|
+
max_sparse_per_tier: int = 2000,
|
|
802
|
+
min_prevalence: float = 0.05,
|
|
803
|
+
include_tool_calls: bool = False,
|
|
804
|
+
) -> FeatureTables:
|
|
805
|
+
"""Retain sparse features by peak prompt-balanced prevalence per tier.
|
|
806
|
+
|
|
807
|
+
Average within prompts, then across all prompts at each run/step, including
|
|
808
|
+
zeros for absent features. Ties break lexically. Recorded tool features are
|
|
809
|
+
included only when requested and require complete structured call evidence.
|
|
810
|
+
"""
|
|
811
|
+
_validate_selection(max_sparse_per_tier, min_prevalence)
|
|
812
|
+
if type(include_tool_calls) is not bool:
|
|
813
|
+
raise ValueError("include_tool_calls must be Boolean")
|
|
814
|
+
dense, raw_sparse = _extract_features(
|
|
815
|
+
rollouts, include_tool_calls=include_tool_calls
|
|
816
|
+
)
|
|
817
|
+
return _retain_features(
|
|
818
|
+
rollouts,
|
|
819
|
+
dense,
|
|
820
|
+
raw_sparse,
|
|
821
|
+
max_sparse_per_tier=max_sparse_per_tier,
|
|
822
|
+
min_prevalence=min_prevalence,
|
|
823
|
+
)
|