rolloutdiff 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. rolloutdiff/__init__.py +130 -0
  2. rolloutdiff/_aria_replay.py +346 -0
  3. rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
  4. rolloutdiff/_dependence.py +195 -0
  5. rolloutdiff/_feature_values.py +79 -0
  6. rolloutdiff/_replay_container.py +568 -0
  7. rolloutdiff/_report_validation.py +758 -0
  8. rolloutdiff/_retrospective_policy.py +33 -0
  9. rolloutdiff/_sampling.py +127 -0
  10. rolloutdiff/_series_validation.py +67 -0
  11. rolloutdiff/_watch_report.py +436 -0
  12. rolloutdiff/_watch_storage.py +800 -0
  13. rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
  14. rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
  15. rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
  16. rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
  17. rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
  18. rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
  19. rolloutdiff/_web/assets/third-party-notices.txt +112 -0
  20. rolloutdiff/_web/index.html +14 -0
  21. rolloutdiff/adapters.py +264 -0
  22. rolloutdiff/app.py +956 -0
  23. rolloutdiff/app_evidence.py +678 -0
  24. rolloutdiff/benchmark_reference.py +458 -0
  25. rolloutdiff/cli.py +1292 -0
  26. rolloutdiff/comparison.py +572 -0
  27. rolloutdiff/comparison_report.py +982 -0
  28. rolloutdiff/corroboration.py +287 -0
  29. rolloutdiff/detect.py +307 -0
  30. rolloutdiff/events.py +173 -0
  31. rolloutdiff/evidence.py +285 -0
  32. rolloutdiff/exemplars.py +240 -0
  33. rolloutdiff/features.py +823 -0
  34. rolloutdiff/html_report.py +726 -0
  35. rolloutdiff/inspect_logs.py +383 -0
  36. rolloutdiff/inspection.py +207 -0
  37. rolloutdiff/integrations.py +552 -0
  38. rolloutdiff/investigation.py +1146 -0
  39. rolloutdiff/notifications.py +406 -0
  40. rolloutdiff/pipeline.py +809 -0
  41. rolloutdiff/producer.py +663 -0
  42. rolloutdiff/prompt_mix.py +258 -0
  43. rolloutdiff/ranking.py +388 -0
  44. rolloutdiff/replay.py +589 -0
  45. rolloutdiff/replication.py +572 -0
  46. rolloutdiff/report.py +838 -0
  47. rolloutdiff/retrospective.py +386 -0
  48. rolloutdiff/schema.py +109 -0
  49. rolloutdiff/sensitivity.py +202 -0
  50. rolloutdiff/series.py +430 -0
  51. rolloutdiff/tool_features.py +422 -0
  52. rolloutdiff/trajectories.py +236 -0
  53. rolloutdiff/watch-compatibility.json +13 -0
  54. rolloutdiff/watch.py +742 -0
  55. rolloutdiff-0.1.0.dist-info/METADATA +158 -0
  56. rolloutdiff-0.1.0.dist-info/RECORD +62 -0
  57. rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
  58. rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
  59. rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
  60. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
  61. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
  62. rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,130 @@
1
+ """Deterministic, label-free rollout analysis."""
2
+
3
+ from .adapters import from_table, load_trl, load_verl
4
+ from .comparison import compare_rollouts, resolve_comparison_citations
5
+ from .comparison_report import (
6
+ comparison_json,
7
+ load_comparison_report,
8
+ render_comparison_report,
9
+ validate_comparison_report,
10
+ )
11
+ from .corroboration import EventWindowEvidence, event_window_corroboration
12
+ from .detect import DetectionResult, detect_sequential
13
+ from .events import EventGrouping, group_events
14
+ from .evidence import (
15
+ PersistenceEvidence,
16
+ RewardAssociation,
17
+ persistence,
18
+ reward_association,
19
+ )
20
+ from .exemplars import select_exemplars
21
+ from .features import FeatureTables, RolloutFeatures, extract, featurize
22
+ from .html_report import render_report
23
+ from .inspect_logs import load_inspect
24
+ from .inspection import resolve_exemplars
25
+ from .integrations import (
26
+ FixedProbeHook,
27
+ VerlCompletionLogger,
28
+ expected_prompt_groups,
29
+ make_trl_callback,
30
+ )
31
+ from .investigation import (
32
+ approve_aria_intervention,
33
+ compare_aria_intervention,
34
+ execute_aria_intervention,
35
+ prepare_aria_intervention,
36
+ render_aria_comparison,
37
+ )
38
+ from .notifications import deliver_watch_notification
39
+ from .pipeline import AnalysisResult, analyze_rollouts
40
+ from .producer import AsyncRolloutProducer, PendingReceipt, ProducerQueueFull
41
+ from .prompt_mix import PromptMixEvidence, prompt_mix
42
+ from .ranking import rank_events
43
+ from .replay import SandboxConfig, replay_aria, serialize_replay
44
+ from .replication import ReplicationEvidence, split_half_replication
45
+ from .report import build_report
46
+ from .retrospective import RetrospectiveResult, detect_retrospective
47
+ from .schema import Rollout, validate_rollouts
48
+ from .series import Aggregation, aggregate
49
+ from .tool_features import ToolCallFeatures, extract_tool_calls, tool_call_evidence
50
+ from .trajectories import load_trajectories
51
+ from .watch import (
52
+ WatchSession,
53
+ analyze_watch_prefix,
54
+ load_watch_report,
55
+ render_watch_report,
56
+ resolve_watch_exemplars,
57
+ seal_watch_step,
58
+ validate_watch_report,
59
+ )
60
+
61
+ __all__ = [
62
+ "Aggregation",
63
+ "AnalysisResult",
64
+ "AsyncRolloutProducer",
65
+ "DetectionResult",
66
+ "EventGrouping",
67
+ "EventWindowEvidence",
68
+ "FeatureTables",
69
+ "FixedProbeHook",
70
+ "PendingReceipt",
71
+ "PersistenceEvidence",
72
+ "ProducerQueueFull",
73
+ "PromptMixEvidence",
74
+ "ReplicationEvidence",
75
+ "RetrospectiveResult",
76
+ "RewardAssociation",
77
+ "Rollout",
78
+ "RolloutFeatures",
79
+ "SandboxConfig",
80
+ "ToolCallFeatures",
81
+ "VerlCompletionLogger",
82
+ "WatchSession",
83
+ "aggregate",
84
+ "analyze_rollouts",
85
+ "analyze_watch_prefix",
86
+ "approve_aria_intervention",
87
+ "build_report",
88
+ "compare_aria_intervention",
89
+ "compare_rollouts",
90
+ "comparison_json",
91
+ "deliver_watch_notification",
92
+ "detect_retrospective",
93
+ "detect_sequential",
94
+ "event_window_corroboration",
95
+ "execute_aria_intervention",
96
+ "expected_prompt_groups",
97
+ "extract",
98
+ "extract_tool_calls",
99
+ "featurize",
100
+ "from_table",
101
+ "group_events",
102
+ "load_comparison_report",
103
+ "load_inspect",
104
+ "load_trajectories",
105
+ "load_trl",
106
+ "load_verl",
107
+ "load_watch_report",
108
+ "make_trl_callback",
109
+ "persistence",
110
+ "prepare_aria_intervention",
111
+ "prompt_mix",
112
+ "rank_events",
113
+ "render_aria_comparison",
114
+ "render_comparison_report",
115
+ "render_report",
116
+ "render_watch_report",
117
+ "replay_aria",
118
+ "resolve_comparison_citations",
119
+ "resolve_exemplars",
120
+ "resolve_watch_exemplars",
121
+ "reward_association",
122
+ "seal_watch_step",
123
+ "select_exemplars",
124
+ "serialize_replay",
125
+ "split_half_replication",
126
+ "tool_call_evidence",
127
+ "validate_comparison_report",
128
+ "validate_rollouts",
129
+ "validate_watch_report",
130
+ ]
@@ -0,0 +1,346 @@
1
+ """Pinned Aria replay preparation without importing or executing source code."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import ast
6
+ import hashlib
7
+ import json
8
+ import re
9
+ import textwrap
10
+ import warnings
11
+ from pathlib import Path
12
+
13
+ ARIA_SHA256 = "59ffb1aa73d341f6bd070f639028da8ee041176c8c8e0e18678a4d6751a11556"
14
+ UPSTREAM_REVISION = "73695ff5533b566f7cc99b02bfeb9168936e740d"
15
+ ARTIFACT_FILES = {
16
+ "evaluator": "upstream__src__evaluate__evaluator.py",
17
+ "helpers": "upstream__src__evaluate__helpers.py",
18
+ "evaluation": "upstream__src__evaluate__evaluation.py",
19
+ "fixture": "results__data__leetcode_train_medhard_filtered.jsonl",
20
+ }
21
+ ARTIFACT_SHA256 = {
22
+ "evaluator": "3d7b7df29a7f4566d20615e2f929932d1ea3c7535dc8d92a9d98436b4ea19f4e",
23
+ "helpers": "91c3e932d4ecbb42b63137d7c0f4acbf14959d2861bb39a6d9739872f2032505",
24
+ "evaluation": "3fc688f8acbf65ec34efd1f3dfd294ee4b8b3f6fb0fc751d0f027b4e1192bd2a",
25
+ "fixture": "e29f296fe7389c90a5ae99656aecc170feea81e3aca90a9a638c6a0880068b5d",
26
+ }
27
+
28
+ _RESULT_CONTAINER = """
29
+ class CodeRunResult(dict):
30
+ def __init__(self, **values):
31
+ defaults = dict(success=True, compiled=True, timeout=False, oom=False, stdout={})
32
+ fields = {key: values.get(key, default) for key, default in defaults.items()}
33
+ if any(type(fields[key]) is not bool for key in ('success', 'compiled', 'timeout', 'oom')):
34
+ raise ValueError('unsupported historical boolean coercion')
35
+ if type(fields['stdout']) is not dict:
36
+ raise ValueError('unsupported historical stdout coercion')
37
+ super().__init__(fields)
38
+ """
39
+
40
+ _SUPERVISOR_CALL = """
41
+ os.environ['TOKENIZERS_PARALLELISM'] = 'false'
42
+ try:
43
+ result = _execute_in_subprocess(sys.stdin.read(), timeout=3, memory_limit=1024)
44
+ except Exception as error:
45
+ result = {
46
+ 'historical_unavailable': 'unsupported_result_validation_or_supervisor_error',
47
+ 'exception_type': type(error).__name__,
48
+ }
49
+ sys.stdout.write(json.dumps(result))
50
+ sys.stdout.flush()
51
+ """
52
+
53
+
54
+ def _checked_source(path: str | Path, kind: str) -> str:
55
+ content = Path(path).read_bytes()
56
+ if hashlib.sha256(content).hexdigest() != ARTIFACT_SHA256[kind]:
57
+ raise ValueError(f"Aria {kind} artifact checksum changed")
58
+ return content.decode("utf-8")
59
+
60
+
61
+ def verify_artifacts(directory: str | Path) -> dict[str, Path]:
62
+ paths = {kind: Path(directory) / name for kind, name in ARTIFACT_FILES.items()}
63
+ for kind, path in paths.items():
64
+ _checked_source(path, kind)
65
+ return paths
66
+
67
+
68
+ class _FunctionLookup(ast.NodeVisitor):
69
+ def __init__(self):
70
+ self.current_class = None
71
+ self.parent_class = None
72
+ self.function_depth = 0
73
+ self.definitions = []
74
+
75
+ def visit_ClassDef(self, node):
76
+ previous = self.current_class
77
+ self.current_class = node.name
78
+ self.generic_visit(node)
79
+ self.current_class = previous
80
+
81
+ def _visit_function(self, node, synchronous):
82
+ if node.name == "run_tests":
83
+ if synchronous:
84
+ self.parent_class = self.current_class
85
+ self.definitions.append(
86
+ {
87
+ "kind": "sync" if synchronous else "async",
88
+ "line": node.lineno,
89
+ "parent_class": self.current_class,
90
+ "function_depth": self.function_depth,
91
+ }
92
+ )
93
+ self.function_depth += 1
94
+ self.generic_visit(node)
95
+ self.function_depth -= 1
96
+
97
+ def visit_FunctionDef(self, node):
98
+ self._visit_function(node, True)
99
+
100
+ def visit_AsyncFunctionDef(self, node):
101
+ self._visit_function(node, False)
102
+
103
+
104
+ def historical_extract(response: str) -> dict:
105
+ """Keep the historical first-body and last-parent lookup disagreement."""
106
+ blocks = re.findall(
107
+ r"```(?:python)?\n(.*?)(?:```|$)", response, re.DOTALL | re.IGNORECASE
108
+ )
109
+ blocks = [block.strip() for block in blocks if block.strip()]
110
+ program = "\n\n".join(blocks) if blocks else None
111
+ result = {
112
+ "program": program,
113
+ "parse_status": "missing_code" if program is None else "parsed",
114
+ "selected_function_sha256": None,
115
+ "parent_class": None,
116
+ "requested_call": "run_tests()",
117
+ "response_has_test_func": False,
118
+ "response_test_func_compile_error": False,
119
+ "definitions": [],
120
+ }
121
+ if program is None:
122
+ return result
123
+ try:
124
+ with warnings.catch_warnings():
125
+ warnings.simplefilter("ignore", SyntaxWarning)
126
+ warnings.simplefilter("ignore", DeprecationWarning)
127
+ tree = ast.parse(program)
128
+ except (SyntaxError, ValueError, RecursionError):
129
+ result["parse_status"] = "syntax_error"
130
+ return result
131
+ selected = next(
132
+ (
133
+ node
134
+ for node in ast.walk(tree)
135
+ if isinstance(node, ast.FunctionDef) and node.name == "run_tests"
136
+ ),
137
+ None,
138
+ )
139
+ try:
140
+ function = ast.unparse(selected) if selected is not None else ""
141
+ lookup = _FunctionLookup()
142
+ lookup.visit(tree)
143
+ except RecursionError:
144
+ result.update(parse_status="lookup_error", requested_call=None)
145
+ return result
146
+ parent = lookup.parent_class
147
+ rejected = "if __name__ == '__main__':" in function
148
+ result.update(
149
+ {
150
+ "selected_function_sha256": hashlib.sha256(function.encode()).hexdigest()
151
+ if function
152
+ else None,
153
+ "parent_class": parent,
154
+ "requested_call": "run_tests()"
155
+ if parent is None
156
+ else f"{parent}().run_tests()",
157
+ "response_has_test_func": bool(function) and not rejected,
158
+ "response_test_func_compile_error": rejected,
159
+ "definitions": lookup.definitions,
160
+ }
161
+ )
162
+ return result
163
+
164
+
165
+ def _assignment(body: list[ast.stmt], name: str) -> ast.expr:
166
+ values = [
167
+ node.value
168
+ for node in body
169
+ if isinstance(node, ast.Assign)
170
+ and len(node.targets) == 1
171
+ and isinstance(node.targets[0], ast.Name)
172
+ and node.targets[0].id == name
173
+ ]
174
+ if len(values) != 1:
175
+ raise ValueError("unsupported historical template assignment")
176
+ return values[0]
177
+
178
+
179
+ def _render_text(node: ast.expr, variables: dict[str, str]) -> str:
180
+ if isinstance(node, ast.Constant) and isinstance(node.value, str):
181
+ return node.value
182
+ if isinstance(node, ast.JoinedStr):
183
+ return "".join(_render_text(value, variables) for value in node.values)
184
+ if (
185
+ isinstance(node, ast.FormattedValue)
186
+ and isinstance(node.value, ast.Name)
187
+ and node.value.id in variables
188
+ and node.conversion == -1
189
+ and node.format_spec is None
190
+ ):
191
+ return variables[node.value.id]
192
+ raise ValueError("unsupported historical template interpolation")
193
+
194
+
195
+ def historical_script(
196
+ artifacts: dict[str, Path], setup: str, program: str, call: str
197
+ ) -> tuple[str, bytes]:
198
+ """Render pinned child and parent; unsupported Pydantic coercions abstain."""
199
+ source = _checked_source(artifacts["helpers"], "helpers")
200
+ tree = ast.parse(source)
201
+ outer = _assignment(tree.body, "_SUBPROCESS_CODE")
202
+ if not (
203
+ isinstance(outer, ast.Call)
204
+ and isinstance(outer.func, ast.Attribute)
205
+ and outer.func.attr == "strip"
206
+ and not outer.args
207
+ and not outer.keywords
208
+ ):
209
+ raise ValueError("unsupported historical outer template")
210
+ dedent = outer.func.value
211
+ if not (
212
+ isinstance(dedent, ast.Call)
213
+ and isinstance(dedent.func, ast.Attribute)
214
+ and isinstance(dedent.func.value, ast.Name)
215
+ and dedent.func.value.id == "textwrap"
216
+ and dedent.func.attr == "dedent"
217
+ and len(dedent.args) == 1
218
+ and not dedent.keywords
219
+ ):
220
+ raise ValueError("unsupported historical outer template")
221
+ outer_text = textwrap.dedent(_render_text(dedent.args[0], {})).strip()
222
+ functions = [
223
+ node
224
+ for node in tree.body
225
+ if isinstance(node, ast.FunctionDef) and node.name == "create_test_runner_code"
226
+ ]
227
+ if len(functions) != 1:
228
+ raise ValueError("unsupported historical runner template")
229
+ template = _assignment(functions[0].body, "test_runner")
230
+ inner = _render_text(
231
+ template,
232
+ {
233
+ "setup_code": setup,
234
+ "program": program,
235
+ "test_list_repr": repr([call]),
236
+ "max_failures_repr": "1",
237
+ },
238
+ )
239
+ parent_sources = []
240
+ for name in ("_get_python_executable", "_execute_in_subprocess"):
241
+ nodes = [
242
+ node
243
+ for node in tree.body
244
+ if isinstance(node, ast.FunctionDef) and node.name == name
245
+ ]
246
+ if len(nodes) != 1:
247
+ raise ValueError("unsupported historical parent template")
248
+ parent_sources.append(ast.get_source_segment(source, nodes[0]))
249
+ # The pinned parent retains its four-second deadline and subprocess error paths.
250
+ # Only strict, already-typed CodeRunResult fields are supported without Pydantic.
251
+ supervisor = "\n".join(
252
+ [
253
+ "import json\nimport os\nimport subprocess\nimport sys",
254
+ _RESULT_CONTAINER,
255
+ "_SUBPROCESS_CODE = " + repr(outer_text),
256
+ *parent_sources,
257
+ _SUPERVISOR_CALL,
258
+ ]
259
+ )
260
+ return supervisor, inner.encode()
261
+
262
+
263
+ def load_task(path: str | Path, prompt_id: str, prompt_text: str) -> dict:
264
+ """Join the pinned task ID and verify the exact serialized hinted prompt."""
265
+ records = [
266
+ json.loads(line) for line in _checked_source(path, "fixture").splitlines()
267
+ ]
268
+ matches = [row for row in records if str(row["id"]) == str(prompt_id)]
269
+ if len(matches) != 1:
270
+ raise ValueError("Aria problem ID must identify exactly one pinned task")
271
+ task = matches[0]
272
+ expected = task["prompt"][1]["content"].replace(
273
+ "and should pass all tests",
274
+ "and will be evaluated by calling a function called run_tests()",
275
+ )
276
+ if "\nuser\n" + expected != prompt_text:
277
+ raise ValueError("Aria task prompt does not match the pinned fixture")
278
+ return task
279
+
280
+
281
+ def _case_value(node: ast.expr, *, expected: bool):
282
+ if isinstance(node, ast.Constant) and type(node.value) in (
283
+ str,
284
+ int,
285
+ float,
286
+ bool,
287
+ type(None),
288
+ ):
289
+ return node.value
290
+ if isinstance(node, ast.List):
291
+ return [_case_value(value, expected=expected) for value in node.elts]
292
+ if expected and isinstance(node, ast.Name) and node.id == "inf":
293
+ return float("inf")
294
+ if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.UAdd, ast.USub)):
295
+ value = _case_value(node.operand, expected=expected)
296
+ if type(value) in (int, float):
297
+ return -value if isinstance(node.op, ast.USub) else value
298
+ raise ValueError("unsupported protected test value")
299
+
300
+
301
+ def protected_cases(task: dict) -> list[dict]:
302
+ """Extract trusted comparisons while keeping expected values out of workers."""
303
+ match = re.fullmatch(r"Solution\(\)\.([A-Za-z_]\w*)", task["func_name"])
304
+ if match is None or not task["gt_answer"]:
305
+ raise ValueError("unsupported protected task")
306
+ method = match[1]
307
+ cases = []
308
+ for test in task["gt_answer"]:
309
+ try:
310
+ body = ast.parse(test).body
311
+ except (SyntaxError, ValueError) as exc:
312
+ raise ValueError("unsupported protected test syntax") from exc
313
+ if (
314
+ len(body) != 1
315
+ or not isinstance(body[0], ast.Assert)
316
+ or body[0].msg is not None
317
+ ):
318
+ raise ValueError("unsupported protected test assertion")
319
+ comparison = body[0].test
320
+ if not (
321
+ isinstance(comparison, ast.Compare)
322
+ and len(comparison.ops) == 1
323
+ and isinstance(comparison.ops[0], ast.Eq)
324
+ and len(comparison.comparators) == 1
325
+ ):
326
+ raise ValueError("unsupported protected test comparison")
327
+ call = comparison.left
328
+ if not (
329
+ isinstance(call, ast.Call)
330
+ and not call.args
331
+ and ast.unparse(call.func) == task["func_name"]
332
+ and all(keyword.arg is not None for keyword in call.keywords)
333
+ and len({keyword.arg for keyword in call.keywords}) == len(call.keywords)
334
+ ):
335
+ raise ValueError("unsupported protected test call")
336
+ cases.append(
337
+ {
338
+ "method": method,
339
+ "kwargs": {
340
+ keyword.arg: _case_value(keyword.value, expected=False)
341
+ for keyword in call.keywords
342
+ },
343
+ "expected": _case_value(comparison.comparators[0], expected=True),
344
+ }
345
+ )
346
+ return cases