rolloutdiff 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rolloutdiff/__init__.py +130 -0
- rolloutdiff/_aria_replay.py +346 -0
- rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
- rolloutdiff/_dependence.py +195 -0
- rolloutdiff/_feature_values.py +79 -0
- rolloutdiff/_replay_container.py +568 -0
- rolloutdiff/_report_validation.py +758 -0
- rolloutdiff/_retrospective_policy.py +33 -0
- rolloutdiff/_sampling.py +127 -0
- rolloutdiff/_series_validation.py +67 -0
- rolloutdiff/_watch_report.py +436 -0
- rolloutdiff/_watch_storage.py +800 -0
- rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
- rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
- rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
- rolloutdiff/_web/assets/third-party-notices.txt +112 -0
- rolloutdiff/_web/index.html +14 -0
- rolloutdiff/adapters.py +264 -0
- rolloutdiff/app.py +956 -0
- rolloutdiff/app_evidence.py +678 -0
- rolloutdiff/benchmark_reference.py +458 -0
- rolloutdiff/cli.py +1292 -0
- rolloutdiff/comparison.py +572 -0
- rolloutdiff/comparison_report.py +982 -0
- rolloutdiff/corroboration.py +287 -0
- rolloutdiff/detect.py +307 -0
- rolloutdiff/events.py +173 -0
- rolloutdiff/evidence.py +285 -0
- rolloutdiff/exemplars.py +240 -0
- rolloutdiff/features.py +823 -0
- rolloutdiff/html_report.py +726 -0
- rolloutdiff/inspect_logs.py +383 -0
- rolloutdiff/inspection.py +207 -0
- rolloutdiff/integrations.py +552 -0
- rolloutdiff/investigation.py +1146 -0
- rolloutdiff/notifications.py +406 -0
- rolloutdiff/pipeline.py +809 -0
- rolloutdiff/producer.py +663 -0
- rolloutdiff/prompt_mix.py +258 -0
- rolloutdiff/ranking.py +388 -0
- rolloutdiff/replay.py +589 -0
- rolloutdiff/replication.py +572 -0
- rolloutdiff/report.py +838 -0
- rolloutdiff/retrospective.py +386 -0
- rolloutdiff/schema.py +109 -0
- rolloutdiff/sensitivity.py +202 -0
- rolloutdiff/series.py +430 -0
- rolloutdiff/tool_features.py +422 -0
- rolloutdiff/trajectories.py +236 -0
- rolloutdiff/watch-compatibility.json +13 -0
- rolloutdiff/watch.py +742 -0
- rolloutdiff-0.1.0.dist-info/METADATA +158 -0
- rolloutdiff-0.1.0.dist-info/RECORD +62 -0
- rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
- rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
- rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
- rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
rolloutdiff/__init__.py
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
"""Deterministic, label-free rollout analysis."""
|
|
2
|
+
|
|
3
|
+
from .adapters import from_table, load_trl, load_verl
|
|
4
|
+
from .comparison import compare_rollouts, resolve_comparison_citations
|
|
5
|
+
from .comparison_report import (
|
|
6
|
+
comparison_json,
|
|
7
|
+
load_comparison_report,
|
|
8
|
+
render_comparison_report,
|
|
9
|
+
validate_comparison_report,
|
|
10
|
+
)
|
|
11
|
+
from .corroboration import EventWindowEvidence, event_window_corroboration
|
|
12
|
+
from .detect import DetectionResult, detect_sequential
|
|
13
|
+
from .events import EventGrouping, group_events
|
|
14
|
+
from .evidence import (
|
|
15
|
+
PersistenceEvidence,
|
|
16
|
+
RewardAssociation,
|
|
17
|
+
persistence,
|
|
18
|
+
reward_association,
|
|
19
|
+
)
|
|
20
|
+
from .exemplars import select_exemplars
|
|
21
|
+
from .features import FeatureTables, RolloutFeatures, extract, featurize
|
|
22
|
+
from .html_report import render_report
|
|
23
|
+
from .inspect_logs import load_inspect
|
|
24
|
+
from .inspection import resolve_exemplars
|
|
25
|
+
from .integrations import (
|
|
26
|
+
FixedProbeHook,
|
|
27
|
+
VerlCompletionLogger,
|
|
28
|
+
expected_prompt_groups,
|
|
29
|
+
make_trl_callback,
|
|
30
|
+
)
|
|
31
|
+
from .investigation import (
|
|
32
|
+
approve_aria_intervention,
|
|
33
|
+
compare_aria_intervention,
|
|
34
|
+
execute_aria_intervention,
|
|
35
|
+
prepare_aria_intervention,
|
|
36
|
+
render_aria_comparison,
|
|
37
|
+
)
|
|
38
|
+
from .notifications import deliver_watch_notification
|
|
39
|
+
from .pipeline import AnalysisResult, analyze_rollouts
|
|
40
|
+
from .producer import AsyncRolloutProducer, PendingReceipt, ProducerQueueFull
|
|
41
|
+
from .prompt_mix import PromptMixEvidence, prompt_mix
|
|
42
|
+
from .ranking import rank_events
|
|
43
|
+
from .replay import SandboxConfig, replay_aria, serialize_replay
|
|
44
|
+
from .replication import ReplicationEvidence, split_half_replication
|
|
45
|
+
from .report import build_report
|
|
46
|
+
from .retrospective import RetrospectiveResult, detect_retrospective
|
|
47
|
+
from .schema import Rollout, validate_rollouts
|
|
48
|
+
from .series import Aggregation, aggregate
|
|
49
|
+
from .tool_features import ToolCallFeatures, extract_tool_calls, tool_call_evidence
|
|
50
|
+
from .trajectories import load_trajectories
|
|
51
|
+
from .watch import (
|
|
52
|
+
WatchSession,
|
|
53
|
+
analyze_watch_prefix,
|
|
54
|
+
load_watch_report,
|
|
55
|
+
render_watch_report,
|
|
56
|
+
resolve_watch_exemplars,
|
|
57
|
+
seal_watch_step,
|
|
58
|
+
validate_watch_report,
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
__all__ = [
|
|
62
|
+
"Aggregation",
|
|
63
|
+
"AnalysisResult",
|
|
64
|
+
"AsyncRolloutProducer",
|
|
65
|
+
"DetectionResult",
|
|
66
|
+
"EventGrouping",
|
|
67
|
+
"EventWindowEvidence",
|
|
68
|
+
"FeatureTables",
|
|
69
|
+
"FixedProbeHook",
|
|
70
|
+
"PendingReceipt",
|
|
71
|
+
"PersistenceEvidence",
|
|
72
|
+
"ProducerQueueFull",
|
|
73
|
+
"PromptMixEvidence",
|
|
74
|
+
"ReplicationEvidence",
|
|
75
|
+
"RetrospectiveResult",
|
|
76
|
+
"RewardAssociation",
|
|
77
|
+
"Rollout",
|
|
78
|
+
"RolloutFeatures",
|
|
79
|
+
"SandboxConfig",
|
|
80
|
+
"ToolCallFeatures",
|
|
81
|
+
"VerlCompletionLogger",
|
|
82
|
+
"WatchSession",
|
|
83
|
+
"aggregate",
|
|
84
|
+
"analyze_rollouts",
|
|
85
|
+
"analyze_watch_prefix",
|
|
86
|
+
"approve_aria_intervention",
|
|
87
|
+
"build_report",
|
|
88
|
+
"compare_aria_intervention",
|
|
89
|
+
"compare_rollouts",
|
|
90
|
+
"comparison_json",
|
|
91
|
+
"deliver_watch_notification",
|
|
92
|
+
"detect_retrospective",
|
|
93
|
+
"detect_sequential",
|
|
94
|
+
"event_window_corroboration",
|
|
95
|
+
"execute_aria_intervention",
|
|
96
|
+
"expected_prompt_groups",
|
|
97
|
+
"extract",
|
|
98
|
+
"extract_tool_calls",
|
|
99
|
+
"featurize",
|
|
100
|
+
"from_table",
|
|
101
|
+
"group_events",
|
|
102
|
+
"load_comparison_report",
|
|
103
|
+
"load_inspect",
|
|
104
|
+
"load_trajectories",
|
|
105
|
+
"load_trl",
|
|
106
|
+
"load_verl",
|
|
107
|
+
"load_watch_report",
|
|
108
|
+
"make_trl_callback",
|
|
109
|
+
"persistence",
|
|
110
|
+
"prepare_aria_intervention",
|
|
111
|
+
"prompt_mix",
|
|
112
|
+
"rank_events",
|
|
113
|
+
"render_aria_comparison",
|
|
114
|
+
"render_comparison_report",
|
|
115
|
+
"render_report",
|
|
116
|
+
"render_watch_report",
|
|
117
|
+
"replay_aria",
|
|
118
|
+
"resolve_comparison_citations",
|
|
119
|
+
"resolve_exemplars",
|
|
120
|
+
"resolve_watch_exemplars",
|
|
121
|
+
"reward_association",
|
|
122
|
+
"seal_watch_step",
|
|
123
|
+
"select_exemplars",
|
|
124
|
+
"serialize_replay",
|
|
125
|
+
"split_half_replication",
|
|
126
|
+
"tool_call_evidence",
|
|
127
|
+
"validate_comparison_report",
|
|
128
|
+
"validate_rollouts",
|
|
129
|
+
"validate_watch_report",
|
|
130
|
+
]
|
|
@@ -0,0 +1,346 @@
|
|
|
1
|
+
"""Pinned Aria replay preparation without importing or executing source code."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import ast
|
|
6
|
+
import hashlib
|
|
7
|
+
import json
|
|
8
|
+
import re
|
|
9
|
+
import textwrap
|
|
10
|
+
import warnings
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
ARIA_SHA256 = "59ffb1aa73d341f6bd070f639028da8ee041176c8c8e0e18678a4d6751a11556"
|
|
14
|
+
UPSTREAM_REVISION = "73695ff5533b566f7cc99b02bfeb9168936e740d"
|
|
15
|
+
ARTIFACT_FILES = {
|
|
16
|
+
"evaluator": "upstream__src__evaluate__evaluator.py",
|
|
17
|
+
"helpers": "upstream__src__evaluate__helpers.py",
|
|
18
|
+
"evaluation": "upstream__src__evaluate__evaluation.py",
|
|
19
|
+
"fixture": "results__data__leetcode_train_medhard_filtered.jsonl",
|
|
20
|
+
}
|
|
21
|
+
ARTIFACT_SHA256 = {
|
|
22
|
+
"evaluator": "3d7b7df29a7f4566d20615e2f929932d1ea3c7535dc8d92a9d98436b4ea19f4e",
|
|
23
|
+
"helpers": "91c3e932d4ecbb42b63137d7c0f4acbf14959d2861bb39a6d9739872f2032505",
|
|
24
|
+
"evaluation": "3fc688f8acbf65ec34efd1f3dfd294ee4b8b3f6fb0fc751d0f027b4e1192bd2a",
|
|
25
|
+
"fixture": "e29f296fe7389c90a5ae99656aecc170feea81e3aca90a9a638c6a0880068b5d",
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
_RESULT_CONTAINER = """
|
|
29
|
+
class CodeRunResult(dict):
|
|
30
|
+
def __init__(self, **values):
|
|
31
|
+
defaults = dict(success=True, compiled=True, timeout=False, oom=False, stdout={})
|
|
32
|
+
fields = {key: values.get(key, default) for key, default in defaults.items()}
|
|
33
|
+
if any(type(fields[key]) is not bool for key in ('success', 'compiled', 'timeout', 'oom')):
|
|
34
|
+
raise ValueError('unsupported historical boolean coercion')
|
|
35
|
+
if type(fields['stdout']) is not dict:
|
|
36
|
+
raise ValueError('unsupported historical stdout coercion')
|
|
37
|
+
super().__init__(fields)
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
_SUPERVISOR_CALL = """
|
|
41
|
+
os.environ['TOKENIZERS_PARALLELISM'] = 'false'
|
|
42
|
+
try:
|
|
43
|
+
result = _execute_in_subprocess(sys.stdin.read(), timeout=3, memory_limit=1024)
|
|
44
|
+
except Exception as error:
|
|
45
|
+
result = {
|
|
46
|
+
'historical_unavailable': 'unsupported_result_validation_or_supervisor_error',
|
|
47
|
+
'exception_type': type(error).__name__,
|
|
48
|
+
}
|
|
49
|
+
sys.stdout.write(json.dumps(result))
|
|
50
|
+
sys.stdout.flush()
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _checked_source(path: str | Path, kind: str) -> str:
|
|
55
|
+
content = Path(path).read_bytes()
|
|
56
|
+
if hashlib.sha256(content).hexdigest() != ARTIFACT_SHA256[kind]:
|
|
57
|
+
raise ValueError(f"Aria {kind} artifact checksum changed")
|
|
58
|
+
return content.decode("utf-8")
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def verify_artifacts(directory: str | Path) -> dict[str, Path]:
|
|
62
|
+
paths = {kind: Path(directory) / name for kind, name in ARTIFACT_FILES.items()}
|
|
63
|
+
for kind, path in paths.items():
|
|
64
|
+
_checked_source(path, kind)
|
|
65
|
+
return paths
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class _FunctionLookup(ast.NodeVisitor):
|
|
69
|
+
def __init__(self):
|
|
70
|
+
self.current_class = None
|
|
71
|
+
self.parent_class = None
|
|
72
|
+
self.function_depth = 0
|
|
73
|
+
self.definitions = []
|
|
74
|
+
|
|
75
|
+
def visit_ClassDef(self, node):
|
|
76
|
+
previous = self.current_class
|
|
77
|
+
self.current_class = node.name
|
|
78
|
+
self.generic_visit(node)
|
|
79
|
+
self.current_class = previous
|
|
80
|
+
|
|
81
|
+
def _visit_function(self, node, synchronous):
|
|
82
|
+
if node.name == "run_tests":
|
|
83
|
+
if synchronous:
|
|
84
|
+
self.parent_class = self.current_class
|
|
85
|
+
self.definitions.append(
|
|
86
|
+
{
|
|
87
|
+
"kind": "sync" if synchronous else "async",
|
|
88
|
+
"line": node.lineno,
|
|
89
|
+
"parent_class": self.current_class,
|
|
90
|
+
"function_depth": self.function_depth,
|
|
91
|
+
}
|
|
92
|
+
)
|
|
93
|
+
self.function_depth += 1
|
|
94
|
+
self.generic_visit(node)
|
|
95
|
+
self.function_depth -= 1
|
|
96
|
+
|
|
97
|
+
def visit_FunctionDef(self, node):
|
|
98
|
+
self._visit_function(node, True)
|
|
99
|
+
|
|
100
|
+
def visit_AsyncFunctionDef(self, node):
|
|
101
|
+
self._visit_function(node, False)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def historical_extract(response: str) -> dict:
|
|
105
|
+
"""Keep the historical first-body and last-parent lookup disagreement."""
|
|
106
|
+
blocks = re.findall(
|
|
107
|
+
r"```(?:python)?\n(.*?)(?:```|$)", response, re.DOTALL | re.IGNORECASE
|
|
108
|
+
)
|
|
109
|
+
blocks = [block.strip() for block in blocks if block.strip()]
|
|
110
|
+
program = "\n\n".join(blocks) if blocks else None
|
|
111
|
+
result = {
|
|
112
|
+
"program": program,
|
|
113
|
+
"parse_status": "missing_code" if program is None else "parsed",
|
|
114
|
+
"selected_function_sha256": None,
|
|
115
|
+
"parent_class": None,
|
|
116
|
+
"requested_call": "run_tests()",
|
|
117
|
+
"response_has_test_func": False,
|
|
118
|
+
"response_test_func_compile_error": False,
|
|
119
|
+
"definitions": [],
|
|
120
|
+
}
|
|
121
|
+
if program is None:
|
|
122
|
+
return result
|
|
123
|
+
try:
|
|
124
|
+
with warnings.catch_warnings():
|
|
125
|
+
warnings.simplefilter("ignore", SyntaxWarning)
|
|
126
|
+
warnings.simplefilter("ignore", DeprecationWarning)
|
|
127
|
+
tree = ast.parse(program)
|
|
128
|
+
except (SyntaxError, ValueError, RecursionError):
|
|
129
|
+
result["parse_status"] = "syntax_error"
|
|
130
|
+
return result
|
|
131
|
+
selected = next(
|
|
132
|
+
(
|
|
133
|
+
node
|
|
134
|
+
for node in ast.walk(tree)
|
|
135
|
+
if isinstance(node, ast.FunctionDef) and node.name == "run_tests"
|
|
136
|
+
),
|
|
137
|
+
None,
|
|
138
|
+
)
|
|
139
|
+
try:
|
|
140
|
+
function = ast.unparse(selected) if selected is not None else ""
|
|
141
|
+
lookup = _FunctionLookup()
|
|
142
|
+
lookup.visit(tree)
|
|
143
|
+
except RecursionError:
|
|
144
|
+
result.update(parse_status="lookup_error", requested_call=None)
|
|
145
|
+
return result
|
|
146
|
+
parent = lookup.parent_class
|
|
147
|
+
rejected = "if __name__ == '__main__':" in function
|
|
148
|
+
result.update(
|
|
149
|
+
{
|
|
150
|
+
"selected_function_sha256": hashlib.sha256(function.encode()).hexdigest()
|
|
151
|
+
if function
|
|
152
|
+
else None,
|
|
153
|
+
"parent_class": parent,
|
|
154
|
+
"requested_call": "run_tests()"
|
|
155
|
+
if parent is None
|
|
156
|
+
else f"{parent}().run_tests()",
|
|
157
|
+
"response_has_test_func": bool(function) and not rejected,
|
|
158
|
+
"response_test_func_compile_error": rejected,
|
|
159
|
+
"definitions": lookup.definitions,
|
|
160
|
+
}
|
|
161
|
+
)
|
|
162
|
+
return result
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _assignment(body: list[ast.stmt], name: str) -> ast.expr:
|
|
166
|
+
values = [
|
|
167
|
+
node.value
|
|
168
|
+
for node in body
|
|
169
|
+
if isinstance(node, ast.Assign)
|
|
170
|
+
and len(node.targets) == 1
|
|
171
|
+
and isinstance(node.targets[0], ast.Name)
|
|
172
|
+
and node.targets[0].id == name
|
|
173
|
+
]
|
|
174
|
+
if len(values) != 1:
|
|
175
|
+
raise ValueError("unsupported historical template assignment")
|
|
176
|
+
return values[0]
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _render_text(node: ast.expr, variables: dict[str, str]) -> str:
|
|
180
|
+
if isinstance(node, ast.Constant) and isinstance(node.value, str):
|
|
181
|
+
return node.value
|
|
182
|
+
if isinstance(node, ast.JoinedStr):
|
|
183
|
+
return "".join(_render_text(value, variables) for value in node.values)
|
|
184
|
+
if (
|
|
185
|
+
isinstance(node, ast.FormattedValue)
|
|
186
|
+
and isinstance(node.value, ast.Name)
|
|
187
|
+
and node.value.id in variables
|
|
188
|
+
and node.conversion == -1
|
|
189
|
+
and node.format_spec is None
|
|
190
|
+
):
|
|
191
|
+
return variables[node.value.id]
|
|
192
|
+
raise ValueError("unsupported historical template interpolation")
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def historical_script(
|
|
196
|
+
artifacts: dict[str, Path], setup: str, program: str, call: str
|
|
197
|
+
) -> tuple[str, bytes]:
|
|
198
|
+
"""Render pinned child and parent; unsupported Pydantic coercions abstain."""
|
|
199
|
+
source = _checked_source(artifacts["helpers"], "helpers")
|
|
200
|
+
tree = ast.parse(source)
|
|
201
|
+
outer = _assignment(tree.body, "_SUBPROCESS_CODE")
|
|
202
|
+
if not (
|
|
203
|
+
isinstance(outer, ast.Call)
|
|
204
|
+
and isinstance(outer.func, ast.Attribute)
|
|
205
|
+
and outer.func.attr == "strip"
|
|
206
|
+
and not outer.args
|
|
207
|
+
and not outer.keywords
|
|
208
|
+
):
|
|
209
|
+
raise ValueError("unsupported historical outer template")
|
|
210
|
+
dedent = outer.func.value
|
|
211
|
+
if not (
|
|
212
|
+
isinstance(dedent, ast.Call)
|
|
213
|
+
and isinstance(dedent.func, ast.Attribute)
|
|
214
|
+
and isinstance(dedent.func.value, ast.Name)
|
|
215
|
+
and dedent.func.value.id == "textwrap"
|
|
216
|
+
and dedent.func.attr == "dedent"
|
|
217
|
+
and len(dedent.args) == 1
|
|
218
|
+
and not dedent.keywords
|
|
219
|
+
):
|
|
220
|
+
raise ValueError("unsupported historical outer template")
|
|
221
|
+
outer_text = textwrap.dedent(_render_text(dedent.args[0], {})).strip()
|
|
222
|
+
functions = [
|
|
223
|
+
node
|
|
224
|
+
for node in tree.body
|
|
225
|
+
if isinstance(node, ast.FunctionDef) and node.name == "create_test_runner_code"
|
|
226
|
+
]
|
|
227
|
+
if len(functions) != 1:
|
|
228
|
+
raise ValueError("unsupported historical runner template")
|
|
229
|
+
template = _assignment(functions[0].body, "test_runner")
|
|
230
|
+
inner = _render_text(
|
|
231
|
+
template,
|
|
232
|
+
{
|
|
233
|
+
"setup_code": setup,
|
|
234
|
+
"program": program,
|
|
235
|
+
"test_list_repr": repr([call]),
|
|
236
|
+
"max_failures_repr": "1",
|
|
237
|
+
},
|
|
238
|
+
)
|
|
239
|
+
parent_sources = []
|
|
240
|
+
for name in ("_get_python_executable", "_execute_in_subprocess"):
|
|
241
|
+
nodes = [
|
|
242
|
+
node
|
|
243
|
+
for node in tree.body
|
|
244
|
+
if isinstance(node, ast.FunctionDef) and node.name == name
|
|
245
|
+
]
|
|
246
|
+
if len(nodes) != 1:
|
|
247
|
+
raise ValueError("unsupported historical parent template")
|
|
248
|
+
parent_sources.append(ast.get_source_segment(source, nodes[0]))
|
|
249
|
+
# The pinned parent retains its four-second deadline and subprocess error paths.
|
|
250
|
+
# Only strict, already-typed CodeRunResult fields are supported without Pydantic.
|
|
251
|
+
supervisor = "\n".join(
|
|
252
|
+
[
|
|
253
|
+
"import json\nimport os\nimport subprocess\nimport sys",
|
|
254
|
+
_RESULT_CONTAINER,
|
|
255
|
+
"_SUBPROCESS_CODE = " + repr(outer_text),
|
|
256
|
+
*parent_sources,
|
|
257
|
+
_SUPERVISOR_CALL,
|
|
258
|
+
]
|
|
259
|
+
)
|
|
260
|
+
return supervisor, inner.encode()
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def load_task(path: str | Path, prompt_id: str, prompt_text: str) -> dict:
|
|
264
|
+
"""Join the pinned task ID and verify the exact serialized hinted prompt."""
|
|
265
|
+
records = [
|
|
266
|
+
json.loads(line) for line in _checked_source(path, "fixture").splitlines()
|
|
267
|
+
]
|
|
268
|
+
matches = [row for row in records if str(row["id"]) == str(prompt_id)]
|
|
269
|
+
if len(matches) != 1:
|
|
270
|
+
raise ValueError("Aria problem ID must identify exactly one pinned task")
|
|
271
|
+
task = matches[0]
|
|
272
|
+
expected = task["prompt"][1]["content"].replace(
|
|
273
|
+
"and should pass all tests",
|
|
274
|
+
"and will be evaluated by calling a function called run_tests()",
|
|
275
|
+
)
|
|
276
|
+
if "\nuser\n" + expected != prompt_text:
|
|
277
|
+
raise ValueError("Aria task prompt does not match the pinned fixture")
|
|
278
|
+
return task
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def _case_value(node: ast.expr, *, expected: bool):
|
|
282
|
+
if isinstance(node, ast.Constant) and type(node.value) in (
|
|
283
|
+
str,
|
|
284
|
+
int,
|
|
285
|
+
float,
|
|
286
|
+
bool,
|
|
287
|
+
type(None),
|
|
288
|
+
):
|
|
289
|
+
return node.value
|
|
290
|
+
if isinstance(node, ast.List):
|
|
291
|
+
return [_case_value(value, expected=expected) for value in node.elts]
|
|
292
|
+
if expected and isinstance(node, ast.Name) and node.id == "inf":
|
|
293
|
+
return float("inf")
|
|
294
|
+
if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.UAdd, ast.USub)):
|
|
295
|
+
value = _case_value(node.operand, expected=expected)
|
|
296
|
+
if type(value) in (int, float):
|
|
297
|
+
return -value if isinstance(node.op, ast.USub) else value
|
|
298
|
+
raise ValueError("unsupported protected test value")
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def protected_cases(task: dict) -> list[dict]:
|
|
302
|
+
"""Extract trusted comparisons while keeping expected values out of workers."""
|
|
303
|
+
match = re.fullmatch(r"Solution\(\)\.([A-Za-z_]\w*)", task["func_name"])
|
|
304
|
+
if match is None or not task["gt_answer"]:
|
|
305
|
+
raise ValueError("unsupported protected task")
|
|
306
|
+
method = match[1]
|
|
307
|
+
cases = []
|
|
308
|
+
for test in task["gt_answer"]:
|
|
309
|
+
try:
|
|
310
|
+
body = ast.parse(test).body
|
|
311
|
+
except (SyntaxError, ValueError) as exc:
|
|
312
|
+
raise ValueError("unsupported protected test syntax") from exc
|
|
313
|
+
if (
|
|
314
|
+
len(body) != 1
|
|
315
|
+
or not isinstance(body[0], ast.Assert)
|
|
316
|
+
or body[0].msg is not None
|
|
317
|
+
):
|
|
318
|
+
raise ValueError("unsupported protected test assertion")
|
|
319
|
+
comparison = body[0].test
|
|
320
|
+
if not (
|
|
321
|
+
isinstance(comparison, ast.Compare)
|
|
322
|
+
and len(comparison.ops) == 1
|
|
323
|
+
and isinstance(comparison.ops[0], ast.Eq)
|
|
324
|
+
and len(comparison.comparators) == 1
|
|
325
|
+
):
|
|
326
|
+
raise ValueError("unsupported protected test comparison")
|
|
327
|
+
call = comparison.left
|
|
328
|
+
if not (
|
|
329
|
+
isinstance(call, ast.Call)
|
|
330
|
+
and not call.args
|
|
331
|
+
and ast.unparse(call.func) == task["func_name"]
|
|
332
|
+
and all(keyword.arg is not None for keyword in call.keywords)
|
|
333
|
+
and len({keyword.arg for keyword in call.keywords}) == len(call.keywords)
|
|
334
|
+
):
|
|
335
|
+
raise ValueError("unsupported protected test call")
|
|
336
|
+
cases.append(
|
|
337
|
+
{
|
|
338
|
+
"method": method,
|
|
339
|
+
"kwargs": {
|
|
340
|
+
keyword.arg: _case_value(keyword.value, expected=False)
|
|
341
|
+
for keyword in call.keywords
|
|
342
|
+
},
|
|
343
|
+
"expected": _case_value(comparison.comparators[0], expected=True),
|
|
344
|
+
}
|
|
345
|
+
)
|
|
346
|
+
return cases
|