codeplain 0.4.3.dev60__py3-none-any.whl → 0.4.3.dev62__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {codeplain-0.4.3.dev60.dist-info → codeplain-0.4.3.dev62.dist-info}/METADATA +1 -1
- {codeplain-0.4.3.dev60.dist-info → codeplain-0.4.3.dev62.dist-info}/RECORD +25 -20
- codeplain_REST_api.py +51 -47
- plain2code.py +2 -0
- render_machine/actions/fix_conformance_test.py +347 -145
- render_machine/actions/fix_unit_tests.py +1 -0
- render_machine/actions/prepare_testing_environment.py +89 -15
- render_machine/actions/render_conformance_tests.py +2 -0
- render_machine/actions/run_conformance_tests.py +5 -0
- render_machine/agent_tools.py +414 -83
- render_machine/conformance_fix_conversation.py +409 -0
- render_machine/render_context.py +3 -1
- render_machine/render_types.py +69 -0
- render_machine/render_utils.py +10 -1
- tests/test_agent_tools.py +180 -0
- tests/test_api_client_retries.py +52 -0
- tests/test_background_preparation.py +73 -0
- tests/test_conformance_fix_conversation.py +677 -0
- tests/test_fix_unit_tests_action.py +7 -0
- tests/test_fix_unit_tests_conformance_context.py +8 -106
- tests/test_render_finished_report.py +14 -0
- tests/test_script_timeout_hint.py +24 -0
- {codeplain-0.4.3.dev60.dist-info → codeplain-0.4.3.dev62.dist-info}/WHEEL +0 -0
- {codeplain-0.4.3.dev60.dist-info → codeplain-0.4.3.dev62.dist-info}/entry_points.txt +0 -0
- {codeplain-0.4.3.dev60.dist-info → codeplain-0.4.3.dev62.dist-info}/licenses/LICENSE +0 -0
|
@@ -0,0 +1,677 @@
|
|
|
1
|
+
"""Tests for fixing conformance tests through a conversation between a fixer agent and a reviewer agent."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from types import SimpleNamespace
|
|
5
|
+
|
|
6
|
+
import pytest
|
|
7
|
+
|
|
8
|
+
import plain_spec
|
|
9
|
+
from memory_management import MemoryManager
|
|
10
|
+
from plain2code_exceptions import ConflictingRequirements
|
|
11
|
+
from plain2code_utils import qualified_frid
|
|
12
|
+
from render_machine import conformance_fix_conversation as conversation
|
|
13
|
+
from render_machine.actions.fix_conformance_test import FixConformanceTest
|
|
14
|
+
from render_machine.actions.run_conformance_tests import RunConformanceTests
|
|
15
|
+
from render_machine.implementation_code_helpers import ImplementationCodeHelpers
|
|
16
|
+
from render_machine.render_types import ConformanceTestsRunningContext, ScriptExecutionHistory
|
|
17
|
+
|
|
18
|
+
ISSUE = "FAILED test_app.py::test_add - assert 2 == 3"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def write(path, content):
|
|
22
|
+
return {"name": "write_file", "args": {"file_path": path, "content": content}}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def submit_fix(failing_reason="IMPLEMENTATION_CODE", **args):
|
|
26
|
+
return {
|
|
27
|
+
"name": "submit_fix",
|
|
28
|
+
"args": {"root_cause": "off by one", "changes_made": "add one", "failing_reason": failing_reason, **args},
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def review(verdict, comments="", objected_files=None):
|
|
33
|
+
args = {"verdict": verdict, "comments": comments}
|
|
34
|
+
if objected_files is not None:
|
|
35
|
+
args["objected_files"] = objected_files
|
|
36
|
+
return {"name": "submit_review", "args": args}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class ScriptedAgentAPI:
|
|
40
|
+
"""Plays the server side of the agent sessions: each task type answers with its scripted turns in order.
|
|
41
|
+
A turn is a list of tool calls (each turn of the server asking for them), or a raw response dict."""
|
|
42
|
+
|
|
43
|
+
def __init__(self, fixer_turns=(), reviewer_turns=()):
|
|
44
|
+
self.turns = {conversation.FIXER_TASK: list(fixer_turns), conversation.REVIEWER_TASK: list(reviewer_turns)}
|
|
45
|
+
self.session_tasks = {}
|
|
46
|
+
self.calls = []
|
|
47
|
+
self.conflict_reports = []
|
|
48
|
+
|
|
49
|
+
def agent_start(self, task_type, task_params, frid, module_name, run_state):
|
|
50
|
+
session_id = f"{task_type}-{len(self.session_tasks) + 1}"
|
|
51
|
+
self.session_tasks[session_id] = task_type
|
|
52
|
+
self.calls.append(("start", task_type, session_id, task_params))
|
|
53
|
+
return self._next(session_id)
|
|
54
|
+
|
|
55
|
+
def agent_continue(self, session_id, tool_results, frid, module_name, run_state):
|
|
56
|
+
self.calls.append(("continue", self.session_tasks[session_id], session_id, tool_results))
|
|
57
|
+
return self._next(session_id)
|
|
58
|
+
|
|
59
|
+
def report_conflicting_requirements(self, *args, **kwargs):
|
|
60
|
+
self.conflict_reports.append(args)
|
|
61
|
+
raise ConflictingRequirements("Conflicting requirements: they conflict")
|
|
62
|
+
|
|
63
|
+
def _next(self, session_id):
|
|
64
|
+
turn = self.turns[self.session_tasks[session_id]].pop(0)
|
|
65
|
+
if isinstance(turn, dict):
|
|
66
|
+
return {"session_id": session_id, **turn}
|
|
67
|
+
calls = [{"id": f"call-{len(self.calls)}-{index}", **call} for index, call in enumerate(turn)]
|
|
68
|
+
return {"session_id": session_id, "status": "tool_calls", "calls": calls}
|
|
69
|
+
|
|
70
|
+
def of(self, kind, task_type):
|
|
71
|
+
return [call for call in self.calls if call[0] == kind and call[1] == task_type]
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class FakeConformanceTests:
|
|
75
|
+
def __init__(self, base):
|
|
76
|
+
self.base = base
|
|
77
|
+
|
|
78
|
+
def get_module_conformance_tests_folder(self, module_name):
|
|
79
|
+
return str(self.base / "conformance") if module_name == "mod" else str(self.base / f"conformance_{module_name}")
|
|
80
|
+
|
|
81
|
+
def get_source_conformance_test_folder_name(self, module_name, required_modules, testing_module, folder):
|
|
82
|
+
return folder, folder + "_copy"
|
|
83
|
+
|
|
84
|
+
def get_conformance_tests_json(self, module_name):
|
|
85
|
+
return getattr(self, "tests_json", {}).get(module_name, {})
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
@pytest.fixture
|
|
89
|
+
def project(tmp_path, monkeypatch):
|
|
90
|
+
build = tmp_path / "build"
|
|
91
|
+
build.mkdir()
|
|
92
|
+
(build / "app.py").write_text("def add(a, b):\n return a + b\n")
|
|
93
|
+
tests_folder = tmp_path / "conformance" / "add"
|
|
94
|
+
tests_folder.mkdir(parents=True)
|
|
95
|
+
(tests_folder / "test_app.py").write_text("def test_add():\n assert add(1, 2) == 3\n")
|
|
96
|
+
memory = tmp_path / "memory"
|
|
97
|
+
memory.mkdir()
|
|
98
|
+
monkeypatch.setattr(ImplementationCodeHelpers, "get_code_diff", staticmethod(lambda *args: {"app.py": "+x"}))
|
|
99
|
+
monkeypatch.setattr(
|
|
100
|
+
plain_spec,
|
|
101
|
+
"get_specifications_for_frid",
|
|
102
|
+
lambda tree, frid: (
|
|
103
|
+
{
|
|
104
|
+
plain_spec.DEFINITIONS: ["- :Number: is a number."],
|
|
105
|
+
plain_spec.FUNCTIONAL_REQUIREMENTS: ["- Add numbers."],
|
|
106
|
+
plain_spec.ACCEPTANCE_TESTS: [{"id": "a1", "text": "1 + 2 is 3"}],
|
|
107
|
+
},
|
|
108
|
+
None,
|
|
109
|
+
),
|
|
110
|
+
)
|
|
111
|
+
return SimpleNamespace(root=tmp_path, build=build, tests_folder=tests_folder, memory=memory)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def make_render_context(project, api, testing_frid="1", frid="1", testing_module="mod", required_module_names=()):
|
|
115
|
+
ctx = ConformanceTestsRunningContext(
|
|
116
|
+
current_testing_module_name=testing_module,
|
|
117
|
+
current_testing_frid=testing_frid,
|
|
118
|
+
fix_attempts=0,
|
|
119
|
+
conformance_tests_json={testing_frid: {"folder_name": str(project.tests_folder)}},
|
|
120
|
+
conformance_tests_render_attempts=0,
|
|
121
|
+
current_testing_frid_specifications={
|
|
122
|
+
plain_spec.FUNCTIONAL_REQUIREMENTS: ["- Add numbers."],
|
|
123
|
+
plain_spec.CONFORMANCE_TESTS: [{"id": "s1", "text": "adding 1 and 2 gives 3"}],
|
|
124
|
+
},
|
|
125
|
+
should_prepare_testing_environment=False,
|
|
126
|
+
frid_being_implemented=frid,
|
|
127
|
+
)
|
|
128
|
+
return SimpleNamespace(
|
|
129
|
+
codeplain_api=api,
|
|
130
|
+
build_folder=str(project.build),
|
|
131
|
+
memory_manager=MemoryManager(api, str(project.memory)),
|
|
132
|
+
conformance_tests=FakeConformanceTests(project.root),
|
|
133
|
+
plain_module=SimpleNamespace(
|
|
134
|
+
build_folder=str(project.root / "plain_modules"),
|
|
135
|
+
all_required_modules=[SimpleNamespace(module_name=name) for name in required_module_names],
|
|
136
|
+
),
|
|
137
|
+
conformance_tests_running_context=ctx,
|
|
138
|
+
plain_source_tree={},
|
|
139
|
+
module_name="mod",
|
|
140
|
+
required_modules=[],
|
|
141
|
+
frid_context=SimpleNamespace(frid=frid, linked_resources={}, changed_files={"app.py"}),
|
|
142
|
+
get_required_modules_functionalities=lambda: {},
|
|
143
|
+
testing_functionality_label=lambda: f"{ctx.current_testing_module_name}:{ctx.current_testing_frid}",
|
|
144
|
+
testing_functionality_position=lambda: None,
|
|
145
|
+
run_state=SimpleNamespace(render_id="render-id"),
|
|
146
|
+
conformance_tests_script=None,
|
|
147
|
+
prepare_environment_script=None,
|
|
148
|
+
script_execution_history=ScriptExecutionHistory(),
|
|
149
|
+
test_script_timeout=None,
|
|
150
|
+
stop_event=None,
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def run_fix(render_context, issue=ISSUE):
|
|
155
|
+
return FixConformanceTest().execute(render_context, {"previous_conformance_tests_issue": issue})
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def test_approved_implementation_fix_runs_the_unit_tests_next_and_is_handed_to_the_unit_tests_fixer(project):
|
|
159
|
+
api = ScriptedAgentAPI(
|
|
160
|
+
fixer_turns=[[write("app.py", "def add(a, b):\n return a + b + 1\n")], [submit_fix()]],
|
|
161
|
+
reviewer_turns=[[review("approve", "Addresses the root cause.")]],
|
|
162
|
+
)
|
|
163
|
+
render_context = make_render_context(project, api)
|
|
164
|
+
|
|
165
|
+
outcome, _ = run_fix(render_context)
|
|
166
|
+
|
|
167
|
+
ctx = render_context.conformance_tests_running_context
|
|
168
|
+
assert outcome == FixConformanceTest.IMPLEMENTATION_CODE_UPDATED
|
|
169
|
+
assert ctx.should_prepare_testing_environment and ctx.test_that_triggered_code_change == ("mod", "1")
|
|
170
|
+
[recorded] = ctx.implementation_code_fixes
|
|
171
|
+
assert recorded["hypothesis"] == "off by one" and recorded["approach"] == "add one"
|
|
172
|
+
assert list(recorded["code_diff"]) == ["app.py"] and "+ return a + b + 1" in recorded["code_diff"]["app.py"]
|
|
173
|
+
|
|
174
|
+
[(_, _, _, fixer_params)] = api.of("start", conversation.FIXER_TASK)
|
|
175
|
+
assert fixer_params["conformance_tests_issue"] == ISSUE
|
|
176
|
+
assert fixer_params["conformance_tests_folder"] == str(project.tests_folder)
|
|
177
|
+
assert fixer_params["is_regression"] is False and "code_diff" not in fixer_params
|
|
178
|
+
assert "#### Scenario `s1`" in fixer_params["conformance_scenarios"]
|
|
179
|
+
assert "#### Scenario `a1`" in fixer_params["acceptance_tests"]
|
|
180
|
+
assert str(project.tests_folder / "test_app.py") in fixer_params["conformance_tests_files"]
|
|
181
|
+
assert "app.py" in fixer_params["relevant_files"]
|
|
182
|
+
|
|
183
|
+
[(_, _, _, reviewer_params)] = api.of("start", conversation.REVIEWER_TASK)
|
|
184
|
+
fix_submission = reviewer_params["fix_submission"]
|
|
185
|
+
assert reviewer_params["conformance_tests_issue"] == ISSUE
|
|
186
|
+
assert (
|
|
187
|
+
fix_submission["changes_made"] == "add one" and "+ return a + b + 1" in fix_submission["code_diff"]["app.py"]
|
|
188
|
+
)
|
|
189
|
+
# The reviewer sees the conformance tests as they were before the fix.
|
|
190
|
+
assert (
|
|
191
|
+
"assert add(1, 2) == 3" in reviewer_params["conformance_tests_files"][str(project.tests_folder / "test_app.py")]
|
|
192
|
+
)
|
|
193
|
+
|
|
194
|
+
agents = ctx.conformance_fix_agents
|
|
195
|
+
assert agents.fixer.pending_call_id and agents.reviewer.pending_call_id and agents.approved_fix_tested
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def test_conformance_tests_fix_runs_the_conformance_tests_next(project):
|
|
199
|
+
test_file = str(project.tests_folder / "test_app.py")
|
|
200
|
+
api = ScriptedAgentAPI(
|
|
201
|
+
fixer_turns=[
|
|
202
|
+
[write(test_file, "def test_add():\n assert add(1, 2) == 4\n")],
|
|
203
|
+
[submit_fix("CONFORMANCE_TESTS")],
|
|
204
|
+
],
|
|
205
|
+
reviewer_turns=[[review("approve")]],
|
|
206
|
+
)
|
|
207
|
+
render_context = make_render_context(project, api)
|
|
208
|
+
|
|
209
|
+
outcome, _ = run_fix(render_context)
|
|
210
|
+
|
|
211
|
+
assert outcome == FixConformanceTest.IMPLEMENTATION_CODE_NOT_UPDATED
|
|
212
|
+
assert render_context.conformance_tests_running_context.implementation_code_fixes == []
|
|
213
|
+
assert "== 4" in open(test_file).read()
|
|
214
|
+
[(_, _, _, reviewer_params)] = api.of("start", conversation.REVIEWER_TASK)
|
|
215
|
+
assert list(reviewer_params["fix_submission"]["code_diff"]) == [test_file]
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _sessions_ending_without_an_approved_fix(first_turn):
|
|
219
|
+
ended = {"status": "completed", "result": "handoff"}
|
|
220
|
+
return [first_turn, ended] + [ended] * (conversation.MAX_FIXER_SESSIONS_PER_FIX - 1)
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def test_unapproved_implementation_changes_are_kept_and_tested_after_the_unit_tests(project):
|
|
224
|
+
# The fixer never got a fix approved: its changes stay, and like an approved implementation change they are
|
|
225
|
+
# tested after the unit tests and a fresh preparation, never on the environment prepared for the older code.
|
|
226
|
+
api = ScriptedAgentAPI(
|
|
227
|
+
fixer_turns=_sessions_ending_without_an_approved_fix([write("app.py", "def add(a, b):\n return 3\n")])
|
|
228
|
+
)
|
|
229
|
+
render_context = make_render_context(project, api)
|
|
230
|
+
|
|
231
|
+
outcome, _ = run_fix(render_context)
|
|
232
|
+
|
|
233
|
+
ctx = render_context.conformance_tests_running_context
|
|
234
|
+
assert len(api.of("start", conversation.FIXER_TASK)) == conversation.MAX_FIXER_SESSIONS_PER_FIX
|
|
235
|
+
assert not api.of("start", conversation.REVIEWER_TASK)
|
|
236
|
+
assert outcome == FixConformanceTest.IMPLEMENTATION_CODE_UPDATED
|
|
237
|
+
assert "return 3" in (project.build / "app.py").read_text()
|
|
238
|
+
assert ctx.should_prepare_testing_environment and ctx.test_that_triggered_code_change == ("mod", "1")
|
|
239
|
+
# Handed to the unit tests fixer, so it keeps the change instead of reverting it.
|
|
240
|
+
[recorded] = ctx.implementation_code_fixes
|
|
241
|
+
assert "without getting them approved" in recorded["approach"]
|
|
242
|
+
assert "+ return 3" in recorded["code_diff"]["app.py"]
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def test_unapproved_changes_to_the_conformance_tests_only_run_the_conformance_tests_again(project):
|
|
246
|
+
test_file = str(project.tests_folder / "test_app.py")
|
|
247
|
+
api = ScriptedAgentAPI(
|
|
248
|
+
fixer_turns=_sessions_ending_without_an_approved_fix([write(test_file, "def test_add():\n pass\n")])
|
|
249
|
+
)
|
|
250
|
+
render_context = make_render_context(project, api)
|
|
251
|
+
|
|
252
|
+
outcome, _ = run_fix(render_context)
|
|
253
|
+
|
|
254
|
+
ctx = render_context.conformance_tests_running_context
|
|
255
|
+
assert outcome == FixConformanceTest.IMPLEMENTATION_CODE_NOT_UPDATED
|
|
256
|
+
assert "pass" in open(test_file).read()
|
|
257
|
+
assert not ctx.should_prepare_testing_environment and ctx.implementation_code_fixes == []
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def test_rejected_fix_goes_back_to_the_fixer_with_the_comments_and_the_revision_is_reviewed(project):
|
|
261
|
+
api = ScriptedAgentAPI(
|
|
262
|
+
fixer_turns=[
|
|
263
|
+
[write("app.py", "def add(a, b):\n return 3\n")],
|
|
264
|
+
[submit_fix()],
|
|
265
|
+
[write("app.py", "def add(a, b):\n return a + b\n# fixed\n")],
|
|
266
|
+
[submit_fix(review_response="Implemented addition instead of a constant.")],
|
|
267
|
+
],
|
|
268
|
+
reviewer_turns=[[review("reject", "Returning a constant only fits the test.")], [review("approve")]],
|
|
269
|
+
)
|
|
270
|
+
render_context = make_render_context(project, api)
|
|
271
|
+
|
|
272
|
+
outcome, _ = run_fix(render_context)
|
|
273
|
+
|
|
274
|
+
assert outcome == FixConformanceTest.IMPLEMENTATION_CODE_UPDATED
|
|
275
|
+
fixer_answers = [results[-1] for _, _, _, results in api.of("continue", conversation.FIXER_TASK)]
|
|
276
|
+
[rejection] = [answer for answer in fixer_answers if "review" in answer]
|
|
277
|
+
assert rejection["output"] == conversation.REJECTED_MESSAGE
|
|
278
|
+
assert rejection["review"] == {"comments": "Returning a constant only fits the test."}
|
|
279
|
+
|
|
280
|
+
[(_, _, _, reviewer_results)] = api.of("continue", conversation.REVIEWER_TASK)
|
|
281
|
+
revision = reviewer_results[-1]
|
|
282
|
+
assert revision["output"] == conversation.REVIEW_REVISION_MESSAGE and "test_output" not in revision
|
|
283
|
+
assert revision["fix_submission"]["review_response"] == "Implemented addition instead of a constant."
|
|
284
|
+
# The revision is shown as the whole fix since the tests ran: the constant is gone again.
|
|
285
|
+
assert "+# fixed" in revision["fix_submission"]["code_diff"]["app.py"]
|
|
286
|
+
assert "return 3" not in revision["fix_submission"]["code_diff"]["app.py"]
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def test_fix_rejected_too_often_loses_only_the_objected_changes_and_the_sessions_go_on(project):
|
|
290
|
+
rejections = conversation.MAX_REJECTIONS
|
|
291
|
+
fixer_turns = [[write("test_app_extra.py", "x = 1\n"), write("app.py", "def add(a, b):\n return b + a\n")]]
|
|
292
|
+
fixer_turns += [[submit_fix()] for _ in range(rejections + 1)]
|
|
293
|
+
reviewer_turns = [[review("reject", f"No {number}.", ["test_app_extra.py"])] for number in range(rejections)]
|
|
294
|
+
api = ScriptedAgentAPI(fixer_turns=fixer_turns, reviewer_turns=reviewer_turns + [[review("approve")]])
|
|
295
|
+
render_context = make_render_context(project, api)
|
|
296
|
+
|
|
297
|
+
outcome, _ = run_fix(render_context)
|
|
298
|
+
|
|
299
|
+
assert outcome == FixConformanceTest.IMPLEMENTATION_CODE_UPDATED
|
|
300
|
+
# Only the objected file was reverted; the other change stays, and nobody started over.
|
|
301
|
+
assert not (project.build / "test_app_extra.py").exists()
|
|
302
|
+
assert "return b + a" in (project.build / "app.py").read_text()
|
|
303
|
+
assert len(api.of("start", conversation.FIXER_TASK)) == 1 and len(api.of("start", conversation.REVIEWER_TASK)) == 1
|
|
304
|
+
answers = [results[-1] for (_, _, _, results) in api.of("continue", conversation.FIXER_TASK)]
|
|
305
|
+
rejected = [a for a in answers if a.get("review")]
|
|
306
|
+
assert "reverted_files" not in rejected[rejections - 2]["review"]
|
|
307
|
+
assert rejected[rejections - 1]["review"]["reverted_files"] == ["test_app_extra.py"]
|
|
308
|
+
# The reviewer is told how many fixes it rejected in a row.
|
|
309
|
+
reviews = [results[-1]["fix_submission"] for (_, _, _, results) in api.of("continue", conversation.REVIEWER_TASK)]
|
|
310
|
+
assert [r["consecutive_rejections"] for r in reviews] == list(range(1, rejections + 1))
|
|
311
|
+
assert list(reviews[-1]["code_diff"]) == ["app.py"]
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def test_a_file_read_again_after_it_was_reverted_is_read_from_disk(project):
|
|
315
|
+
rejections = conversation.MAX_REJECTIONS
|
|
316
|
+
read_extra = {"name": "read_file", "args": {"file_path": "test_app_extra.py"}}
|
|
317
|
+
fixer_turns = [[write("test_app_extra.py", "x = 1\n"), write("app.py", "v1\n")], [read_extra]]
|
|
318
|
+
fixer_turns += [[submit_fix()] for _ in range(rejections)] + [[read_extra], [submit_fix()]]
|
|
319
|
+
reviewer_turns = [[review("reject", "No.", ["test_app_extra.py"])] for _ in range(rejections)]
|
|
320
|
+
api = ScriptedAgentAPI(fixer_turns=fixer_turns, reviewer_turns=reviewer_turns + [[review("approve")]])
|
|
321
|
+
|
|
322
|
+
run_fix(make_render_context(project, api))
|
|
323
|
+
|
|
324
|
+
first_read, second_read = [
|
|
325
|
+
results
|
|
326
|
+
for (_, _, _, results) in api.of("continue", conversation.FIXER_TASK)
|
|
327
|
+
if len(results) == 1 and "review" not in results[0]
|
|
328
|
+
]
|
|
329
|
+
assert "x = 1" in str(first_read)
|
|
330
|
+
# Not the cached content from before the revert: the file is gone.
|
|
331
|
+
assert "x = 1" not in str(second_read) and "not found" in str(second_read).lower()
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def test_fix_that_changes_nothing_is_sent_back_without_a_review(project):
|
|
335
|
+
api = ScriptedAgentAPI(
|
|
336
|
+
fixer_turns=[[submit_fix()], [write("app.py", "def add(a, b):\n return a + b # ok\n")], [submit_fix()]],
|
|
337
|
+
reviewer_turns=[[review("approve")]],
|
|
338
|
+
)
|
|
339
|
+
render_context = make_render_context(project, api)
|
|
340
|
+
|
|
341
|
+
run_fix(render_context)
|
|
342
|
+
|
|
343
|
+
first_answer = api.of("continue", conversation.FIXER_TASK)[0][3][-1]
|
|
344
|
+
assert first_answer["output"] == conversation.EMPTY_FIX_MESSAGE
|
|
345
|
+
assert len(api.of("start", conversation.REVIEWER_TASK)) == 1
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def test_tests_still_failing_continue_both_sessions_with_the_new_output(project):
|
|
349
|
+
api = ScriptedAgentAPI(
|
|
350
|
+
fixer_turns=[
|
|
351
|
+
[write("app.py", "v1\n")],
|
|
352
|
+
[submit_fix()],
|
|
353
|
+
[write("app.py", "v2\n")],
|
|
354
|
+
[submit_fix()],
|
|
355
|
+
],
|
|
356
|
+
reviewer_turns=[[review("approve")], [review("approve")]],
|
|
357
|
+
)
|
|
358
|
+
render_context = make_render_context(project, api)
|
|
359
|
+
run_fix(render_context)
|
|
360
|
+
|
|
361
|
+
outcome, _ = run_fix(render_context, issue="STILL FAILING")
|
|
362
|
+
|
|
363
|
+
assert outcome == FixConformanceTest.IMPLEMENTATION_CODE_UPDATED
|
|
364
|
+
assert len(api.of("start", conversation.FIXER_TASK)) == 1 and len(api.of("start", conversation.REVIEWER_TASK)) == 1
|
|
365
|
+
fixer_answer = api.of("continue", conversation.FIXER_TASK)[1][3][-1]
|
|
366
|
+
assert fixer_answer["output"].startswith(conversation.TESTS_STILL_FAIL_MESSAGE)
|
|
367
|
+
assert fixer_answer["test_output"] == "STILL FAILING"
|
|
368
|
+
[(_, _, _, reviewer_results)] = api.of("continue", conversation.REVIEWER_TASK)
|
|
369
|
+
assert reviewer_results[-1]["output"].startswith(conversation.REVIEW_AFTER_TESTS_MESSAGE)
|
|
370
|
+
assert reviewer_results[-1]["test_output"] == "STILL FAILING"
|
|
371
|
+
# Only the second fix is shown: the first one was approved and tested.
|
|
372
|
+
assert "-v1" in reviewer_results[-1]["fix_submission"]["code_diff"]["app.py"]
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def test_fixer_is_told_when_runs_after_approved_fixes_keep_failing(project, monkeypatch):
|
|
376
|
+
monkeypatch.setattr(conversation, "NOT_CONVERGING_AFTER_FAILED_RUNS", 2)
|
|
377
|
+
fixer_turns, reviewer_turns = [], []
|
|
378
|
+
for number in range(3):
|
|
379
|
+
fixer_turns += [[write("app.py", f"v{number}\n")], [submit_fix()]]
|
|
380
|
+
reviewer_turns.append([review("approve")])
|
|
381
|
+
api = ScriptedAgentAPI(fixer_turns=fixer_turns, reviewer_turns=reviewer_turns)
|
|
382
|
+
render_context = make_render_context(project, api)
|
|
383
|
+
|
|
384
|
+
for _ in range(3):
|
|
385
|
+
run_fix(render_context, issue="STILL FAILING")
|
|
386
|
+
|
|
387
|
+
answers = [results[-1]["output"] for (_, _, _, results) in api.of("continue", conversation.FIXER_TASK)]
|
|
388
|
+
still_failing = [a for a in answers if a.startswith(conversation.TESTS_STILL_FAIL_MESSAGE)]
|
|
389
|
+
assert len(still_failing) == 2
|
|
390
|
+
assert "not converging" not in still_failing[0]
|
|
391
|
+
assert "failed 2 times in a row" in still_failing[1] and "not converging" in still_failing[1]
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
def test_passing_tests_end_both_agents(project, monkeypatch):
|
|
395
|
+
api = ScriptedAgentAPI(
|
|
396
|
+
fixer_turns=[[write("app.py", "v1\n")], [submit_fix()]], reviewer_turns=[[review("approve")]]
|
|
397
|
+
)
|
|
398
|
+
render_context = make_render_context(project, api)
|
|
399
|
+
render_context.conformance_tests_script = "run.sh"
|
|
400
|
+
run_fix(render_context)
|
|
401
|
+
monkeypatch.setattr(
|
|
402
|
+
"render_machine.render_utils.execute_script", lambda *args, **kwargs: (0, "", None), raising=True
|
|
403
|
+
)
|
|
404
|
+
|
|
405
|
+
outcome, _ = RunConformanceTests().execute(render_context, None)
|
|
406
|
+
|
|
407
|
+
assert outcome == RunConformanceTests.SUCCESSFUL_OUTCOME
|
|
408
|
+
assert render_context.conformance_tests_running_context.conformance_fix_agents is None
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
def test_failing_tests_of_another_functionality_get_new_agents(project):
|
|
412
|
+
api = ScriptedAgentAPI(
|
|
413
|
+
fixer_turns=[[write("app.py", "v1\n")], [submit_fix()], [write("app.py", "v2\n")], [submit_fix()]],
|
|
414
|
+
reviewer_turns=[[review("approve")], [review("approve")]],
|
|
415
|
+
)
|
|
416
|
+
render_context = make_render_context(project, api, testing_frid="1", frid="2")
|
|
417
|
+
run_fix(render_context)
|
|
418
|
+
ctx = render_context.conformance_tests_running_context
|
|
419
|
+
ctx.current_testing_frid = "2"
|
|
420
|
+
ctx.set_conformance_tests_json("mod", {"2": {"folder_name": str(project.tests_folder)}})
|
|
421
|
+
|
|
422
|
+
run_fix(render_context)
|
|
423
|
+
|
|
424
|
+
assert len(api.of("start", conversation.FIXER_TASK)) == 2
|
|
425
|
+
assert ctx.conformance_fix_agents.tested == ("mod", "2")
|
|
426
|
+
regression_params = api.of("start", conversation.FIXER_TASK)[0][3]
|
|
427
|
+
assert regression_params["is_regression"] is True and regression_params["code_diff"] == {"app.py": "+x"}
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
def test_agreed_conflict_is_counted_first_and_reported_the_second_time(project):
|
|
431
|
+
conflict = submit_fix("CONFLICTING_REQUIREMENTS", conflict_reason="1 and 2 contradict each other.")
|
|
432
|
+
api = ScriptedAgentAPI(
|
|
433
|
+
fixer_turns=[[write("app.py", "v1\n")], [conflict], [conflict]],
|
|
434
|
+
reviewer_turns=[[review("approve")], [review("approve")]],
|
|
435
|
+
)
|
|
436
|
+
render_context = make_render_context(project, api, testing_frid="1", frid="2")
|
|
437
|
+
ctx = render_context.conformance_tests_running_context
|
|
438
|
+
|
|
439
|
+
outcome, _ = run_fix(render_context)
|
|
440
|
+
assert outcome == FixConformanceTest.IMPLEMENTATION_CODE_UPDATED
|
|
441
|
+
assert ctx.conflicting_requirement_count == 1 and api.conflict_reports == []
|
|
442
|
+
# A second declaration in the same session is still reviewed and, once agreed, ends the render.
|
|
443
|
+
assert "conflict_already_reported" not in api.of("start", conversation.FIXER_TASK)[0][3]
|
|
444
|
+
|
|
445
|
+
with pytest.raises(ConflictingRequirements):
|
|
446
|
+
run_fix(render_context)
|
|
447
|
+
[report] = api.conflict_reports
|
|
448
|
+
assert report[6] == "CONFLICTING_REQUIREMENTS" and report[7] == "1 and 2 contradict each other."
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
def test_conflict_of_the_current_functionality_is_with_its_acceptance_tests(project):
|
|
452
|
+
conflict = submit_fix("CONFLICTING_REQUIREMENTS", conflict_reason="a1 contradicts the requirement.")
|
|
453
|
+
api = ScriptedAgentAPI(fixer_turns=[[conflict]], reviewer_turns=[[review("approve")]])
|
|
454
|
+
render_context = make_render_context(project, api)
|
|
455
|
+
ctx = render_context.conformance_tests_running_context
|
|
456
|
+
ctx.conflicting_requirement_count = 1
|
|
457
|
+
ctx.conflicting_module_name, ctx.conflicting_frid = "mod", "1"
|
|
458
|
+
|
|
459
|
+
with pytest.raises(ConflictingRequirements):
|
|
460
|
+
run_fix(render_context)
|
|
461
|
+
|
|
462
|
+
assert api.conflict_reports[0][6] == "CONFLICTING_ACCEPTANCE_TESTS"
|
|
463
|
+
assert api.of("start", conversation.FIXER_TASK)[0][3]["conflict_already_reported"] is True
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
def test_reviewer_cannot_change_files_and_fixer_cannot_write_outside_its_folders(project, tmp_path):
|
|
467
|
+
outside = str(tmp_path / "outside.py")
|
|
468
|
+
api = ScriptedAgentAPI(
|
|
469
|
+
fixer_turns=[[write(outside, "x\n"), write("app.py", "v1\n")], [submit_fix()]],
|
|
470
|
+
reviewer_turns=[[write("app.py", "reviewer\n")], [review("approve")]],
|
|
471
|
+
)
|
|
472
|
+
render_context = make_render_context(project, api)
|
|
473
|
+
|
|
474
|
+
run_fix(render_context)
|
|
475
|
+
|
|
476
|
+
fixer_results = api.of("continue", conversation.FIXER_TASK)[0][3]
|
|
477
|
+
assert fixer_results[0]["output"].startswith("Error: write access denied")
|
|
478
|
+
[(_, _, _, reviewer_results)] = api.of("continue", conversation.REVIEWER_TASK)
|
|
479
|
+
assert reviewer_results[0]["output"].startswith("Error: unknown tool")
|
|
480
|
+
assert (project.build / "app.py").read_text() == "v1\n" and not os.path.exists(outside)
|
|
481
|
+
|
|
482
|
+
|
|
483
|
+
def read(path):
|
|
484
|
+
return {"name": "read_file", "args": {"file_path": path}}
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def test_agents_read_the_tests_of_other_functionalities_and_required_modules_but_only_write_their_own(project):
|
|
488
|
+
sibling = project.root / "conformance" / "login" / "test_login.py"
|
|
489
|
+
sibling.parent.mkdir()
|
|
490
|
+
sibling.write_text("WORKING HARNESS\n")
|
|
491
|
+
required = project.root / "conformance_base" / "health" / "test_health.py"
|
|
492
|
+
required.parent.mkdir(parents=True)
|
|
493
|
+
required.write_text("BASE HARNESS\n")
|
|
494
|
+
api = ScriptedAgentAPI(
|
|
495
|
+
fixer_turns=[
|
|
496
|
+
[read(str(sibling)), read(str(required)), write(str(sibling), "x\n"), write("app.py", "v1\n")],
|
|
497
|
+
[submit_fix()],
|
|
498
|
+
],
|
|
499
|
+
reviewer_turns=[[read(str(sibling)), read(str(required))], [review("approve")]],
|
|
500
|
+
)
|
|
501
|
+
render_context = make_render_context(project, api, required_module_names=["base"])
|
|
502
|
+
|
|
503
|
+
run_fix(render_context)
|
|
504
|
+
|
|
505
|
+
fixer_results = [r["output"] for r in api.of("continue", conversation.FIXER_TASK)[0][3]]
|
|
506
|
+
assert "WORKING HARNESS" in fixer_results[0] and "BASE HARNESS" in fixer_results[1]
|
|
507
|
+
assert fixer_results[2].startswith("Error: write access denied") and sibling.read_text() == "WORKING HARNESS\n"
|
|
508
|
+
reviewer_results = [r["output"] for r in api.of("continue", conversation.REVIEWER_TASK)[0][3]]
|
|
509
|
+
assert "WORKING HARNESS" in reviewer_results[0] and "BASE HARNESS" in reviewer_results[1]
|
|
510
|
+
[(_, _, _, start_params)] = api.of("start", conversation.FIXER_TASK)
|
|
511
|
+
assert start_params["readable_conformance_tests_folders"] == [
|
|
512
|
+
str(project.root / "conformance"),
|
|
513
|
+
str(project.root / "conformance_base"),
|
|
514
|
+
]
|
|
515
|
+
|
|
516
|
+
|
|
517
|
+
def test_agents_read_every_rendered_module_and_are_told_the_layout(project):
|
|
518
|
+
other = project.root / "plain_modules" / "base" / "code" / "lib.py"
|
|
519
|
+
other.parent.mkdir(parents=True)
|
|
520
|
+
other.write_text("BASE CODE\n")
|
|
521
|
+
api = ScriptedAgentAPI(
|
|
522
|
+
fixer_turns=[[read(str(other)), write(str(other), "x\n"), write("app.py", "v1\n")], [submit_fix()]],
|
|
523
|
+
reviewer_turns=[[read(str(other))], [review("approve")]],
|
|
524
|
+
)
|
|
525
|
+
render_context = make_render_context(project, api, required_module_names=["base"])
|
|
526
|
+
|
|
527
|
+
run_fix(render_context)
|
|
528
|
+
|
|
529
|
+
fixer_results = [r["output"] for r in api.of("continue", conversation.FIXER_TASK)[0][3]]
|
|
530
|
+
assert "BASE CODE" in fixer_results[0] and fixer_results[1].startswith("Error: write access denied")
|
|
531
|
+
assert other.read_text() == "BASE CODE\n"
|
|
532
|
+
assert "BASE CODE" in api.of("continue", conversation.REVIEWER_TASK)[0][3][0]["output"]
|
|
533
|
+
[(_, _, _, start_params)] = api.of("start", conversation.FIXER_TASK)
|
|
534
|
+
assert start_params["plain_modules_folder"] == str(project.root / "plain_modules")
|
|
535
|
+
assert start_params["required_module_names"] == ["base"]
|
|
536
|
+
|
|
537
|
+
|
|
538
|
+
@pytest.mark.parametrize("regression", [True, False])
|
|
539
|
+
def test_a_regression_fixer_may_fix_the_other_earlier_tests_of_the_module(project, regression):
|
|
540
|
+
sibling = project.root / "conformance" / "login" / "test_login.py"
|
|
541
|
+
sibling.parent.mkdir()
|
|
542
|
+
sibling.write_text("OLD SETUP\n")
|
|
543
|
+
api = ScriptedAgentAPI(
|
|
544
|
+
fixer_turns=[[write(str(sibling), "NEW SETUP\n"), write("app.py", "v1\n")], [submit_fix()]],
|
|
545
|
+
reviewer_turns=[[review("approve")]],
|
|
546
|
+
)
|
|
547
|
+
render_context = make_render_context(project, api, testing_frid="1", frid="2" if regression else "1")
|
|
548
|
+
|
|
549
|
+
run_fix(render_context)
|
|
550
|
+
|
|
551
|
+
result = api.of("continue", conversation.FIXER_TASK)[0][3][0]["output"]
|
|
552
|
+
[(_, _, _, start_params)] = api.of("start", conversation.FIXER_TASK)
|
|
553
|
+
if regression:
|
|
554
|
+
assert sibling.read_text() == "NEW SETUP\n"
|
|
555
|
+
assert start_params["writable_module_tests_folder"] == str(project.root / "conformance")
|
|
556
|
+
else:
|
|
557
|
+
assert result.startswith("Error: write access denied") and sibling.read_text() == "OLD SETUP\n"
|
|
558
|
+
assert start_params["writable_module_tests_folder"] == ""
|
|
559
|
+
|
|
560
|
+
|
|
561
|
+
def test_failing_tests_of_a_required_module_get_their_siblings_copied_for_the_same_fix(project):
|
|
562
|
+
sibling = project.root / "base_tests" / "health"
|
|
563
|
+
sibling.mkdir(parents=True)
|
|
564
|
+
(sibling / "test_health.py").write_text("SETUP\n")
|
|
565
|
+
api = ScriptedAgentAPI(
|
|
566
|
+
fixer_turns=[[write("app.py", "v1\n")], [submit_fix()]], reviewer_turns=[[review("approve")]]
|
|
567
|
+
)
|
|
568
|
+
render_context = make_render_context(project, api, testing_module="base")
|
|
569
|
+
render_context.conformance_tests.tests_json = {
|
|
570
|
+
"base": {"1": {"folder_name": str(project.tests_folder)}, "2": {"folder_name": str(sibling)}}
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
run_fix(render_context)
|
|
574
|
+
|
|
575
|
+
assert (project.root / "base_tests" / "health_copy" / "test_health.py").read_text() == "SETUP\n"
|
|
576
|
+
assert os.path.isdir(str(project.tests_folder) + "_copy")
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
def test_reviewer_last_verdict_counts_and_the_next_review_starts_a_new_session(project):
|
|
580
|
+
api = ScriptedAgentAPI(
|
|
581
|
+
fixer_turns=[[write("app.py", "v1\n")], [submit_fix()], [write("app.py", "v2\n")], [submit_fix()]],
|
|
582
|
+
reviewer_turns=[
|
|
583
|
+
{"status": "tool_calls", "calls": [{"id": "v", **review("reject", "No.")}], "session_ended": True},
|
|
584
|
+
[review("approve")],
|
|
585
|
+
],
|
|
586
|
+
)
|
|
587
|
+
render_context = make_render_context(project, api)
|
|
588
|
+
|
|
589
|
+
run_fix(render_context)
|
|
590
|
+
|
|
591
|
+
reviewer_starts = api.of("start", conversation.REVIEWER_TASK)
|
|
592
|
+
assert len(reviewer_starts) == 2 and reviewer_starts[1][3]["previous_session_id"] == reviewer_starts[0][2]
|
|
593
|
+
|
|
594
|
+
|
|
595
|
+
def test_fixer_session_ending_without_a_submission_restarts_both_keeping_the_changes(project):
|
|
596
|
+
api = ScriptedAgentAPI(
|
|
597
|
+
fixer_turns=[[write("app.py", "v1\n")], {"status": "completed", "result": "handoff"}, [submit_fix()]],
|
|
598
|
+
reviewer_turns=[[review("approve")]],
|
|
599
|
+
)
|
|
600
|
+
render_context = make_render_context(project, api)
|
|
601
|
+
|
|
602
|
+
outcome, _ = run_fix(render_context)
|
|
603
|
+
|
|
604
|
+
assert outcome == FixConformanceTest.IMPLEMENTATION_CODE_UPDATED
|
|
605
|
+
fixer_starts = api.of("start", conversation.FIXER_TASK)
|
|
606
|
+
assert len(fixer_starts) == 2 and fixer_starts[1][3]["previous_session_id"] == fixer_starts[0][2]
|
|
607
|
+
assert (project.build / "app.py").read_text() == "v1\n"
|
|
608
|
+
|
|
609
|
+
|
|
610
|
+
def test_journal_records_the_approved_fix(project):
|
|
611
|
+
api = ScriptedAgentAPI(
|
|
612
|
+
fixer_turns=[[write("app.py", "v1\n")], [submit_fix()]], reviewer_turns=[[review("approve")]]
|
|
613
|
+
)
|
|
614
|
+
render_context = make_render_context(project, api)
|
|
615
|
+
|
|
616
|
+
run_fix(render_context)
|
|
617
|
+
|
|
618
|
+
[entry] = render_context.memory_manager.journal.build_digest("mod", "1")
|
|
619
|
+
assert entry["hypothesis"] == "off by one" and entry["target"] == "IMPLEMENTATION_CODE"
|
|
620
|
+
assert entry["files_changed"] == ["app.py"]
|
|
621
|
+
|
|
622
|
+
|
|
623
|
+
def test_a_fixer_is_shown_the_fixes_made_for_other_functionalities_tests(project):
|
|
624
|
+
api = ScriptedAgentAPI(
|
|
625
|
+
fixer_turns=[[write("app.py", "v1\n")], [submit_fix()], [write("app.py", "v2\n")], [submit_fix()]],
|
|
626
|
+
reviewer_turns=[[review("approve")], [review("approve")]],
|
|
627
|
+
)
|
|
628
|
+
render_context = make_render_context(project, api, testing_frid="2", frid="2")
|
|
629
|
+
run_fix(render_context)
|
|
630
|
+
assert "other_functionalities_fixes" not in api.of("start", conversation.FIXER_TASK)[0][3]
|
|
631
|
+
ctx = render_context.conformance_tests_running_context
|
|
632
|
+
ctx.current_testing_frid = "1"
|
|
633
|
+
ctx.set_conformance_tests_json("mod", {"1": {"folder_name": str(project.tests_folder)}})
|
|
634
|
+
|
|
635
|
+
run_fix(render_context)
|
|
636
|
+
|
|
637
|
+
[other_fix] = api.of("start", conversation.FIXER_TASK)[1][3]["other_functionalities_fixes"]
|
|
638
|
+
assert other_fix["functionality"] == qualified_frid("mod", "2") and other_fix["approach"] == "add one"
|
|
639
|
+
assert "+v1" in other_fix["code_diff"]["app.py"]
|
|
640
|
+
|
|
641
|
+
|
|
642
|
+
def test_a_conflict_of_a_functionality_without_acceptance_tests_is_with_the_one_an_earlier_fix_was_for(
|
|
643
|
+
project, monkeypatch
|
|
644
|
+
):
|
|
645
|
+
monkeypatch.setattr(
|
|
646
|
+
plain_spec,
|
|
647
|
+
"get_specifications_for_frid",
|
|
648
|
+
lambda tree, frid: ({plain_spec.FUNCTIONAL_REQUIREMENTS: ["- Add numbers."]}, None),
|
|
649
|
+
)
|
|
650
|
+
conflict = submit_fix("CONFLICTING_REQUIREMENTS", conflict_reason="1 and 2 contradict each other.")
|
|
651
|
+
api = ScriptedAgentAPI(fixer_turns=[[conflict]], reviewer_turns=[[review("approve")]])
|
|
652
|
+
render_context = make_render_context(project, api, testing_frid="2", frid="2")
|
|
653
|
+
ctx = render_context.conformance_tests_running_context
|
|
654
|
+
ctx.conflicting_requirement_count = 1
|
|
655
|
+
ctx.conflicting_module_name, ctx.conflicting_frid = "mod", "2"
|
|
656
|
+
ctx.implementation_code_fixes.append({"hypothesis": "h", "approach": "a", "code_diff": {}, "tested": ("mod", "1")})
|
|
657
|
+
|
|
658
|
+
with pytest.raises(ConflictingRequirements):
|
|
659
|
+
run_fix(render_context)
|
|
660
|
+
|
|
661
|
+
[report] = api.conflict_reports
|
|
662
|
+
assert report[1] == "1" and report[4] == "mod" and report[6] == "CONFLICTING_REQUIREMENTS"
|
|
663
|
+
|
|
664
|
+
|
|
665
|
+
def test_a_test_run_before_any_change_repeats_the_failing_run_instead_of_running_it(project):
|
|
666
|
+
api = ScriptedAgentAPI(
|
|
667
|
+
fixer_turns=[[{"name": "run_conformance_tests", "args": {}}], [write("app.py", "v1\n")], [submit_fix()]],
|
|
668
|
+
reviewer_turns=[[review("approve")]],
|
|
669
|
+
)
|
|
670
|
+
render_context = make_render_context(project, api)
|
|
671
|
+
|
|
672
|
+
run_fix(render_context)
|
|
673
|
+
|
|
674
|
+
[(_, _, _, results)] = [c for c in api.of("continue", conversation.FIXER_TASK)][:1]
|
|
675
|
+
assert results[0]["output"].startswith("Nothing changed") and "failing run whose output you were given" in (
|
|
676
|
+
results[0]["output"]
|
|
677
|
+
)
|