codeplain 0.4.3.dev61__py3-none-any.whl → 0.4.3.dev62__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,677 @@
1
+ """Tests for fixing conformance tests through a conversation between a fixer agent and a reviewer agent."""
2
+
3
+ import os
4
+ from types import SimpleNamespace
5
+
6
+ import pytest
7
+
8
+ import plain_spec
9
+ from memory_management import MemoryManager
10
+ from plain2code_exceptions import ConflictingRequirements
11
+ from plain2code_utils import qualified_frid
12
+ from render_machine import conformance_fix_conversation as conversation
13
+ from render_machine.actions.fix_conformance_test import FixConformanceTest
14
+ from render_machine.actions.run_conformance_tests import RunConformanceTests
15
+ from render_machine.implementation_code_helpers import ImplementationCodeHelpers
16
+ from render_machine.render_types import ConformanceTestsRunningContext, ScriptExecutionHistory
17
+
18
+ ISSUE = "FAILED test_app.py::test_add - assert 2 == 3"
19
+
20
+
21
+ def write(path, content):
22
+ return {"name": "write_file", "args": {"file_path": path, "content": content}}
23
+
24
+
25
+ def submit_fix(failing_reason="IMPLEMENTATION_CODE", **args):
26
+ return {
27
+ "name": "submit_fix",
28
+ "args": {"root_cause": "off by one", "changes_made": "add one", "failing_reason": failing_reason, **args},
29
+ }
30
+
31
+
32
+ def review(verdict, comments="", objected_files=None):
33
+ args = {"verdict": verdict, "comments": comments}
34
+ if objected_files is not None:
35
+ args["objected_files"] = objected_files
36
+ return {"name": "submit_review", "args": args}
37
+
38
+
39
+ class ScriptedAgentAPI:
40
+ """Plays the server side of the agent sessions: each task type answers with its scripted turns in order.
41
+ A turn is a list of tool calls (each turn of the server asking for them), or a raw response dict."""
42
+
43
+ def __init__(self, fixer_turns=(), reviewer_turns=()):
44
+ self.turns = {conversation.FIXER_TASK: list(fixer_turns), conversation.REVIEWER_TASK: list(reviewer_turns)}
45
+ self.session_tasks = {}
46
+ self.calls = []
47
+ self.conflict_reports = []
48
+
49
+ def agent_start(self, task_type, task_params, frid, module_name, run_state):
50
+ session_id = f"{task_type}-{len(self.session_tasks) + 1}"
51
+ self.session_tasks[session_id] = task_type
52
+ self.calls.append(("start", task_type, session_id, task_params))
53
+ return self._next(session_id)
54
+
55
+ def agent_continue(self, session_id, tool_results, frid, module_name, run_state):
56
+ self.calls.append(("continue", self.session_tasks[session_id], session_id, tool_results))
57
+ return self._next(session_id)
58
+
59
+ def report_conflicting_requirements(self, *args, **kwargs):
60
+ self.conflict_reports.append(args)
61
+ raise ConflictingRequirements("Conflicting requirements: they conflict")
62
+
63
+ def _next(self, session_id):
64
+ turn = self.turns[self.session_tasks[session_id]].pop(0)
65
+ if isinstance(turn, dict):
66
+ return {"session_id": session_id, **turn}
67
+ calls = [{"id": f"call-{len(self.calls)}-{index}", **call} for index, call in enumerate(turn)]
68
+ return {"session_id": session_id, "status": "tool_calls", "calls": calls}
69
+
70
+ def of(self, kind, task_type):
71
+ return [call for call in self.calls if call[0] == kind and call[1] == task_type]
72
+
73
+
74
+ class FakeConformanceTests:
75
+ def __init__(self, base):
76
+ self.base = base
77
+
78
+ def get_module_conformance_tests_folder(self, module_name):
79
+ return str(self.base / "conformance") if module_name == "mod" else str(self.base / f"conformance_{module_name}")
80
+
81
+ def get_source_conformance_test_folder_name(self, module_name, required_modules, testing_module, folder):
82
+ return folder, folder + "_copy"
83
+
84
+ def get_conformance_tests_json(self, module_name):
85
+ return getattr(self, "tests_json", {}).get(module_name, {})
86
+
87
+
88
+ @pytest.fixture
89
+ def project(tmp_path, monkeypatch):
90
+ build = tmp_path / "build"
91
+ build.mkdir()
92
+ (build / "app.py").write_text("def add(a, b):\n return a + b\n")
93
+ tests_folder = tmp_path / "conformance" / "add"
94
+ tests_folder.mkdir(parents=True)
95
+ (tests_folder / "test_app.py").write_text("def test_add():\n assert add(1, 2) == 3\n")
96
+ memory = tmp_path / "memory"
97
+ memory.mkdir()
98
+ monkeypatch.setattr(ImplementationCodeHelpers, "get_code_diff", staticmethod(lambda *args: {"app.py": "+x"}))
99
+ monkeypatch.setattr(
100
+ plain_spec,
101
+ "get_specifications_for_frid",
102
+ lambda tree, frid: (
103
+ {
104
+ plain_spec.DEFINITIONS: ["- :Number: is a number."],
105
+ plain_spec.FUNCTIONAL_REQUIREMENTS: ["- Add numbers."],
106
+ plain_spec.ACCEPTANCE_TESTS: [{"id": "a1", "text": "1 + 2 is 3"}],
107
+ },
108
+ None,
109
+ ),
110
+ )
111
+ return SimpleNamespace(root=tmp_path, build=build, tests_folder=tests_folder, memory=memory)
112
+
113
+
114
+ def make_render_context(project, api, testing_frid="1", frid="1", testing_module="mod", required_module_names=()):
115
+ ctx = ConformanceTestsRunningContext(
116
+ current_testing_module_name=testing_module,
117
+ current_testing_frid=testing_frid,
118
+ fix_attempts=0,
119
+ conformance_tests_json={testing_frid: {"folder_name": str(project.tests_folder)}},
120
+ conformance_tests_render_attempts=0,
121
+ current_testing_frid_specifications={
122
+ plain_spec.FUNCTIONAL_REQUIREMENTS: ["- Add numbers."],
123
+ plain_spec.CONFORMANCE_TESTS: [{"id": "s1", "text": "adding 1 and 2 gives 3"}],
124
+ },
125
+ should_prepare_testing_environment=False,
126
+ frid_being_implemented=frid,
127
+ )
128
+ return SimpleNamespace(
129
+ codeplain_api=api,
130
+ build_folder=str(project.build),
131
+ memory_manager=MemoryManager(api, str(project.memory)),
132
+ conformance_tests=FakeConformanceTests(project.root),
133
+ plain_module=SimpleNamespace(
134
+ build_folder=str(project.root / "plain_modules"),
135
+ all_required_modules=[SimpleNamespace(module_name=name) for name in required_module_names],
136
+ ),
137
+ conformance_tests_running_context=ctx,
138
+ plain_source_tree={},
139
+ module_name="mod",
140
+ required_modules=[],
141
+ frid_context=SimpleNamespace(frid=frid, linked_resources={}, changed_files={"app.py"}),
142
+ get_required_modules_functionalities=lambda: {},
143
+ testing_functionality_label=lambda: f"{ctx.current_testing_module_name}:{ctx.current_testing_frid}",
144
+ testing_functionality_position=lambda: None,
145
+ run_state=SimpleNamespace(render_id="render-id"),
146
+ conformance_tests_script=None,
147
+ prepare_environment_script=None,
148
+ script_execution_history=ScriptExecutionHistory(),
149
+ test_script_timeout=None,
150
+ stop_event=None,
151
+ )
152
+
153
+
154
+ def run_fix(render_context, issue=ISSUE):
155
+ return FixConformanceTest().execute(render_context, {"previous_conformance_tests_issue": issue})
156
+
157
+
158
+ def test_approved_implementation_fix_runs_the_unit_tests_next_and_is_handed_to_the_unit_tests_fixer(project):
159
+ api = ScriptedAgentAPI(
160
+ fixer_turns=[[write("app.py", "def add(a, b):\n return a + b + 1\n")], [submit_fix()]],
161
+ reviewer_turns=[[review("approve", "Addresses the root cause.")]],
162
+ )
163
+ render_context = make_render_context(project, api)
164
+
165
+ outcome, _ = run_fix(render_context)
166
+
167
+ ctx = render_context.conformance_tests_running_context
168
+ assert outcome == FixConformanceTest.IMPLEMENTATION_CODE_UPDATED
169
+ assert ctx.should_prepare_testing_environment and ctx.test_that_triggered_code_change == ("mod", "1")
170
+ [recorded] = ctx.implementation_code_fixes
171
+ assert recorded["hypothesis"] == "off by one" and recorded["approach"] == "add one"
172
+ assert list(recorded["code_diff"]) == ["app.py"] and "+ return a + b + 1" in recorded["code_diff"]["app.py"]
173
+
174
+ [(_, _, _, fixer_params)] = api.of("start", conversation.FIXER_TASK)
175
+ assert fixer_params["conformance_tests_issue"] == ISSUE
176
+ assert fixer_params["conformance_tests_folder"] == str(project.tests_folder)
177
+ assert fixer_params["is_regression"] is False and "code_diff" not in fixer_params
178
+ assert "#### Scenario `s1`" in fixer_params["conformance_scenarios"]
179
+ assert "#### Scenario `a1`" in fixer_params["acceptance_tests"]
180
+ assert str(project.tests_folder / "test_app.py") in fixer_params["conformance_tests_files"]
181
+ assert "app.py" in fixer_params["relevant_files"]
182
+
183
+ [(_, _, _, reviewer_params)] = api.of("start", conversation.REVIEWER_TASK)
184
+ fix_submission = reviewer_params["fix_submission"]
185
+ assert reviewer_params["conformance_tests_issue"] == ISSUE
186
+ assert (
187
+ fix_submission["changes_made"] == "add one" and "+ return a + b + 1" in fix_submission["code_diff"]["app.py"]
188
+ )
189
+ # The reviewer sees the conformance tests as they were before the fix.
190
+ assert (
191
+ "assert add(1, 2) == 3" in reviewer_params["conformance_tests_files"][str(project.tests_folder / "test_app.py")]
192
+ )
193
+
194
+ agents = ctx.conformance_fix_agents
195
+ assert agents.fixer.pending_call_id and agents.reviewer.pending_call_id and agents.approved_fix_tested
196
+
197
+
198
+ def test_conformance_tests_fix_runs_the_conformance_tests_next(project):
199
+ test_file = str(project.tests_folder / "test_app.py")
200
+ api = ScriptedAgentAPI(
201
+ fixer_turns=[
202
+ [write(test_file, "def test_add():\n assert add(1, 2) == 4\n")],
203
+ [submit_fix("CONFORMANCE_TESTS")],
204
+ ],
205
+ reviewer_turns=[[review("approve")]],
206
+ )
207
+ render_context = make_render_context(project, api)
208
+
209
+ outcome, _ = run_fix(render_context)
210
+
211
+ assert outcome == FixConformanceTest.IMPLEMENTATION_CODE_NOT_UPDATED
212
+ assert render_context.conformance_tests_running_context.implementation_code_fixes == []
213
+ assert "== 4" in open(test_file).read()
214
+ [(_, _, _, reviewer_params)] = api.of("start", conversation.REVIEWER_TASK)
215
+ assert list(reviewer_params["fix_submission"]["code_diff"]) == [test_file]
216
+
217
+
218
+ def _sessions_ending_without_an_approved_fix(first_turn):
219
+ ended = {"status": "completed", "result": "handoff"}
220
+ return [first_turn, ended] + [ended] * (conversation.MAX_FIXER_SESSIONS_PER_FIX - 1)
221
+
222
+
223
+ def test_unapproved_implementation_changes_are_kept_and_tested_after_the_unit_tests(project):
224
+ # The fixer never got a fix approved: its changes stay, and like an approved implementation change they are
225
+ # tested after the unit tests and a fresh preparation, never on the environment prepared for the older code.
226
+ api = ScriptedAgentAPI(
227
+ fixer_turns=_sessions_ending_without_an_approved_fix([write("app.py", "def add(a, b):\n return 3\n")])
228
+ )
229
+ render_context = make_render_context(project, api)
230
+
231
+ outcome, _ = run_fix(render_context)
232
+
233
+ ctx = render_context.conformance_tests_running_context
234
+ assert len(api.of("start", conversation.FIXER_TASK)) == conversation.MAX_FIXER_SESSIONS_PER_FIX
235
+ assert not api.of("start", conversation.REVIEWER_TASK)
236
+ assert outcome == FixConformanceTest.IMPLEMENTATION_CODE_UPDATED
237
+ assert "return 3" in (project.build / "app.py").read_text()
238
+ assert ctx.should_prepare_testing_environment and ctx.test_that_triggered_code_change == ("mod", "1")
239
+ # Handed to the unit tests fixer, so it keeps the change instead of reverting it.
240
+ [recorded] = ctx.implementation_code_fixes
241
+ assert "without getting them approved" in recorded["approach"]
242
+ assert "+ return 3" in recorded["code_diff"]["app.py"]
243
+
244
+
245
+ def test_unapproved_changes_to_the_conformance_tests_only_run_the_conformance_tests_again(project):
246
+ test_file = str(project.tests_folder / "test_app.py")
247
+ api = ScriptedAgentAPI(
248
+ fixer_turns=_sessions_ending_without_an_approved_fix([write(test_file, "def test_add():\n pass\n")])
249
+ )
250
+ render_context = make_render_context(project, api)
251
+
252
+ outcome, _ = run_fix(render_context)
253
+
254
+ ctx = render_context.conformance_tests_running_context
255
+ assert outcome == FixConformanceTest.IMPLEMENTATION_CODE_NOT_UPDATED
256
+ assert "pass" in open(test_file).read()
257
+ assert not ctx.should_prepare_testing_environment and ctx.implementation_code_fixes == []
258
+
259
+
260
+ def test_rejected_fix_goes_back_to_the_fixer_with_the_comments_and_the_revision_is_reviewed(project):
261
+ api = ScriptedAgentAPI(
262
+ fixer_turns=[
263
+ [write("app.py", "def add(a, b):\n return 3\n")],
264
+ [submit_fix()],
265
+ [write("app.py", "def add(a, b):\n return a + b\n# fixed\n")],
266
+ [submit_fix(review_response="Implemented addition instead of a constant.")],
267
+ ],
268
+ reviewer_turns=[[review("reject", "Returning a constant only fits the test.")], [review("approve")]],
269
+ )
270
+ render_context = make_render_context(project, api)
271
+
272
+ outcome, _ = run_fix(render_context)
273
+
274
+ assert outcome == FixConformanceTest.IMPLEMENTATION_CODE_UPDATED
275
+ fixer_answers = [results[-1] for _, _, _, results in api.of("continue", conversation.FIXER_TASK)]
276
+ [rejection] = [answer for answer in fixer_answers if "review" in answer]
277
+ assert rejection["output"] == conversation.REJECTED_MESSAGE
278
+ assert rejection["review"] == {"comments": "Returning a constant only fits the test."}
279
+
280
+ [(_, _, _, reviewer_results)] = api.of("continue", conversation.REVIEWER_TASK)
281
+ revision = reviewer_results[-1]
282
+ assert revision["output"] == conversation.REVIEW_REVISION_MESSAGE and "test_output" not in revision
283
+ assert revision["fix_submission"]["review_response"] == "Implemented addition instead of a constant."
284
+ # The revision is shown as the whole fix since the tests ran: the constant is gone again.
285
+ assert "+# fixed" in revision["fix_submission"]["code_diff"]["app.py"]
286
+ assert "return 3" not in revision["fix_submission"]["code_diff"]["app.py"]
287
+
288
+
289
+ def test_fix_rejected_too_often_loses_only_the_objected_changes_and_the_sessions_go_on(project):
290
+ rejections = conversation.MAX_REJECTIONS
291
+ fixer_turns = [[write("test_app_extra.py", "x = 1\n"), write("app.py", "def add(a, b):\n return b + a\n")]]
292
+ fixer_turns += [[submit_fix()] for _ in range(rejections + 1)]
293
+ reviewer_turns = [[review("reject", f"No {number}.", ["test_app_extra.py"])] for number in range(rejections)]
294
+ api = ScriptedAgentAPI(fixer_turns=fixer_turns, reviewer_turns=reviewer_turns + [[review("approve")]])
295
+ render_context = make_render_context(project, api)
296
+
297
+ outcome, _ = run_fix(render_context)
298
+
299
+ assert outcome == FixConformanceTest.IMPLEMENTATION_CODE_UPDATED
300
+ # Only the objected file was reverted; the other change stays, and nobody started over.
301
+ assert not (project.build / "test_app_extra.py").exists()
302
+ assert "return b + a" in (project.build / "app.py").read_text()
303
+ assert len(api.of("start", conversation.FIXER_TASK)) == 1 and len(api.of("start", conversation.REVIEWER_TASK)) == 1
304
+ answers = [results[-1] for (_, _, _, results) in api.of("continue", conversation.FIXER_TASK)]
305
+ rejected = [a for a in answers if a.get("review")]
306
+ assert "reverted_files" not in rejected[rejections - 2]["review"]
307
+ assert rejected[rejections - 1]["review"]["reverted_files"] == ["test_app_extra.py"]
308
+ # The reviewer is told how many fixes it rejected in a row.
309
+ reviews = [results[-1]["fix_submission"] for (_, _, _, results) in api.of("continue", conversation.REVIEWER_TASK)]
310
+ assert [r["consecutive_rejections"] for r in reviews] == list(range(1, rejections + 1))
311
+ assert list(reviews[-1]["code_diff"]) == ["app.py"]
312
+
313
+
314
+ def test_a_file_read_again_after_it_was_reverted_is_read_from_disk(project):
315
+ rejections = conversation.MAX_REJECTIONS
316
+ read_extra = {"name": "read_file", "args": {"file_path": "test_app_extra.py"}}
317
+ fixer_turns = [[write("test_app_extra.py", "x = 1\n"), write("app.py", "v1\n")], [read_extra]]
318
+ fixer_turns += [[submit_fix()] for _ in range(rejections)] + [[read_extra], [submit_fix()]]
319
+ reviewer_turns = [[review("reject", "No.", ["test_app_extra.py"])] for _ in range(rejections)]
320
+ api = ScriptedAgentAPI(fixer_turns=fixer_turns, reviewer_turns=reviewer_turns + [[review("approve")]])
321
+
322
+ run_fix(make_render_context(project, api))
323
+
324
+ first_read, second_read = [
325
+ results
326
+ for (_, _, _, results) in api.of("continue", conversation.FIXER_TASK)
327
+ if len(results) == 1 and "review" not in results[0]
328
+ ]
329
+ assert "x = 1" in str(first_read)
330
+ # Not the cached content from before the revert: the file is gone.
331
+ assert "x = 1" not in str(second_read) and "not found" in str(second_read).lower()
332
+
333
+
334
+ def test_fix_that_changes_nothing_is_sent_back_without_a_review(project):
335
+ api = ScriptedAgentAPI(
336
+ fixer_turns=[[submit_fix()], [write("app.py", "def add(a, b):\n return a + b # ok\n")], [submit_fix()]],
337
+ reviewer_turns=[[review("approve")]],
338
+ )
339
+ render_context = make_render_context(project, api)
340
+
341
+ run_fix(render_context)
342
+
343
+ first_answer = api.of("continue", conversation.FIXER_TASK)[0][3][-1]
344
+ assert first_answer["output"] == conversation.EMPTY_FIX_MESSAGE
345
+ assert len(api.of("start", conversation.REVIEWER_TASK)) == 1
346
+
347
+
348
+ def test_tests_still_failing_continue_both_sessions_with_the_new_output(project):
349
+ api = ScriptedAgentAPI(
350
+ fixer_turns=[
351
+ [write("app.py", "v1\n")],
352
+ [submit_fix()],
353
+ [write("app.py", "v2\n")],
354
+ [submit_fix()],
355
+ ],
356
+ reviewer_turns=[[review("approve")], [review("approve")]],
357
+ )
358
+ render_context = make_render_context(project, api)
359
+ run_fix(render_context)
360
+
361
+ outcome, _ = run_fix(render_context, issue="STILL FAILING")
362
+
363
+ assert outcome == FixConformanceTest.IMPLEMENTATION_CODE_UPDATED
364
+ assert len(api.of("start", conversation.FIXER_TASK)) == 1 and len(api.of("start", conversation.REVIEWER_TASK)) == 1
365
+ fixer_answer = api.of("continue", conversation.FIXER_TASK)[1][3][-1]
366
+ assert fixer_answer["output"].startswith(conversation.TESTS_STILL_FAIL_MESSAGE)
367
+ assert fixer_answer["test_output"] == "STILL FAILING"
368
+ [(_, _, _, reviewer_results)] = api.of("continue", conversation.REVIEWER_TASK)
369
+ assert reviewer_results[-1]["output"].startswith(conversation.REVIEW_AFTER_TESTS_MESSAGE)
370
+ assert reviewer_results[-1]["test_output"] == "STILL FAILING"
371
+ # Only the second fix is shown: the first one was approved and tested.
372
+ assert "-v1" in reviewer_results[-1]["fix_submission"]["code_diff"]["app.py"]
373
+
374
+
375
+ def test_fixer_is_told_when_runs_after_approved_fixes_keep_failing(project, monkeypatch):
376
+ monkeypatch.setattr(conversation, "NOT_CONVERGING_AFTER_FAILED_RUNS", 2)
377
+ fixer_turns, reviewer_turns = [], []
378
+ for number in range(3):
379
+ fixer_turns += [[write("app.py", f"v{number}\n")], [submit_fix()]]
380
+ reviewer_turns.append([review("approve")])
381
+ api = ScriptedAgentAPI(fixer_turns=fixer_turns, reviewer_turns=reviewer_turns)
382
+ render_context = make_render_context(project, api)
383
+
384
+ for _ in range(3):
385
+ run_fix(render_context, issue="STILL FAILING")
386
+
387
+ answers = [results[-1]["output"] for (_, _, _, results) in api.of("continue", conversation.FIXER_TASK)]
388
+ still_failing = [a for a in answers if a.startswith(conversation.TESTS_STILL_FAIL_MESSAGE)]
389
+ assert len(still_failing) == 2
390
+ assert "not converging" not in still_failing[0]
391
+ assert "failed 2 times in a row" in still_failing[1] and "not converging" in still_failing[1]
392
+
393
+
394
+ def test_passing_tests_end_both_agents(project, monkeypatch):
395
+ api = ScriptedAgentAPI(
396
+ fixer_turns=[[write("app.py", "v1\n")], [submit_fix()]], reviewer_turns=[[review("approve")]]
397
+ )
398
+ render_context = make_render_context(project, api)
399
+ render_context.conformance_tests_script = "run.sh"
400
+ run_fix(render_context)
401
+ monkeypatch.setattr(
402
+ "render_machine.render_utils.execute_script", lambda *args, **kwargs: (0, "", None), raising=True
403
+ )
404
+
405
+ outcome, _ = RunConformanceTests().execute(render_context, None)
406
+
407
+ assert outcome == RunConformanceTests.SUCCESSFUL_OUTCOME
408
+ assert render_context.conformance_tests_running_context.conformance_fix_agents is None
409
+
410
+
411
+ def test_failing_tests_of_another_functionality_get_new_agents(project):
412
+ api = ScriptedAgentAPI(
413
+ fixer_turns=[[write("app.py", "v1\n")], [submit_fix()], [write("app.py", "v2\n")], [submit_fix()]],
414
+ reviewer_turns=[[review("approve")], [review("approve")]],
415
+ )
416
+ render_context = make_render_context(project, api, testing_frid="1", frid="2")
417
+ run_fix(render_context)
418
+ ctx = render_context.conformance_tests_running_context
419
+ ctx.current_testing_frid = "2"
420
+ ctx.set_conformance_tests_json("mod", {"2": {"folder_name": str(project.tests_folder)}})
421
+
422
+ run_fix(render_context)
423
+
424
+ assert len(api.of("start", conversation.FIXER_TASK)) == 2
425
+ assert ctx.conformance_fix_agents.tested == ("mod", "2")
426
+ regression_params = api.of("start", conversation.FIXER_TASK)[0][3]
427
+ assert regression_params["is_regression"] is True and regression_params["code_diff"] == {"app.py": "+x"}
428
+
429
+
430
+ def test_agreed_conflict_is_counted_first_and_reported_the_second_time(project):
431
+ conflict = submit_fix("CONFLICTING_REQUIREMENTS", conflict_reason="1 and 2 contradict each other.")
432
+ api = ScriptedAgentAPI(
433
+ fixer_turns=[[write("app.py", "v1\n")], [conflict], [conflict]],
434
+ reviewer_turns=[[review("approve")], [review("approve")]],
435
+ )
436
+ render_context = make_render_context(project, api, testing_frid="1", frid="2")
437
+ ctx = render_context.conformance_tests_running_context
438
+
439
+ outcome, _ = run_fix(render_context)
440
+ assert outcome == FixConformanceTest.IMPLEMENTATION_CODE_UPDATED
441
+ assert ctx.conflicting_requirement_count == 1 and api.conflict_reports == []
442
+ # A second declaration in the same session is still reviewed and, once agreed, ends the render.
443
+ assert "conflict_already_reported" not in api.of("start", conversation.FIXER_TASK)[0][3]
444
+
445
+ with pytest.raises(ConflictingRequirements):
446
+ run_fix(render_context)
447
+ [report] = api.conflict_reports
448
+ assert report[6] == "CONFLICTING_REQUIREMENTS" and report[7] == "1 and 2 contradict each other."
449
+
450
+
451
+ def test_conflict_of_the_current_functionality_is_with_its_acceptance_tests(project):
452
+ conflict = submit_fix("CONFLICTING_REQUIREMENTS", conflict_reason="a1 contradicts the requirement.")
453
+ api = ScriptedAgentAPI(fixer_turns=[[conflict]], reviewer_turns=[[review("approve")]])
454
+ render_context = make_render_context(project, api)
455
+ ctx = render_context.conformance_tests_running_context
456
+ ctx.conflicting_requirement_count = 1
457
+ ctx.conflicting_module_name, ctx.conflicting_frid = "mod", "1"
458
+
459
+ with pytest.raises(ConflictingRequirements):
460
+ run_fix(render_context)
461
+
462
+ assert api.conflict_reports[0][6] == "CONFLICTING_ACCEPTANCE_TESTS"
463
+ assert api.of("start", conversation.FIXER_TASK)[0][3]["conflict_already_reported"] is True
464
+
465
+
466
+ def test_reviewer_cannot_change_files_and_fixer_cannot_write_outside_its_folders(project, tmp_path):
467
+ outside = str(tmp_path / "outside.py")
468
+ api = ScriptedAgentAPI(
469
+ fixer_turns=[[write(outside, "x\n"), write("app.py", "v1\n")], [submit_fix()]],
470
+ reviewer_turns=[[write("app.py", "reviewer\n")], [review("approve")]],
471
+ )
472
+ render_context = make_render_context(project, api)
473
+
474
+ run_fix(render_context)
475
+
476
+ fixer_results = api.of("continue", conversation.FIXER_TASK)[0][3]
477
+ assert fixer_results[0]["output"].startswith("Error: write access denied")
478
+ [(_, _, _, reviewer_results)] = api.of("continue", conversation.REVIEWER_TASK)
479
+ assert reviewer_results[0]["output"].startswith("Error: unknown tool")
480
+ assert (project.build / "app.py").read_text() == "v1\n" and not os.path.exists(outside)
481
+
482
+
483
+ def read(path):
484
+ return {"name": "read_file", "args": {"file_path": path}}
485
+
486
+
487
+ def test_agents_read_the_tests_of_other_functionalities_and_required_modules_but_only_write_their_own(project):
488
+ sibling = project.root / "conformance" / "login" / "test_login.py"
489
+ sibling.parent.mkdir()
490
+ sibling.write_text("WORKING HARNESS\n")
491
+ required = project.root / "conformance_base" / "health" / "test_health.py"
492
+ required.parent.mkdir(parents=True)
493
+ required.write_text("BASE HARNESS\n")
494
+ api = ScriptedAgentAPI(
495
+ fixer_turns=[
496
+ [read(str(sibling)), read(str(required)), write(str(sibling), "x\n"), write("app.py", "v1\n")],
497
+ [submit_fix()],
498
+ ],
499
+ reviewer_turns=[[read(str(sibling)), read(str(required))], [review("approve")]],
500
+ )
501
+ render_context = make_render_context(project, api, required_module_names=["base"])
502
+
503
+ run_fix(render_context)
504
+
505
+ fixer_results = [r["output"] for r in api.of("continue", conversation.FIXER_TASK)[0][3]]
506
+ assert "WORKING HARNESS" in fixer_results[0] and "BASE HARNESS" in fixer_results[1]
507
+ assert fixer_results[2].startswith("Error: write access denied") and sibling.read_text() == "WORKING HARNESS\n"
508
+ reviewer_results = [r["output"] for r in api.of("continue", conversation.REVIEWER_TASK)[0][3]]
509
+ assert "WORKING HARNESS" in reviewer_results[0] and "BASE HARNESS" in reviewer_results[1]
510
+ [(_, _, _, start_params)] = api.of("start", conversation.FIXER_TASK)
511
+ assert start_params["readable_conformance_tests_folders"] == [
512
+ str(project.root / "conformance"),
513
+ str(project.root / "conformance_base"),
514
+ ]
515
+
516
+
517
+ def test_agents_read_every_rendered_module_and_are_told_the_layout(project):
518
+ other = project.root / "plain_modules" / "base" / "code" / "lib.py"
519
+ other.parent.mkdir(parents=True)
520
+ other.write_text("BASE CODE\n")
521
+ api = ScriptedAgentAPI(
522
+ fixer_turns=[[read(str(other)), write(str(other), "x\n"), write("app.py", "v1\n")], [submit_fix()]],
523
+ reviewer_turns=[[read(str(other))], [review("approve")]],
524
+ )
525
+ render_context = make_render_context(project, api, required_module_names=["base"])
526
+
527
+ run_fix(render_context)
528
+
529
+ fixer_results = [r["output"] for r in api.of("continue", conversation.FIXER_TASK)[0][3]]
530
+ assert "BASE CODE" in fixer_results[0] and fixer_results[1].startswith("Error: write access denied")
531
+ assert other.read_text() == "BASE CODE\n"
532
+ assert "BASE CODE" in api.of("continue", conversation.REVIEWER_TASK)[0][3][0]["output"]
533
+ [(_, _, _, start_params)] = api.of("start", conversation.FIXER_TASK)
534
+ assert start_params["plain_modules_folder"] == str(project.root / "plain_modules")
535
+ assert start_params["required_module_names"] == ["base"]
536
+
537
+
538
+ @pytest.mark.parametrize("regression", [True, False])
539
+ def test_a_regression_fixer_may_fix_the_other_earlier_tests_of_the_module(project, regression):
540
+ sibling = project.root / "conformance" / "login" / "test_login.py"
541
+ sibling.parent.mkdir()
542
+ sibling.write_text("OLD SETUP\n")
543
+ api = ScriptedAgentAPI(
544
+ fixer_turns=[[write(str(sibling), "NEW SETUP\n"), write("app.py", "v1\n")], [submit_fix()]],
545
+ reviewer_turns=[[review("approve")]],
546
+ )
547
+ render_context = make_render_context(project, api, testing_frid="1", frid="2" if regression else "1")
548
+
549
+ run_fix(render_context)
550
+
551
+ result = api.of("continue", conversation.FIXER_TASK)[0][3][0]["output"]
552
+ [(_, _, _, start_params)] = api.of("start", conversation.FIXER_TASK)
553
+ if regression:
554
+ assert sibling.read_text() == "NEW SETUP\n"
555
+ assert start_params["writable_module_tests_folder"] == str(project.root / "conformance")
556
+ else:
557
+ assert result.startswith("Error: write access denied") and sibling.read_text() == "OLD SETUP\n"
558
+ assert start_params["writable_module_tests_folder"] == ""
559
+
560
+
561
+ def test_failing_tests_of_a_required_module_get_their_siblings_copied_for_the_same_fix(project):
562
+ sibling = project.root / "base_tests" / "health"
563
+ sibling.mkdir(parents=True)
564
+ (sibling / "test_health.py").write_text("SETUP\n")
565
+ api = ScriptedAgentAPI(
566
+ fixer_turns=[[write("app.py", "v1\n")], [submit_fix()]], reviewer_turns=[[review("approve")]]
567
+ )
568
+ render_context = make_render_context(project, api, testing_module="base")
569
+ render_context.conformance_tests.tests_json = {
570
+ "base": {"1": {"folder_name": str(project.tests_folder)}, "2": {"folder_name": str(sibling)}}
571
+ }
572
+
573
+ run_fix(render_context)
574
+
575
+ assert (project.root / "base_tests" / "health_copy" / "test_health.py").read_text() == "SETUP\n"
576
+ assert os.path.isdir(str(project.tests_folder) + "_copy")
577
+
578
+
579
+ def test_reviewer_last_verdict_counts_and_the_next_review_starts_a_new_session(project):
580
+ api = ScriptedAgentAPI(
581
+ fixer_turns=[[write("app.py", "v1\n")], [submit_fix()], [write("app.py", "v2\n")], [submit_fix()]],
582
+ reviewer_turns=[
583
+ {"status": "tool_calls", "calls": [{"id": "v", **review("reject", "No.")}], "session_ended": True},
584
+ [review("approve")],
585
+ ],
586
+ )
587
+ render_context = make_render_context(project, api)
588
+
589
+ run_fix(render_context)
590
+
591
+ reviewer_starts = api.of("start", conversation.REVIEWER_TASK)
592
+ assert len(reviewer_starts) == 2 and reviewer_starts[1][3]["previous_session_id"] == reviewer_starts[0][2]
593
+
594
+
595
+ def test_fixer_session_ending_without_a_submission_restarts_both_keeping_the_changes(project):
596
+ api = ScriptedAgentAPI(
597
+ fixer_turns=[[write("app.py", "v1\n")], {"status": "completed", "result": "handoff"}, [submit_fix()]],
598
+ reviewer_turns=[[review("approve")]],
599
+ )
600
+ render_context = make_render_context(project, api)
601
+
602
+ outcome, _ = run_fix(render_context)
603
+
604
+ assert outcome == FixConformanceTest.IMPLEMENTATION_CODE_UPDATED
605
+ fixer_starts = api.of("start", conversation.FIXER_TASK)
606
+ assert len(fixer_starts) == 2 and fixer_starts[1][3]["previous_session_id"] == fixer_starts[0][2]
607
+ assert (project.build / "app.py").read_text() == "v1\n"
608
+
609
+
610
+ def test_journal_records_the_approved_fix(project):
611
+ api = ScriptedAgentAPI(
612
+ fixer_turns=[[write("app.py", "v1\n")], [submit_fix()]], reviewer_turns=[[review("approve")]]
613
+ )
614
+ render_context = make_render_context(project, api)
615
+
616
+ run_fix(render_context)
617
+
618
+ [entry] = render_context.memory_manager.journal.build_digest("mod", "1")
619
+ assert entry["hypothesis"] == "off by one" and entry["target"] == "IMPLEMENTATION_CODE"
620
+ assert entry["files_changed"] == ["app.py"]
621
+
622
+
623
+ def test_a_fixer_is_shown_the_fixes_made_for_other_functionalities_tests(project):
624
+ api = ScriptedAgentAPI(
625
+ fixer_turns=[[write("app.py", "v1\n")], [submit_fix()], [write("app.py", "v2\n")], [submit_fix()]],
626
+ reviewer_turns=[[review("approve")], [review("approve")]],
627
+ )
628
+ render_context = make_render_context(project, api, testing_frid="2", frid="2")
629
+ run_fix(render_context)
630
+ assert "other_functionalities_fixes" not in api.of("start", conversation.FIXER_TASK)[0][3]
631
+ ctx = render_context.conformance_tests_running_context
632
+ ctx.current_testing_frid = "1"
633
+ ctx.set_conformance_tests_json("mod", {"1": {"folder_name": str(project.tests_folder)}})
634
+
635
+ run_fix(render_context)
636
+
637
+ [other_fix] = api.of("start", conversation.FIXER_TASK)[1][3]["other_functionalities_fixes"]
638
+ assert other_fix["functionality"] == qualified_frid("mod", "2") and other_fix["approach"] == "add one"
639
+ assert "+v1" in other_fix["code_diff"]["app.py"]
640
+
641
+
642
+ def test_a_conflict_of_a_functionality_without_acceptance_tests_is_with_the_one_an_earlier_fix_was_for(
643
+ project, monkeypatch
644
+ ):
645
+ monkeypatch.setattr(
646
+ plain_spec,
647
+ "get_specifications_for_frid",
648
+ lambda tree, frid: ({plain_spec.FUNCTIONAL_REQUIREMENTS: ["- Add numbers."]}, None),
649
+ )
650
+ conflict = submit_fix("CONFLICTING_REQUIREMENTS", conflict_reason="1 and 2 contradict each other.")
651
+ api = ScriptedAgentAPI(fixer_turns=[[conflict]], reviewer_turns=[[review("approve")]])
652
+ render_context = make_render_context(project, api, testing_frid="2", frid="2")
653
+ ctx = render_context.conformance_tests_running_context
654
+ ctx.conflicting_requirement_count = 1
655
+ ctx.conflicting_module_name, ctx.conflicting_frid = "mod", "2"
656
+ ctx.implementation_code_fixes.append({"hypothesis": "h", "approach": "a", "code_diff": {}, "tested": ("mod", "1")})
657
+
658
+ with pytest.raises(ConflictingRequirements):
659
+ run_fix(render_context)
660
+
661
+ [report] = api.conflict_reports
662
+ assert report[1] == "1" and report[4] == "mod" and report[6] == "CONFLICTING_REQUIREMENTS"
663
+
664
+
665
+ def test_a_test_run_before_any_change_repeats_the_failing_run_instead_of_running_it(project):
666
+ api = ScriptedAgentAPI(
667
+ fixer_turns=[[{"name": "run_conformance_tests", "args": {}}], [write("app.py", "v1\n")], [submit_fix()]],
668
+ reviewer_turns=[[review("approve")]],
669
+ )
670
+ render_context = make_render_context(project, api)
671
+
672
+ run_fix(render_context)
673
+
674
+ [(_, _, _, results)] = [c for c in api.of("continue", conversation.FIXER_TASK)][:1]
675
+ assert results[0]["output"].startswith("Nothing changed") and "failing run whose output you were given" in (
676
+ results[0]["output"]
677
+ )