codeplain 0.3.11.dev19__py3-none-any.whl → 0.3.11.dev20__py3-none-any.whl

tests/test_fix_unit_tests_action.py

@@ -0,0 +1,239 @@
+"""Tests for the agentic FixUnitTests action driving a scripted fake API."""
+
+from types import SimpleNamespace
+
+import pytest
+import requests
+
+import plain_spec
+from render_machine.actions.fix_unit_tests import MAX_AGENT_TURNS_PER_ATTEMPT, MAX_RELEVANT_FILES_CHARS, FixUnitTests
+from render_machine.actions.run_unit_tests import RunUnitTests
+from render_machine.render_context import RenderContext
+from render_machine.render_types import ScriptExecutionHistory, UnitTestsRunningContext
+
+
+class FakeRenderContext(SimpleNamespace):
+    """A render context with just the attributes FixUnitTests uses, and the real session lookup."""
+
+    unit_tests_agent_session = RenderContext.unit_tests_agent_session
+
+
+class FakeAPI:
+    def __init__(self, responses):
+        self.responses = list(responses)
+        self.calls = []
+
+    def agent_start(self, task_type, task_params, frid, module_name, run_state):
+        self.calls.append(("start", task_type, task_params, frid, module_name))
+        return self.responses.pop(0)
+
+    def agent_continue(self, session_id, tool_results, frid, module_name, run_state):
+        self.calls.append(("continue", session_id, tool_results, frid, module_name))
+        response = self.responses.pop(0)
+        if isinstance(response, Exception):
+            raise response
+        return response
+
+
+def _tool_calls(*calls):
+    return {"session_id": "s1", "status": "tool_calls", "calls": list(calls)}
+
+
+@pytest.fixture
+def render_context(tmp_path, monkeypatch):
+    build = tmp_path / "build"
+    build.mkdir()
+    (build / "a.py").write_text("x = 1\n")
+    monkeypatch.chdir(tmp_path)
+    plain_source_tree = {"spec": True}
+    specifications = {
+        plain_spec.DEFINITIONS: ["- :Foo: is a thing."],
+        plain_spec.NON_FUNCTIONAL_REQUIREMENTS: ["- Python 3.11."],
+        plain_spec.FUNCTIONAL_REQUIREMENTS: ["- Old feature.", "- New feature."],
+    }
+    monkeypatch.setattr(plain_spec, "get_specifications_for_frid", lambda tree, frid: (specifications, None))
+    return FakeRenderContext(
+        codeplain_api=None,
+        build_folder=str(build),
+        module_name="m",
+        run_state=object(),
+        plain_source_tree=plain_source_tree,
+        unittests_script=None,
+        test_script_timeout=None,
+        stop_event=None,
+        frid_context=SimpleNamespace(frid="2", linked_resources={"schema.json": "{}"}, changed_files={"a.py"}),
+        unit_tests_running_context=UnitTestsRunningContext(fix_attempts=1),
+        conformance_tests_running_context=None,
+        script_execution_history=ScriptExecutionHistory(),
+        get_required_modules_functionalities=lambda: {"base": ["- Base feature."]},
+    )
+
+
+def test_first_attempt_starts_session_runs_tools_and_stops_at_submit_fix(render_context):
+    api = FakeAPI(
+        [
+            _tool_calls({"id": "c1", "name": "read_file", "args": {"file_path": "a.py"}}),
+            _tool_calls(
+                {"id": "c2", "name": "edit_file", "args": {"file_path": "a.py", "search": "x = 1", "replace": "x = 2"}},
+                {"id": "c3", "name": "submit_fix", "args": {"root_cause": "off by one", "changes_made": "x = 2"}},
+            ),
+        ]
+    )
+    render_context.codeplain_api = api
+
+    outcome, payload = FixUnitTests().execute(render_context, {"previous_unittests_issue": "FAILED test_a"})
+
+    assert (outcome, payload) == (FixUnitTests.SUCCESSFUL_OUTCOME, None)
+    kind, task_type, task_params, frid, module_name = api.calls[0]
+    assert (kind, task_type, frid, module_name) == ("start", "fix_unit_tests", "2", "m")
+    assert task_params["unittests_issue"] == "FAILED test_a"
+    assert task_params["definitions"] == "- :Foo: is a thing."
+    assert task_params["linked_resources"] == {"schema.json": "{}"}
+    assert (
+        "### Module: base (Already Implemented, for context)\n- Base feature." in task_params["functional_requirements"]
+    )
+    assert "### Module: m (Already Implemented, for context)\n- Old feature." in task_params["functional_requirements"]
+    assert task_params["functional_requirements"].endswith(
+        "### Module: m (Currently Being Implemented)\n- New feature."
+    )
+
+    # the read_file result went back to the server; the edit was applied locally
+    assert api.calls[1][0] == "continue" and api.calls[1][2][0]["call_id"] == "c1"
+    assert "1: x = 1" in api.calls[1][2][0]["output"]
+    assert open(render_context.build_folder + "/a.py").read() == "x = 2\n"
+
+    context = render_context.unit_tests_running_context
+    session = context.agent_session
+    assert session.session_id == "s1"
+    assert session.pending_submit_call_id == "c3"
+    assert [r["call_id"] for r in session.pending_tool_results] == ["c2"]
+    assert context.changed_files == {"a.py"}
+    assert "conformance_tests_fixes" not in task_params
+
+
+def test_second_attempt_continues_session_answering_submit_fix(render_context):
+    context = render_context.unit_tests_running_context
+    context.agent_used_in_this_loop = True
+    session = context.agent_session
+    session.session_id = "s1"
+    session.pending_submit_call_id = "c3"
+    session.pending_tool_results = [{"call_id": "c2", "output": "Edited"}]
+    api = FakeAPI([_tool_calls({"id": "c4", "name": "submit_fix", "args": {"changes_made": "again"}})])
+    render_context.codeplain_api = api
+
+    outcome, _ = FixUnitTests().execute(render_context, {"previous_unittests_issue": "FAILED test_b"})
+
+    assert outcome == FixUnitTests.SUCCESSFUL_OUTCOME
+    kind, session_id, tool_results, frid, module_name = api.calls[0]
+    assert (kind, session_id, frid, module_name) == ("continue", "s1", "2", "m")
+    assert tool_results[0] == {"call_id": "c2", "output": "Edited"}
+    assert tool_results[1]["call_id"] == "c3" and "still fail" in tool_results[1]["output"]
+    assert tool_results[1]["test_output"] == "FAILED test_b"
+    assert "conformance_tests_fixes" not in tool_results[1]
+    assert session.session_id == "s1" and session.pending_submit_call_id == "c4"
+    assert session.pending_tool_results == []
+
+
+@pytest.mark.parametrize(
+    "final_response",
+    [
+        {"session_id": "s1", "status": "completed", "result": "done"},
+        {"session_id": "s1", "status": "failed", "error": "x"},
+    ],
+)
+def test_session_ending_without_submission_resets_the_session(render_context, final_response):
+    render_context.codeplain_api = FakeAPI([final_response])
+    outcome, _ = FixUnitTests().execute(render_context, {"previous_unittests_issue": "FAILED"})
+    assert outcome == FixUnitTests.SUCCESSFUL_OUTCOME
+    session = render_context.unit_tests_running_context.agent_session
+    assert session.session_id is None and session.pending_submit_call_id is None
+
+
+def test_turn_cap_per_attempt_resets_the_session(render_context):
+    call = {"id": "c", "name": "ls_files", "args": {}}
+    render_context.codeplain_api = FakeAPI([_tool_calls(call)] * (MAX_AGENT_TURNS_PER_ATTEMPT + 1))
+    FixUnitTests().execute(render_context, {"previous_unittests_issue": "FAILED"})
+    assert len(render_context.codeplain_api.calls) == MAX_AGENT_TURNS_PER_ATTEMPT + 1
+    assert render_context.unit_tests_running_context.agent_session.session_id is None
+
+
+def test_missing_issue_is_an_internal_error(render_context):
+    from plain2code_exceptions import InternalClientError
+
+    with pytest.raises(InternalClientError):
+        FixUnitTests().execute(render_context, {})
+
+
+def test_first_turn_is_seeded_with_file_tree_relevant_files_and_log_path(render_context, tmp_path):
+    (tmp_path / "build" / "tests").mkdir()
+    (tmp_path / "build" / "tests" / "test_a.py").write_text("assert True\n")
+    log = tmp_path / "unit.log"
+    log.write_text("full log\nCaused by: boom\n")
+    render_context.script_execution_history.latest_unit_test_output_path = str(log)
+    api = FakeAPI([{"session_id": "s1", "status": "completed", "result": "done"}])
+    render_context.codeplain_api = api
+
+    FixUnitTests().execute(render_context, {"previous_unittests_issue": "FAILED"})
+
+    task_params = api.calls[0][2]
+    assert task_params["file_tree"].split("\n") == ["a.py", "tests/test_a.py"]
+    assert task_params["relevant_files"] == {"a.py": "x = 1\n"}
+    assert task_params["unittests_log_path"] == str(log)
+    assert "previous_session_id" not in task_params
+    # the agent may grep the full log although it is outside the build folder and project root
+    assert str(log) in render_context.unit_tests_running_context.agent_session.readable_log_paths
+
+
+def test_relevant_files_stay_within_budget(tmp_path):
+    (tmp_path / "small.py").write_text("s")
+    (tmp_path / "big.py").write_text("b" * MAX_RELEVANT_FILES_CHARS)
+    assert FixUnitTests._relevant_files(str(tmp_path), {"small.py", "big.py", "deleted.py"}) == {"small.py": "s"}
+
+
+def test_new_session_after_abandoned_one_references_it(render_context):
+    call = {"id": "c", "name": "ls_files", "args": {}}
+    render_context.codeplain_api = FakeAPI([_tool_calls(call)] * (MAX_AGENT_TURNS_PER_ATTEMPT + 1))
+    FixUnitTests().execute(render_context, {"previous_unittests_issue": "FAILED"})
+    assert render_context.unit_tests_running_context.agent_session.previous_session_id == "s1"
+
+    api = FakeAPI([{"session_id": "s2", "status": "completed", "result": "done"}])
+    render_context.codeplain_api = api
+    FixUnitTests().execute(render_context, {"previous_unittests_issue": "FAILED again"})
+    assert api.calls[0][0] == "start" and api.calls[0][2]["previous_session_id"] == "s1"
+
+
+def test_run_unit_tests_action_skips_the_suite_after_a_verified_agent_run(render_context, monkeypatch):
+    import render_machine.render_utils as render_utils
+
+    context = render_context.unit_tests_running_context
+    context.verified_passing, context.verified_passing_log_path = True, "/logs/pass.log"
+    monkeypatch.setattr(render_utils, "execute_script", lambda *a, **k: pytest.fail("suite must not run"))
+
+    assert RunUnitTests().execute(render_context, None) == (RunUnitTests.SUCCESSFUL_OUTCOME, None)
+    assert context.verified_passing is False
+    assert render_context.script_execution_history.latest_unit_test_output_path == "/logs/pass.log"
+
+
+def test_expired_session_is_replaced_by_a_new_one(render_context):
+    context = render_context.unit_tests_running_context
+    context.agent_used_in_this_loop = True
+    session = context.agent_session
+    session.session_id, session.pending_submit_call_id = "expired", "c3"
+    not_found = requests.exceptions.HTTPError(response=SimpleNamespace(status_code=404))
+    api = FakeAPI([not_found, {"session_id": "s2", "status": "completed", "result": "done"}])
+    render_context.codeplain_api = api
+
+    outcome, _ = FixUnitTests().execute(render_context, {"previous_unittests_issue": "FAILED test_b"})
+
+    assert outcome == FixUnitTests.SUCCESSFUL_OUTCOME
+    assert [call[0] for call in api.calls] == ["continue", "start"]
+    assert api.calls[1][2]["unittests_issue"] == "FAILED test_b"
+
+
+def test_other_http_errors_on_continue_are_not_swallowed(render_context):
+    context = render_context.unit_tests_running_context
+    context.agent_used_in_this_loop = True
+    context.agent_session.session_id, context.agent_session.pending_submit_call_id = "s1", "c3"
+    render_context.codeplain_api = FakeAPI([requests.exceptions.HTTPError(response=SimpleNamespace(status_code=500))])
+    with pytest.raises(requests.exceptions.HTTPError):
+        FixUnitTests().execute(render_context, {"previous_unittests_issue": "FAILED"})

tests/test_fix_unit_tests_conformance_context.py

@@ -12,25 +12,44 @@ import pytest
 
 import plain_spec
 from memory_management import MemoryManager
-from render_machine.actions import fix_unit_tests as fix_unit_tests_module
 from render_machine.actions.fix_conformance_test import FixConformanceTest
 from render_machine.actions.fix_unit_tests import FixUnitTests
 from render_machine.implementation_code_helpers import ImplementationCodeHelpers
-from render_machine.render_types import ConformanceTestsRunningContext, UnitTestsRunningContext
+from render_machine.render_context import RenderContext
+from render_machine.render_types import ConformanceTestsRunningContext, ScriptExecutionHistory, UnitTestsRunningContext
 
 
 class FakeCodeplainAPI:
-    def __init__(self, conformance_fix_response=None, unittests_fix_response=None):
+    """Conformance fixes return a scripted response; every agent session submits a fix on its first turn."""
+
+    def __init__(self, conformance_fix_response=None):
         self.conformance_fix_response = conformance_fix_response
-        self.unittests_fix_response = unittests_fix_response or {}
-        self.unittests_fix_calls = []
+        self.agent_calls = []
+        self.sessions_started = 0
 
     def fix_conformance_tests_issue(self, *args, **kwargs):
         return self.conformance_fix_response
 
-    def fix_unittests_issue(self, *args, **kwargs):
-        self.unittests_fix_calls.append(kwargs)
-        return self.unittests_fix_response
+    def agent_start(self, task_type, task_params, frid, module_name, run_state):
+        self.sessions_started += 1
+        self.agent_calls.append(("start", task_params))
+        return self._submit(f"s{self.sessions_started}")
+
+    def agent_continue(self, session_id, tool_results, frid, module_name, run_state):
+        self.agent_calls.append(("continue", session_id, tool_results))
+        return self._submit(session_id)
+
+    def _submit(self, session_id):
+        call_id = f"submit-{len(self.agent_calls)}"
+        return {
+            "session_id": session_id,
+            "status": "tool_calls",
+            "calls": [{"id": call_id, "name": "submit_fix", "args": {"changes_made": "fixed"}}],
+        }
+
+
+class FakeRenderContext(SimpleNamespace):
+    unit_tests_agent_session = RenderContext.unit_tests_agent_session
 
 
 class FakeConformanceTests:
@@ -59,7 +78,11 @@ def memory_folder():
 def isolate_from_git_and_console(monkeypatch):
     monkeypatch.setattr(ImplementationCodeHelpers, "get_code_diff", staticmethod(lambda *args: {}))
     monkeypatch.setattr(plain_spec, "collect_linked_resources", lambda *args: None)
-    monkeypatch.setattr(fix_unit_tests_module.render_utils, "print_inputs", lambda *args: None)
+    monkeypatch.setattr(
+        plain_spec,
+        "get_specifications_for_frid",
+        lambda tree, frid: ({plain_spec.FUNCTIONAL_REQUIREMENTS: ["- Add numbers."]}, None),
+    )
 
 
 def make_conformance_context():
@@ -76,7 +99,7 @@ def make_conformance_context():
 
 
 def make_render_context(api, build_folder, memory_folder, conformance_tests_running_context):
-    return SimpleNamespace(
+    return FakeRenderContext(
         codeplain_api=api,
         build_folder=build_folder,
         memory_manager=MemoryManager(api, memory_folder),
@@ -86,9 +109,11 @@ def make_render_context(api, build_folder, memory_folder, conformance_tests_runn
         plain_source_tree={},
         module_name="mod",
         required_modules=None,
-        frid_context=SimpleNamespace(frid="1", linked_resources={}),
+        frid_context=SimpleNamespace(frid="1", linked_resources={}, changed_files=set()),
         get_required_modules_functionalities=lambda: {},
         run_state=SimpleNamespace(render_id="test-render-id", unittest_batch_id=1),
+        unittests_script=None,
+        script_execution_history=ScriptExecutionHistory(),
     )
 
 
@@ -191,34 +216,104 @@ def test_implementation_fix_with_no_files_is_not_remembered(build_folder, memory
     assert ctx.implementation_code_fixes == []
 
 
-def run_unit_tests_fix(render_context):
-    return FixUnitTests().execute(render_context, {"previous_unittests_issue": "1 failed"})
+def run_unit_tests_fix(render_context, issue="1 failed"):
+    return FixUnitTests().execute(render_context, {"previous_unittests_issue": issue})
+
+
+def start_new_unit_test_loop(render_context):
+    """What RenderContext.start_unittests_processing does when the unit tests are run again."""
+    render_context.unit_tests_running_context = UnitTestsRunningContext(fix_attempts=0)
+
+
+def conformance_fix(number):
+    return {
+        "hypothesis": f"hypothesis {number}",
+        "approach": f"approach {number}",
+        "code_diff": {"app.py": f"+{number}"},
+    }
 
 
-def test_unit_tests_fix_forwards_conformance_fixes(build_folder, memory_folder):
-    api = FakeCodeplainAPI(unittests_fix_response={"test_app.py": "def test_add(): pass\n"})
+def test_first_unit_test_loop_of_conformance_phase_seeds_the_session_with_the_fixes(build_folder, memory_folder):
+    api = FakeCodeplainAPI()
     ctx = make_conformance_context()
-    ctx.implementation_code_fixes.append(
-        {"hypothesis": "off by one", "approach": "add one", "code_diff": {"app.py": "+    return a + b + 1"}}
-    )
+    ctx.implementation_code_fixes.append(conformance_fix(1))
     render_context = make_render_context(api, build_folder, memory_folder, ctx)
 
     outcome, _ = run_unit_tests_fix(render_context)
 
     assert outcome == FixUnitTests.SUCCESSFUL_OUTCOME
-    assert len(api.unittests_fix_calls) == 1
-    assert api.unittests_fix_calls[0]["conformance_tests_fixes"] == ctx.implementation_code_fixes
+    kind, task_params = api.agent_calls[0]
+    assert kind == "start"
+    assert task_params["conformance_tests_fixes"] == ctx.implementation_code_fixes
     # The forwarded list is a copy, so later conformance fixes do not mutate what was sent.
-    assert api.unittests_fix_calls[0]["conformance_tests_fixes"] is not ctx.implementation_code_fixes
+    assert task_params["conformance_tests_fixes"] is not ctx.implementation_code_fixes
+    # The file the conformance fix changed is seeded although the FRID did not change it.
+    assert "app.py" in task_params["relevant_files"]
+    assert ctx.unit_tests_agent_session.session_id == "s1"
 
 
-def test_unit_tests_fix_outside_conformance_phase_sends_no_fixes(build_folder, memory_folder):
+def test_next_unit_test_loop_of_conformance_phase_continues_the_session_with_only_new_fixes(
+    build_folder, memory_folder
+):
     api = FakeCodeplainAPI()
-    render_context = make_render_context(api, build_folder, memory_folder, None)
+    ctx = make_conformance_context()
+    ctx.implementation_code_fixes.append(conformance_fix(1))
+    render_context = make_render_context(api, build_folder, memory_folder, ctx)
+    run_unit_tests_fix(render_context)
+
+    # The fix was accepted; the conformance tests fixer changes the code again and the unit tests fail again.
+    ctx.implementation_code_fixes.append(conformance_fix(2))
+    start_new_unit_test_loop(render_context)
+    run_unit_tests_fix(render_context, issue="2 failed")
+
+    assert api.sessions_started == 1
+    kind, session_id, tool_results = api.agent_calls[1]
+    assert (kind, session_id) == ("continue", "s1")
+    submit_answer = tool_results[-1]
+    assert submit_answer["call_id"] == "submit-1"
+    assert submit_answer["output"].startswith("Your fix was accepted: the unit tests passed.")
+    assert "Conformance Tests Fix below" in submit_answer["output"]
+    assert submit_answer["conformance_tests_fixes"] == [conformance_fix(2)]
+    assert submit_answer["test_output"] == "2 failed"
+
 
+def test_retry_within_a_unit_test_loop_says_the_fix_did_not_work(build_folder, memory_folder):
+    api = FakeCodeplainAPI()
+    ctx = make_conformance_context()
+    ctx.implementation_code_fixes.append(conformance_fix(1))
+    render_context = make_render_context(api, build_folder, memory_folder, ctx)
+    run_unit_tests_fix(render_context)
+
+    run_unit_tests_fix(render_context, issue="still failing")
+
+    submit_answer = api.agent_calls[1][2][-1]
+    assert "still fail" in submit_answer["output"]
+    # Fix 1 was already shown to the session, so it is not sent again.
+    assert "conformance_tests_fixes" not in submit_answer
+
+
+def test_new_conformance_phase_starts_a_new_session(build_folder, memory_folder):
+    api = FakeCodeplainAPI()
+    render_context = make_render_context(api, build_folder, memory_folder, make_conformance_context())
+    run_unit_tests_fix(render_context)
+
+    # E.g. the functionality is re-rendered from scratch: the conformance tests running context is recreated.
+    render_context.conformance_tests_running_context = make_conformance_context()
+    start_new_unit_test_loop(render_context)
+    run_unit_tests_fix(render_context)
+
+    assert [call[0] for call in api.agent_calls] == ["start", "start"]
+
+
+def test_unit_tests_outside_conformance_phase_get_no_fixes_and_a_session_per_loop(build_folder, memory_folder):
+    api = FakeCodeplainAPI()
+    render_context = make_render_context(api, build_folder, memory_folder, None)
+    run_unit_tests_fix(render_context)
+    start_new_unit_test_loop(render_context)
     run_unit_tests_fix(render_context)
 
-    assert api.unittests_fix_calls[0]["conformance_tests_fixes"] is None
+    assert [call[0] for call in api.agent_calls] == ["start", "start"]
+    assert all("conformance_tests_fixes" not in call[1] for call in api.agent_calls)
 
 
 def test_unit_tests_fix_in_conformance_phase_without_implementation_changes_sends_no_fixes(build_folder, memory_folder):
@@ -227,4 +322,4 @@ def test_unit_tests_fix_in_conformance_phase_without_implementation_changes_send
 
     run_unit_tests_fix(render_context)
 
-    assert api.unittests_fix_calls[0]["conformance_tests_fixes"] is None
+    assert "conformance_tests_fixes" not in api.agent_calls[0][1]