codeplain 0.4.3.dev61__py3-none-any.whl → 0.4.3.dev63__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {codeplain-0.4.3.dev61.dist-info → codeplain-0.4.3.dev63.dist-info}/METADATA +2 -2
- {codeplain-0.4.3.dev61.dist-info → codeplain-0.4.3.dev63.dist-info}/RECORD +25 -20
- codeplain_REST_api.py +51 -47
- plain2code.py +2 -0
- render_machine/actions/fix_conformance_test.py +347 -145
- render_machine/actions/fix_unit_tests.py +1 -0
- render_machine/actions/prepare_testing_environment.py +89 -15
- render_machine/actions/render_conformance_tests.py +2 -0
- render_machine/actions/run_conformance_tests.py +5 -0
- render_machine/agent_tools.py +414 -83
- render_machine/conformance_fix_conversation.py +409 -0
- render_machine/render_context.py +3 -1
- render_machine/render_types.py +69 -0
- render_machine/render_utils.py +10 -1
- tests/test_agent_tools.py +180 -0
- tests/test_api_client_retries.py +52 -0
- tests/test_background_preparation.py +73 -0
- tests/test_conformance_fix_conversation.py +677 -0
- tests/test_fix_unit_tests_action.py +7 -0
- tests/test_fix_unit_tests_conformance_context.py +8 -106
- tests/test_render_finished_report.py +14 -0
- tests/test_script_timeout_hint.py +24 -0
- {codeplain-0.4.3.dev61.dist-info → codeplain-0.4.3.dev63.dist-info}/WHEEL +0 -0
- {codeplain-0.4.3.dev61.dist-info → codeplain-0.4.3.dev63.dist-info}/entry_points.txt +0 -0
- {codeplain-0.4.3.dev61.dist-info → codeplain-0.4.3.dev63.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import os
|
|
2
|
+
|
|
1
3
|
"""Tests for the agentic FixUnitTests action driving a scripted fake API."""
|
|
2
4
|
|
|
3
5
|
from types import SimpleNamespace
|
|
@@ -17,6 +19,11 @@ class FakeRenderContext(SimpleNamespace):
|
|
|
17
19
|
|
|
18
20
|
unit_tests_agent_session = RenderContext.unit_tests_agent_session
|
|
19
21
|
|
|
22
|
+
@property
|
|
23
|
+
def plain_module(self):
|
|
24
|
+
# The rendered modules folder is the build folder's parent, as in plain_modules/<module>.
|
|
25
|
+
return SimpleNamespace(build_folder=os.path.dirname(self.build_folder), all_required_modules=[])
|
|
26
|
+
|
|
20
27
|
|
|
21
28
|
class FakeAPI:
|
|
22
29
|
def __init__(self, responses):
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
When the conformance tests fixer changes implementation code, the unit tests are re-run and fixed next.
|
|
4
4
|
The unit tests fixer must know about that change so it adjusts the unit tests instead of reverting it.
|
|
5
|
+
How the conformance tests fixer records those changes is tested in test_conformance_fix_conversation.py.
|
|
5
6
|
"""
|
|
6
7
|
|
|
7
8
|
import os
|
|
@@ -12,7 +13,6 @@ import pytest
|
|
|
12
13
|
|
|
13
14
|
import plain_spec
|
|
14
15
|
from memory_management import MemoryManager
|
|
15
|
-
from render_machine.actions.fix_conformance_test import FixConformanceTest
|
|
16
16
|
from render_machine.actions.fix_unit_tests import FixUnitTests
|
|
17
17
|
from render_machine.implementation_code_helpers import ImplementationCodeHelpers
|
|
18
18
|
from render_machine.render_context import RenderContext
|
|
@@ -20,16 +20,12 @@ from render_machine.render_types import ConformanceTestsRunningContext, ScriptEx
|
|
|
20
20
|
|
|
21
21
|
|
|
22
22
|
class FakeCodeplainAPI:
|
|
23
|
-
"""
|
|
23
|
+
"""Every agent session submits a fix on its first turn."""
|
|
24
24
|
|
|
25
|
-
def __init__(self
|
|
26
|
-
self.conformance_fix_response = conformance_fix_response
|
|
25
|
+
def __init__(self):
|
|
27
26
|
self.agent_calls = []
|
|
28
27
|
self.sessions_started = 0
|
|
29
28
|
|
|
30
|
-
def fix_conformance_tests_issue(self, *args, **kwargs):
|
|
31
|
-
return self.conformance_fix_response
|
|
32
|
-
|
|
33
29
|
def agent_start(self, task_type, task_params, frid, module_name, run_state):
|
|
34
30
|
self.sessions_started += 1
|
|
35
31
|
self.agent_calls.append(("start", task_params))
|
|
@@ -54,6 +50,11 @@ class FakeRenderContext(SimpleNamespace):
|
|
|
54
50
|
def testing_functionality_label(self):
|
|
55
51
|
return self.conformance_tests_running_context.current_testing_frid
|
|
56
52
|
|
|
53
|
+
@property
|
|
54
|
+
def plain_module(self):
|
|
55
|
+
# The rendered modules folder is the build folder's parent, as in plain_modules/<module>.
|
|
56
|
+
return SimpleNamespace(build_folder=os.path.dirname(self.build_folder), all_required_modules=[])
|
|
57
|
+
|
|
57
58
|
|
|
58
59
|
class FakeConformanceTests:
|
|
59
60
|
def fetch_existing_conformance_test_files(self, *args):
|
|
@@ -120,105 +121,6 @@ def make_render_context(api, build_folder, memory_folder, conformance_tests_runn
|
|
|
120
121
|
)
|
|
121
122
|
|
|
122
123
|
|
|
123
|
-
def run_conformance_fix(render_context):
|
|
124
|
-
return FixConformanceTest().execute(render_context, {"previous_conformance_tests_issue": "tests failed"})
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
def test_conformance_fix_of_implementation_code_is_remembered(build_folder, memory_folder):
|
|
128
|
-
api = FakeCodeplainAPI(
|
|
129
|
-
conformance_fix_response=[
|
|
130
|
-
FixConformanceTest.ISSUE_REASON_CODE_IMPLEMENTATION_CODE,
|
|
131
|
-
{"app.py": "def add(a, b):\n return a + b + 1\n"},
|
|
132
|
-
{"hypothesis": "off by one", "approach": "add one"},
|
|
133
|
-
"prepared issue",
|
|
134
|
-
]
|
|
135
|
-
)
|
|
136
|
-
ctx = make_conformance_context()
|
|
137
|
-
render_context = make_render_context(api, build_folder, memory_folder, ctx)
|
|
138
|
-
|
|
139
|
-
outcome, _ = run_conformance_fix(render_context)
|
|
140
|
-
|
|
141
|
-
assert outcome == FixConformanceTest.IMPLEMENTATION_CODE_UPDATED
|
|
142
|
-
assert len(ctx.implementation_code_fixes) == 1
|
|
143
|
-
recorded_fix = ctx.implementation_code_fixes[0]
|
|
144
|
-
assert recorded_fix["hypothesis"] == "off by one"
|
|
145
|
-
assert recorded_fix["approach"] == "add one"
|
|
146
|
-
assert list(recorded_fix["code_diff"]) == ["app.py"]
|
|
147
|
-
assert "+ return a + b + 1" in recorded_fix["code_diff"]["app.py"]
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
def test_consecutive_implementation_fixes_accumulate(build_folder, memory_folder):
|
|
151
|
-
ctx = make_conformance_context()
|
|
152
|
-
for attempt in range(2):
|
|
153
|
-
api = FakeCodeplainAPI(
|
|
154
|
-
conformance_fix_response=[
|
|
155
|
-
FixConformanceTest.ISSUE_REASON_CODE_IMPLEMENTATION_CODE,
|
|
156
|
-
{"app.py": f"def add(a, b):\n return a + b + {attempt + 1}\n"},
|
|
157
|
-
{"hypothesis": f"hypothesis {attempt}", "approach": f"approach {attempt}"},
|
|
158
|
-
"prepared issue",
|
|
159
|
-
]
|
|
160
|
-
)
|
|
161
|
-
render_context = make_render_context(api, build_folder, memory_folder, ctx)
|
|
162
|
-
run_conformance_fix(render_context)
|
|
163
|
-
|
|
164
|
-
assert [fix["hypothesis"] for fix in ctx.implementation_code_fixes] == ["hypothesis 0", "hypothesis 1"]
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
def test_conformance_fix_without_summary_is_still_remembered(build_folder, memory_folder):
|
|
168
|
-
api = FakeCodeplainAPI(
|
|
169
|
-
conformance_fix_response=[
|
|
170
|
-
FixConformanceTest.ISSUE_REASON_CODE_IMPLEMENTATION_CODE,
|
|
171
|
-
{"app.py": "def add(a, b):\n return a + b + 1\n"},
|
|
172
|
-
None,
|
|
173
|
-
"prepared issue",
|
|
174
|
-
]
|
|
175
|
-
)
|
|
176
|
-
ctx = make_conformance_context()
|
|
177
|
-
render_context = make_render_context(api, build_folder, memory_folder, ctx)
|
|
178
|
-
|
|
179
|
-
run_conformance_fix(render_context)
|
|
180
|
-
|
|
181
|
-
assert ctx.implementation_code_fixes[0]["hypothesis"] is None
|
|
182
|
-
assert ctx.implementation_code_fixes[0]["approach"] is None
|
|
183
|
-
assert "app.py" in ctx.implementation_code_fixes[0]["code_diff"]
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
def test_conformance_fix_of_conformance_tests_is_not_remembered(build_folder, memory_folder):
|
|
187
|
-
api = FakeCodeplainAPI(
|
|
188
|
-
conformance_fix_response=[
|
|
189
|
-
FixConformanceTest.ISSUE_REASON_CODE_CONFORMANCE_TESTS,
|
|
190
|
-
{"test_app.py": "def test_add(): pass\n"},
|
|
191
|
-
{"hypothesis": "wrong assertion", "approach": "fix test"},
|
|
192
|
-
"prepared issue",
|
|
193
|
-
]
|
|
194
|
-
)
|
|
195
|
-
ctx = make_conformance_context()
|
|
196
|
-
render_context = make_render_context(api, build_folder, memory_folder, ctx)
|
|
197
|
-
|
|
198
|
-
outcome, _ = run_conformance_fix(render_context)
|
|
199
|
-
|
|
200
|
-
assert outcome == FixConformanceTest.IMPLEMENTATION_CODE_NOT_UPDATED
|
|
201
|
-
assert ctx.implementation_code_fixes == []
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
def test_implementation_fix_with_no_files_is_not_remembered(build_folder, memory_folder):
|
|
205
|
-
api = FakeCodeplainAPI(
|
|
206
|
-
conformance_fix_response=[
|
|
207
|
-
FixConformanceTest.ISSUE_REASON_CODE_IMPLEMENTATION_CODE,
|
|
208
|
-
{},
|
|
209
|
-
{"hypothesis": "nothing", "approach": "nothing"},
|
|
210
|
-
"prepared issue",
|
|
211
|
-
]
|
|
212
|
-
)
|
|
213
|
-
ctx = make_conformance_context()
|
|
214
|
-
render_context = make_render_context(api, build_folder, memory_folder, ctx)
|
|
215
|
-
|
|
216
|
-
outcome, _ = run_conformance_fix(render_context)
|
|
217
|
-
|
|
218
|
-
assert outcome == FixConformanceTest.IMPLEMENTATION_CODE_NOT_UPDATED
|
|
219
|
-
assert ctx.implementation_code_fixes == []
|
|
220
|
-
|
|
221
|
-
|
|
222
124
|
def run_unit_tests_fix(render_context, issue="1 failed"):
|
|
223
125
|
return FixUnitTests().execute(render_context, {"previous_unittests_issue": issue})
|
|
224
126
|
|
|
@@ -162,6 +162,20 @@ class TestMainReportsEveryEnding:
|
|
|
162
162
|
|
|
163
163
|
assert api.render_finished.call_args.args[0] == "failed"
|
|
164
164
|
|
|
165
|
+
def test_an_error_after_a_module_completed_is_reported_as_a_failure(self):
|
|
166
|
+
"""A required module that rendered fine leaves render_succeeded True; an error in a later module must
|
|
167
|
+
still make the render a failure, for the report and for the exit summary."""
|
|
168
|
+
summary = MagicMock()
|
|
169
|
+
|
|
170
|
+
def fail_after_a_module(run_state):
|
|
171
|
+
run_state.set_render_succeeded(True)
|
|
172
|
+
raise Exception("Max retries (4) exceeded")
|
|
173
|
+
|
|
174
|
+
api = run_main(fail_after_a_module, print_exit_summary=summary)
|
|
175
|
+
|
|
176
|
+
assert api.render_finished.call_args.args[0] == "failed"
|
|
177
|
+
assert summary.call_args.args[0].render_succeeded is False
|
|
178
|
+
|
|
165
179
|
def test_an_expected_error_is_reported_as_a_failure(self):
|
|
166
180
|
def fail(run_state):
|
|
167
181
|
raise InvalidAPIKey("key went stale mid-render")
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""What the fixers are told when a test script runs out of time."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import stat
|
|
5
|
+
import sys
|
|
6
|
+
|
|
7
|
+
from render_machine import render_utils
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def test_a_timed_out_script_tells_the_fixer_to_find_the_blocking_test(tmp_path):
|
|
11
|
+
# execute_script runs only PowerShell scripts on Windows.
|
|
12
|
+
if sys.platform == "win32":
|
|
13
|
+
script = tmp_path / "hang.ps1"
|
|
14
|
+
script.write_text("Write-Output starting\nStart-Sleep -Seconds 30\n")
|
|
15
|
+
else:
|
|
16
|
+
script = tmp_path / "hang.sh"
|
|
17
|
+
script.write_text("#!/bin/bash\necho starting\nsleep 30\n")
|
|
18
|
+
script.chmod(script.stat().st_mode | stat.S_IEXEC)
|
|
19
|
+
|
|
20
|
+
exit_code, issue, log_path = render_utils.execute_script(str(script), [], "Conformance Tests", timeout=1)
|
|
21
|
+
|
|
22
|
+
assert exit_code == render_utils.TIMEOUT_ERROR_EXIT_CODE
|
|
23
|
+
assert "did not finish in 1 seconds" in issue and "a test that blocks" in issue
|
|
24
|
+
assert os.path.isfile(log_path)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|