codeplain 0.4.3.dev60__py3-none-any.whl → 0.4.3.dev62__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {codeplain-0.4.3.dev60.dist-info → codeplain-0.4.3.dev62.dist-info}/METADATA +1 -1
- {codeplain-0.4.3.dev60.dist-info → codeplain-0.4.3.dev62.dist-info}/RECORD +25 -20
- codeplain_REST_api.py +51 -47
- plain2code.py +2 -0
- render_machine/actions/fix_conformance_test.py +347 -145
- render_machine/actions/fix_unit_tests.py +1 -0
- render_machine/actions/prepare_testing_environment.py +89 -15
- render_machine/actions/render_conformance_tests.py +2 -0
- render_machine/actions/run_conformance_tests.py +5 -0
- render_machine/agent_tools.py +414 -83
- render_machine/conformance_fix_conversation.py +409 -0
- render_machine/render_context.py +3 -1
- render_machine/render_types.py +69 -0
- render_machine/render_utils.py +10 -1
- tests/test_agent_tools.py +180 -0
- tests/test_api_client_retries.py +52 -0
- tests/test_background_preparation.py +73 -0
- tests/test_conformance_fix_conversation.py +677 -0
- tests/test_fix_unit_tests_action.py +7 -0
- tests/test_fix_unit_tests_conformance_context.py +8 -106
- tests/test_render_finished_report.py +14 -0
- tests/test_script_timeout_hint.py +24 -0
- {codeplain-0.4.3.dev60.dist-info → codeplain-0.4.3.dev62.dist-info}/WHEEL +0 -0
- {codeplain-0.4.3.dev60.dist-info → codeplain-0.4.3.dev62.dist-info}/entry_points.txt +0 -0
- {codeplain-0.4.3.dev60.dist-info → codeplain-0.4.3.dev62.dist-info}/licenses/LICENSE +0 -0
|
@@ -0,0 +1,409 @@
|
|
|
1
|
+
"""The conversation between the conformance tests fixer and its reviewer.
|
|
2
|
+
|
|
3
|
+
Both are server-side agent sessions (tasks fix_conformance_tests and review_conformance_tests_fix in
|
|
4
|
+
codeplain-api) whose tool calls are executed here. The fixer edits the build folder and the conformance
|
|
5
|
+
tests folder and calls submit_fix; the reviewer reads the code and the diff of the fix and calls
|
|
6
|
+
submit_review. A rejection goes back to the fixer as the result of its submit_fix and it revises the fix;
|
|
7
|
+
an approval ends the conversation for now and the harness runs the tests. When they still fail, the same
|
|
8
|
+
two sessions continue with the new failure output. The sessions live as long as the conformance tests of
|
|
9
|
+
one functionality are being fixed (see ConformanceFixAgents).
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import difflib
|
|
13
|
+
import os
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
from http import HTTPStatus
|
|
16
|
+
from typing import Optional
|
|
17
|
+
|
|
18
|
+
import requests
|
|
19
|
+
|
|
20
|
+
from plain2code_console import console
|
|
21
|
+
from render_machine import agent_tools
|
|
22
|
+
from render_machine.render_types import AgentSessionState, ConformanceFixAgents
|
|
23
|
+
|
|
24
|
+
FIXER_TASK = "fix_conformance_tests"
|
|
25
|
+
REVIEWER_TASK = "review_conformance_tests_fix"
|
|
26
|
+
SUBMIT_FIX = "submit_fix"
|
|
27
|
+
SUBMIT_REVIEW = "submit_review"
|
|
28
|
+
CONFLICTING_REQUIREMENTS = "CONFLICTING_REQUIREMENTS"
|
|
29
|
+
FIXER_TOOLS = frozenset(
|
|
30
|
+
{"read_file", "grep", "ls_files", "edit_file", "write_file", "delete_file", "run_conformance_tests"}
|
|
31
|
+
)
|
|
32
|
+
REVIEWER_TOOLS = frozenset({"read_file", "grep", "ls_files"})
|
|
33
|
+
# Upper bounds on LLM turns per fix and per review; the server forces the submission earlier (40 and 15).
|
|
34
|
+
MAX_FIXER_TURNS_PER_FIX = 50
|
|
35
|
+
MAX_REVIEWER_TURNS_PER_REVIEW = 20
|
|
36
|
+
# Rejections of one fix after which it is reverted and the fixer and reviewer start over in fresh sessions.
|
|
37
|
+
MAX_REJECTIONS = 5
|
|
38
|
+
# Fixer sessions one fix may start (after a revert, or when a session ends without submitting) before giving up.
|
|
39
|
+
MAX_FIXER_SESSIONS_PER_FIX = 3
|
|
40
|
+
|
|
41
|
+
TESTS_STILL_FAIL_MESSAGE = (
|
|
42
|
+
"The reviewer approved your fix and it was applied, but the conformance tests still fail (output below). "
|
|
43
|
+
"Files may have changed since you last read them: after an implementation change the unit tests are run, and "
|
|
44
|
+
"fixing them may adjust the code."
|
|
45
|
+
)
|
|
46
|
+
# From this many failed runs in a row after approved fixes, the fixer is told its approach is not converging.
|
|
47
|
+
NOT_CONVERGING_AFTER_FAILED_RUNS = 3
|
|
48
|
+
NOT_CONVERGING_MESSAGE = (
|
|
49
|
+
"\n\nThe tests have now failed {failed_runs} times in a row after approved fixes. If these failures have the same "
|
|
50
|
+
"cause, the current approach is not converging: do not try another variant of it. Step back and change "
|
|
51
|
+
"approach - question the assumption the attempts share (e.g. that an artifact, version, API, file or setting "
|
|
52
|
+
"exists as you assume), satisfy the requirement in a different way, or, if it cannot be met as stated, say so "
|
|
53
|
+
"in root_cause."
|
|
54
|
+
)
|
|
55
|
+
REJECTED_MESSAGE = "The reviewer rejected your fix."
|
|
56
|
+
EMPTY_FIX_MESSAGE = (
|
|
57
|
+
"Your fix changes no files, so it was not sent to the reviewer: the conformance tests would fail again in the "
|
|
58
|
+
"same way. Make the changes that fix the root cause, then call submit_fix again."
|
|
59
|
+
)
|
|
60
|
+
REVIEW_AFTER_TESTS_MESSAGE = (
|
|
61
|
+
"The fix you approved was applied, but the conformance tests still fail (output below). The fixer has "
|
|
62
|
+
"submitted a new fix; review it."
|
|
63
|
+
)
|
|
64
|
+
REVIEW_REVISION_MESSAGE = "The fixer revised the fix you rejected; review the revision."
|
|
65
|
+
NO_VERDICT_COMMENTS = (
|
|
66
|
+
"The review could not be completed. Re-check that your fix addresses the root cause and follows the rules, "
|
|
67
|
+
"then call submit_fix again."
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@dataclass
|
|
72
|
+
class ApprovedFix:
|
|
73
|
+
"""A fix the reviewer approved: the fixer's submit_fix arguments and the files it changed, with their
|
|
74
|
+
content from before the fix (None: the file did not exist)."""
|
|
75
|
+
|
|
76
|
+
args: dict
|
|
77
|
+
changed_files: dict[str, Optional[str]]
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class ConformanceFixConversation:
|
|
81
|
+
def __init__(self, render_context, agents: ConformanceFixAgents, fixer_params: dict, reviewer_params: dict):
|
|
82
|
+
"""`fixer_params` and `reviewer_params` are the task params a new session of each agent starts with,
|
|
83
|
+
without the issue (which `fix` adds)."""
|
|
84
|
+
self.render_context = render_context
|
|
85
|
+
self.agents = agents
|
|
86
|
+
self.fixer_params = fixer_params
|
|
87
|
+
self.reviewer_params = reviewer_params
|
|
88
|
+
self.build_folder = os.path.normpath(os.path.abspath(render_context.build_folder))
|
|
89
|
+
self.fixer_workspace = agent_tools.AgentWorkspace(
|
|
90
|
+
build_folder=self.build_folder,
|
|
91
|
+
writable_folders=[
|
|
92
|
+
self.build_folder,
|
|
93
|
+
agents.conformance_tests_folder,
|
|
94
|
+
*([agents.writable_module_tests_folder] if agents.writable_module_tests_folder else []),
|
|
95
|
+
],
|
|
96
|
+
readable_folders=[agents.plain_modules_folder, *agents.readable_tests_folders],
|
|
97
|
+
readable_log_paths=agents.readable_log_paths,
|
|
98
|
+
tool_result_cache=agents.fixer_tool_result_cache,
|
|
99
|
+
before_change=self._before_change,
|
|
100
|
+
after_change=lambda _path: agents.reviewer_tool_result_cache.clear(),
|
|
101
|
+
allowed_tools=FIXER_TOOLS,
|
|
102
|
+
render_context=render_context,
|
|
103
|
+
conformance_tests_folder=agents.conformance_tests_folder,
|
|
104
|
+
prepared_signature=self._prepared_signature(render_context),
|
|
105
|
+
)
|
|
106
|
+
self.reviewer_workspace = agent_tools.AgentWorkspace(
|
|
107
|
+
build_folder=self.build_folder,
|
|
108
|
+
writable_folders=[],
|
|
109
|
+
readable_folders=[
|
|
110
|
+
agents.plain_modules_folder,
|
|
111
|
+
agents.conformance_tests_folder,
|
|
112
|
+
*agents.readable_tests_folders,
|
|
113
|
+
],
|
|
114
|
+
readable_log_paths=agents.readable_log_paths,
|
|
115
|
+
tool_result_cache=agents.reviewer_tool_result_cache,
|
|
116
|
+
allowed_tools=REVIEWER_TOOLS,
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
@staticmethod
|
|
120
|
+
def _prepared_signature(render_context) -> Optional[tuple]:
|
|
121
|
+
"""The build folder as the test environment was last prepared for, when that is the build folder as it is
|
|
122
|
+
now (no implementation change since the failing run): run_conformance_tests then needs no preparation."""
|
|
123
|
+
ctx = render_context.conformance_tests_running_context
|
|
124
|
+
if not getattr(render_context, "prepare_environment_script", None) or ctx.should_prepare_testing_environment:
|
|
125
|
+
return None
|
|
126
|
+
from render_machine.actions import prepare_testing_environment
|
|
127
|
+
|
|
128
|
+
return prepare_testing_environment._build_folder_signature(render_context.build_folder)
|
|
129
|
+
|
|
130
|
+
# ---------------------------------------------------------------------------------------------------------
|
|
131
|
+
# The conversation
|
|
132
|
+
# ---------------------------------------------------------------------------------------------------------
|
|
133
|
+
|
|
134
|
+
def fix(self, issue: str, log_path: Optional[str]) -> Optional[ApprovedFix]:
|
|
135
|
+
"""Run the conversation until the reviewer approves a fix for `issue` (the raw conformance tests output).
|
|
136
|
+
Returns None if no fix was approved within the session budget.
|
|
137
|
+
|
|
138
|
+
The changes of an unapproved fix are deliberately left in place (see unapproved_changes), not reverted: a
|
|
139
|
+
fixer that keeps failing to convince the reviewer usually disagrees with it, or the specification is
|
|
140
|
+
ambiguous, and an endless fixer-reviewer loop used to crash the render. The tests decide instead. Reverting
|
|
141
|
+
would also confuse the fixer, whose next session gets a digest of the changes it made."""
|
|
142
|
+
agents = self.agents
|
|
143
|
+
agents.snapshot, agents.rejections = {}, 0
|
|
144
|
+
if log_path:
|
|
145
|
+
agent_tools.register_log_path(log_path, self.fixer_workspace)
|
|
146
|
+
# The tests ran (and maybe unit tests were fixed) since the agents last looked at the files.
|
|
147
|
+
agents.clear_tool_result_caches()
|
|
148
|
+
# The harness has just run the tests on the files as they are, so a run_conformance_tests before any change
|
|
149
|
+
# would only repeat that run.
|
|
150
|
+
self.fixer_workspace.last_test_run = (
|
|
151
|
+
agent_tools.test_run_files(self.render_context, self.agents.conformance_tests_folder),
|
|
152
|
+
{
|
|
153
|
+
"output": "The conformance tests failed: this is the failing run whose output you were given."
|
|
154
|
+
+ agent_tools.full_log_pointer(log_path)
|
|
155
|
+
},
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
if agents.approved_fix_tested:
|
|
159
|
+
agents.failed_runs += 1
|
|
160
|
+
response = None
|
|
161
|
+
if agents.fixer.session_id is not None:
|
|
162
|
+
console.info(f"Continuing the conformance tests fixer session {agents.fixer.session_id}.")
|
|
163
|
+
output = TESTS_STILL_FAIL_MESSAGE + agent_tools.full_log_pointer(log_path)
|
|
164
|
+
if agents.failed_runs >= NOT_CONVERGING_AFTER_FAILED_RUNS:
|
|
165
|
+
output += NOT_CONVERGING_MESSAGE.format(failed_runs=agents.failed_runs)
|
|
166
|
+
response = self._answer(agents.fixer, {"output": output, "test_output": issue})
|
|
167
|
+
|
|
168
|
+
sessions_started = 0
|
|
169
|
+
while True:
|
|
170
|
+
if response is None:
|
|
171
|
+
if sessions_started >= MAX_FIXER_SESSIONS_PER_FIX:
|
|
172
|
+
console.warning("The conformance tests fixer did not get a fix approved.")
|
|
173
|
+
return None
|
|
174
|
+
sessions_started += 1
|
|
175
|
+
response = self._start(agents.fixer, FIXER_TASK, self._fixer_task_params(issue, log_path))
|
|
176
|
+
|
|
177
|
+
submission = self._drive(agents.fixer, response, self.fixer_workspace, SUBMIT_FIX, MAX_FIXER_TURNS_PER_FIX)
|
|
178
|
+
if submission is None:
|
|
179
|
+
# The session ended without a submission (handoff, failure, turn budget): both agents start over,
|
|
180
|
+
# keeping the changes, with a digest of their sessions.
|
|
181
|
+
agents.reset_sessions()
|
|
182
|
+
response = None
|
|
183
|
+
continue
|
|
184
|
+
|
|
185
|
+
args = submission["args"]
|
|
186
|
+
console.info(f"The conformance tests fixer submitted a fix: {args.get('changes_made', '')}")
|
|
187
|
+
changed_files = self._changed_files()
|
|
188
|
+
if not changed_files and args.get("failing_reason") != CONFLICTING_REQUIREMENTS:
|
|
189
|
+
response = self._answer(agents.fixer, {"output": EMPTY_FIX_MESSAGE})
|
|
190
|
+
continue
|
|
191
|
+
verdict, comments, objected_files = self._review(args, changed_files, issue, log_path)
|
|
192
|
+
if verdict == "approve":
|
|
193
|
+
console.info(f"The reviewer approved the fix: {comments or ''}")
|
|
194
|
+
agents.approved_fix_tested = True
|
|
195
|
+
return ApprovedFix(args, changed_files)
|
|
196
|
+
|
|
197
|
+
console.info(f"The reviewer rejected the fix: {comments}")
|
|
198
|
+
agents.rejections += 1
|
|
199
|
+
review: dict = {"comments": comments}
|
|
200
|
+
if agents.rejections % MAX_REJECTIONS == 0:
|
|
201
|
+
# The sessions go on (both keep their history); only the changes the reviewer objected to go.
|
|
202
|
+
reverted = self._revert_objected(objected_files, changed_files)
|
|
203
|
+
if reverted:
|
|
204
|
+
console.warning(
|
|
205
|
+
f"The reviewer rejected {agents.rejections} fixes in a row; reverting the changes it objected "
|
|
206
|
+
f"to: {', '.join(reverted)}."
|
|
207
|
+
)
|
|
208
|
+
review["reverted_files"] = reverted
|
|
209
|
+
response = self._answer(agents.fixer, {"output": REJECTED_MESSAGE, "review": review})
|
|
210
|
+
|
|
211
|
+
def _review(
|
|
212
|
+
self, args: dict, changed_files: dict, issue: str, log_path: Optional[str]
|
|
213
|
+
) -> tuple[str, str, list[str]]:
|
|
214
|
+
"""The reviewer's verdict on the submitted fix: ("approve" | "reject", comments, objected files)."""
|
|
215
|
+
agents = self.agents
|
|
216
|
+
fix_submission = {
|
|
217
|
+
"root_cause": args.get("root_cause", ""),
|
|
218
|
+
"failing_reason": args.get("failing_reason", ""),
|
|
219
|
+
"changes_made": args.get("changes_made", ""),
|
|
220
|
+
"review_response": args.get("review_response", ""),
|
|
221
|
+
"conflict_reason": args.get("conflict_reason", ""),
|
|
222
|
+
"code_diff": self.diff(changed_files),
|
|
223
|
+
"consecutive_rejections": agents.rejections,
|
|
224
|
+
}
|
|
225
|
+
for _ in range(2):
|
|
226
|
+
response = None
|
|
227
|
+
if agents.reviewer.session_id is not None:
|
|
228
|
+
result: dict = {"fix_submission": fix_submission}
|
|
229
|
+
if agents.approved_fix_tested:
|
|
230
|
+
result["output"] = REVIEW_AFTER_TESTS_MESSAGE + agent_tools.full_log_pointer(log_path)
|
|
231
|
+
result["test_output"] = issue
|
|
232
|
+
else:
|
|
233
|
+
result["output"] = REVIEW_REVISION_MESSAGE
|
|
234
|
+
response = self._answer(agents.reviewer, result)
|
|
235
|
+
if response is None:
|
|
236
|
+
params = {**self.reviewer_params, **self._issue_params(issue, log_path)}
|
|
237
|
+
params["fix_submission"] = fix_submission
|
|
238
|
+
response = self._start(agents.reviewer, REVIEWER_TASK, params)
|
|
239
|
+
agents.approved_fix_tested = False
|
|
240
|
+
|
|
241
|
+
submission = self._drive(
|
|
242
|
+
agents.reviewer, response, self.reviewer_workspace, SUBMIT_REVIEW, MAX_REVIEWER_TURNS_PER_REVIEW
|
|
243
|
+
)
|
|
244
|
+
if submission is None:
|
|
245
|
+
agents.reviewer.reset()
|
|
246
|
+
continue
|
|
247
|
+
if response_ended(submission):
|
|
248
|
+
# Its last turn: the verdict counts, and the next review starts a new session with a digest.
|
|
249
|
+
agents.reviewer.reset()
|
|
250
|
+
review_args = submission["args"]
|
|
251
|
+
verdict = "approve" if review_args.get("verdict") == "approve" else "reject"
|
|
252
|
+
return verdict, review_args.get("comments") or "", list(review_args.get("objected_files") or [])
|
|
253
|
+
console.warning("The reviewer did not reach a verdict.")
|
|
254
|
+
return "reject", NO_VERDICT_COMMENTS, []
|
|
255
|
+
|
|
256
|
+
# ---------------------------------------------------------------------------------------------------------
|
|
257
|
+
# Sessions
|
|
258
|
+
# ---------------------------------------------------------------------------------------------------------
|
|
259
|
+
|
|
260
|
+
def _start(self, state: AgentSessionState, task_type: str, task_params: dict) -> dict:
|
|
261
|
+
if state.previous_session_id:
|
|
262
|
+
task_params = {**task_params, "previous_session_id": state.previous_session_id}
|
|
263
|
+
api = self.render_context.codeplain_api
|
|
264
|
+
response = api.agent_start(
|
|
265
|
+
task_type,
|
|
266
|
+
task_params,
|
|
267
|
+
self.render_context.frid_context.frid,
|
|
268
|
+
self.render_context.module_name,
|
|
269
|
+
self.render_context.run_state,
|
|
270
|
+
)
|
|
271
|
+
state.session_id = response["session_id"]
|
|
272
|
+
console.info(f"Started the {task_type} agent session {state.session_id}.")
|
|
273
|
+
return response
|
|
274
|
+
|
|
275
|
+
def _answer(self, state: AgentSessionState, result: dict) -> Optional[dict]:
|
|
276
|
+
"""Answer the session's pending submit call with `result` and run its next turn; None when the session
|
|
277
|
+
cannot be continued any more (expired or ended), in which case a new one is to be started."""
|
|
278
|
+
tool_results = [*state.pending_tool_results, {"call_id": state.pending_call_id, **result}]
|
|
279
|
+
state.pending_call_id, state.pending_tool_results = None, []
|
|
280
|
+
return self._continue(state, tool_results)
|
|
281
|
+
|
|
282
|
+
def _continue(self, state: AgentSessionState, tool_results: list[dict]) -> Optional[dict]:
|
|
283
|
+
assert state.session_id is not None
|
|
284
|
+
try:
|
|
285
|
+
return self.render_context.codeplain_api.agent_continue(
|
|
286
|
+
state.session_id,
|
|
287
|
+
tool_results,
|
|
288
|
+
self.render_context.frid_context.frid,
|
|
289
|
+
self.render_context.module_name,
|
|
290
|
+
self.render_context.run_state,
|
|
291
|
+
)
|
|
292
|
+
except requests.exceptions.HTTPError as e:
|
|
293
|
+
status = e.response.status_code if e.response is not None else None
|
|
294
|
+
if status not in (HTTPStatus.NOT_FOUND, HTTPStatus.BAD_REQUEST):
|
|
295
|
+
raise
|
|
296
|
+
console.warning(f"Agent session {state.session_id} can no longer be continued; starting a new one.")
|
|
297
|
+
state.reset()
|
|
298
|
+
return None
|
|
299
|
+
|
|
300
|
+
def _drive(
|
|
301
|
+
self, state: AgentSessionState, response: dict, workspace, submit_name: str, max_turns: int
|
|
302
|
+
) -> Optional[dict]:
|
|
303
|
+
"""Execute the session's tool calls until it calls `submit_name`, which is returned (left pending, to be
|
|
304
|
+
answered by the next _answer). None if the session ended or used up `max_turns` without it."""
|
|
305
|
+
turns = 0
|
|
306
|
+
while response.get("status") == "tool_calls" and turns < max_turns:
|
|
307
|
+
turns += 1
|
|
308
|
+
calls = response["calls"]
|
|
309
|
+
submission = next((call for call in calls if call["name"] == submit_name), None)
|
|
310
|
+
results = agent_tools.execute_calls([call for call in calls if call is not submission], workspace)
|
|
311
|
+
if submission is not None:
|
|
312
|
+
state.pending_call_id, state.pending_tool_results = submission["id"], results
|
|
313
|
+
return {**submission, "session_ended": bool(response.get("session_ended"))}
|
|
314
|
+
next_response = self._continue(state, results)
|
|
315
|
+
if next_response is None:
|
|
316
|
+
return None
|
|
317
|
+
response = next_response
|
|
318
|
+
if response.get("status") == "failed":
|
|
319
|
+
console.warning(f"Agent session {state.session_id} failed: {response.get('error', 'unknown error')}")
|
|
320
|
+
elif response.get("status") == "tool_calls":
|
|
321
|
+
console.warning(f"Agent session {state.session_id} used {max_turns} turns without submitting.")
|
|
322
|
+
return None
|
|
323
|
+
|
|
324
|
+
# ---------------------------------------------------------------------------------------------------------
|
|
325
|
+
# Files
|
|
326
|
+
# ---------------------------------------------------------------------------------------------------------
|
|
327
|
+
|
|
328
|
+
def _before_change(self, full_path: str) -> None:
|
|
329
|
+
if full_path not in self.agents.snapshot:
|
|
330
|
+
self.agents.snapshot[full_path] = _read(full_path)
|
|
331
|
+
|
|
332
|
+
def unapproved_changes(self) -> dict[str, Optional[str]]:
|
|
333
|
+
"""After fix() returned None: the files the fixer changed without getting the change approved, with their
|
|
334
|
+
content from before it (None: the file did not exist)."""
|
|
335
|
+
return self._changed_files()
|
|
336
|
+
|
|
337
|
+
def _changed_files(self) -> dict[str, Optional[str]]:
|
|
338
|
+
"""Files the current fix changed, with their content from before it."""
|
|
339
|
+
return {path: before for path, before in self.agents.snapshot.items() if _read(path) != before}
|
|
340
|
+
|
|
341
|
+
def _revert_objected(self, objected_files: list[str], changed_files: dict[str, Optional[str]]) -> list[str]:
|
|
342
|
+
"""Restore the changed files the reviewer objected to (named as in the diff, or by full path) to their
|
|
343
|
+
content from before the fix; returns their names as the diff shows them."""
|
|
344
|
+
objected = {name.strip().removeprefix("a/").removeprefix("b/") for name in objected_files}
|
|
345
|
+
reverted = []
|
|
346
|
+
for path, before in sorted(changed_files.items()):
|
|
347
|
+
name = self.display_name(path)
|
|
348
|
+
if name not in objected and path not in objected:
|
|
349
|
+
continue
|
|
350
|
+
if before is None:
|
|
351
|
+
if os.path.isfile(path):
|
|
352
|
+
os.remove(path)
|
|
353
|
+
else:
|
|
354
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
355
|
+
with open(path, "w", encoding="utf-8") as f:
|
|
356
|
+
f.write(before)
|
|
357
|
+
reverted.append(name)
|
|
358
|
+
if reverted:
|
|
359
|
+
# Written here, not through a tool: a repeated read must not return the content from before the revert.
|
|
360
|
+
self.agents.clear_tool_result_caches()
|
|
361
|
+
return reverted
|
|
362
|
+
|
|
363
|
+
def display_name(self, full_path: str) -> str:
|
|
364
|
+
"""Build-folder files by their relative path (as the agents address them), others by their full path."""
|
|
365
|
+
if full_path.startswith(self.build_folder + os.sep):
|
|
366
|
+
return os.path.relpath(full_path, self.build_folder)
|
|
367
|
+
return full_path
|
|
368
|
+
|
|
369
|
+
def diff(self, changed_files: dict[str, Optional[str]]) -> dict[str, str]:
|
|
370
|
+
diffs = {}
|
|
371
|
+
for path, before in sorted(changed_files.items()):
|
|
372
|
+
name = self.display_name(path)
|
|
373
|
+
after = _read(path)
|
|
374
|
+
if after is None:
|
|
375
|
+
diffs[name] = "(file deleted)"
|
|
376
|
+
continue
|
|
377
|
+
diffs[name] = "".join(
|
|
378
|
+
difflib.unified_diff(
|
|
379
|
+
(before or "").splitlines(keepends=True),
|
|
380
|
+
after.splitlines(keepends=True),
|
|
381
|
+
fromfile="/dev/null" if before is None else f"a/{name}",
|
|
382
|
+
tofile=f"b/{name}",
|
|
383
|
+
)
|
|
384
|
+
)
|
|
385
|
+
return diffs
|
|
386
|
+
|
|
387
|
+
# ---------------------------------------------------------------------------------------------------------
|
|
388
|
+
# Task params
|
|
389
|
+
# ---------------------------------------------------------------------------------------------------------
|
|
390
|
+
|
|
391
|
+
def _issue_params(self, issue: str, log_path: Optional[str]) -> dict:
|
|
392
|
+
params = {"conformance_tests_issue": issue}
|
|
393
|
+
if log_path:
|
|
394
|
+
params["conformance_tests_log_path"] = log_path
|
|
395
|
+
return params
|
|
396
|
+
|
|
397
|
+
def _fixer_task_params(self, issue: str, log_path: Optional[str]) -> dict:
|
|
398
|
+
return {**self.fixer_params, **self._issue_params(issue, log_path)}
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
def response_ended(submission: dict) -> bool:
|
|
402
|
+
return bool(submission.get("session_ended"))
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
def _read(path: str) -> Optional[str]:
|
|
406
|
+
if not os.path.isfile(path):
|
|
407
|
+
return None
|
|
408
|
+
with open(path, "r", encoding="utf-8", errors="replace") as f:
|
|
409
|
+
return f.read()
|
render_machine/render_context.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import threading
|
|
2
2
|
from copy import deepcopy
|
|
3
|
-
from typing import Callable, Optional
|
|
3
|
+
from typing import Any, Callable, Optional
|
|
4
4
|
|
|
5
5
|
import file_utils
|
|
6
6
|
import git_utils
|
|
@@ -74,6 +74,8 @@ class RenderContext:
|
|
|
74
74
|
self.stop_event = stop_event
|
|
75
75
|
self.enter_pause_event = enter_pause_event
|
|
76
76
|
self.script_execution_history = ScriptExecutionHistory()
|
|
77
|
+
# A preparation script run started while the conformance tests are generated (see PrepareTestingEnvironment).
|
|
78
|
+
self.background_preparation: Any = None
|
|
77
79
|
self.starting_frid = None
|
|
78
80
|
self.test_script_timeout = test_script_timeout
|
|
79
81
|
|
render_machine/render_types.py
CHANGED
|
@@ -80,6 +80,73 @@ class UnitTestsAgentSession:
|
|
|
80
80
|
self.conformance_fixes_handed_off = 0
|
|
81
81
|
|
|
82
82
|
|
|
83
|
+
@dataclass
|
|
84
|
+
class AgentSessionState:
|
|
85
|
+
"""One server-side agent session of the conformance tests fixing conversation, and the call it still owes an
|
|
86
|
+
answer to (the submit call its last attempt ended with, answered when the conversation continues)."""
|
|
87
|
+
|
|
88
|
+
session_id: Optional[str] = None
|
|
89
|
+
pending_call_id: Optional[str] = None
|
|
90
|
+
# Results of tool calls made in the same turn as the pending call; sent along with its answer.
|
|
91
|
+
pending_tool_results: list[dict] = field(default_factory=list)
|
|
92
|
+
# The session this one is to take over from; the next session starts with a digest of it.
|
|
93
|
+
previous_session_id: Optional[str] = None
|
|
94
|
+
|
|
95
|
+
def reset(self) -> None:
|
|
96
|
+
"""Drop the session (keeping its id as the previous one) so the next turn starts a new one."""
|
|
97
|
+
if self.session_id is not None:
|
|
98
|
+
self.previous_session_id = self.session_id
|
|
99
|
+
self.session_id, self.pending_call_id, self.pending_tool_results = None, None, []
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
@dataclass
|
|
103
|
+
class ConformanceFixAgents:
|
|
104
|
+
"""The fixer and the reviewer fixing the conformance tests of one functionality. The reviewer lives alongside
|
|
105
|
+
the fixer: both are started, reset and dropped together. Dropped when that functionality's tests pass."""
|
|
106
|
+
|
|
107
|
+
# (module name, functionality ID) of the conformance tests being fixed.
|
|
108
|
+
tested: tuple[str, str]
|
|
109
|
+
# The writable folder of those conformance tests (a copy in the current module for a required module's tests).
|
|
110
|
+
conformance_tests_folder: str
|
|
111
|
+
# Conformance tests folders of the current module and its required modules, which both agents may read: the
|
|
112
|
+
# tests of other functionalities that already pass show what works (e.g. how they build and start the app).
|
|
113
|
+
readable_tests_folders: list[str] = field(default_factory=list)
|
|
114
|
+
# The rendered modules folder (plain_modules): both agents may read every module's code and tests.
|
|
115
|
+
plain_modules_folder: str = ""
|
|
116
|
+
# For a regression (earlier tests broken by the current functionality): the current module's conformance tests
|
|
117
|
+
# folder, which the fixer may write so it fixes the other earlier tests that share the cause in the same fix.
|
|
118
|
+
writable_module_tests_folder: str = ""
|
|
119
|
+
# For a regression: the folders of the module's other earlier functionalities' tests (copies, for a required
|
|
120
|
+
# module), named to the fixer so it finds them (a required module's copies are in a hidden folder).
|
|
121
|
+
other_tests_folders: list[str] = field(default_factory=list)
|
|
122
|
+
fixer: AgentSessionState = field(default_factory=AgentSessionState)
|
|
123
|
+
reviewer: AgentSessionState = field(default_factory=AgentSessionState)
|
|
124
|
+
# Content of every file the current fix changed, from before the fix (None: the file did not exist), so the fix
|
|
125
|
+
# can be shown as a diff and reverted.
|
|
126
|
+
snapshot: dict[str, Optional[str]] = field(default_factory=dict)
|
|
127
|
+
# Conformance test runs that failed after an approved fix, in a row (the fixer is told once they pile up).
|
|
128
|
+
failed_runs: int = 0
|
|
129
|
+
# The reviewer's rejections in a row since the tests last ran (the reviewer is told; every MAX_REJECTIONS of them,
|
|
130
|
+
# the changes it objected to are reverted).
|
|
131
|
+
rejections: int = 0
|
|
132
|
+
# Whether the reviewer's pending verdict was an approval whose fix was tested (it is told the outcome).
|
|
133
|
+
approved_fix_tested: bool = False
|
|
134
|
+
# Full test logs the agents were pointed to.
|
|
135
|
+
readable_log_paths: set[str] = field(default_factory=set)
|
|
136
|
+
# Each agent's read-only tool results (an agent is pointed to its own earlier results); cleared on any change.
|
|
137
|
+
fixer_tool_result_cache: dict[str, str] = field(default_factory=dict)
|
|
138
|
+
reviewer_tool_result_cache: dict[str, str] = field(default_factory=dict)
|
|
139
|
+
|
|
140
|
+
def reset_sessions(self) -> None:
|
|
141
|
+
self.fixer.reset()
|
|
142
|
+
self.reviewer.reset()
|
|
143
|
+
self.clear_tool_result_caches()
|
|
144
|
+
|
|
145
|
+
def clear_tool_result_caches(self) -> None:
|
|
146
|
+
self.fixer_tool_result_cache.clear()
|
|
147
|
+
self.reviewer_tool_result_cache.clear()
|
|
148
|
+
|
|
149
|
+
|
|
83
150
|
@dataclass
|
|
84
151
|
class UnitTestsRunningContext:
|
|
85
152
|
fix_attempts: int
|
|
@@ -142,6 +209,8 @@ class ConformanceTestsRunningContext:
|
|
|
142
209
|
self.implementation_code_fixes: list[dict] = []
|
|
143
210
|
# The unit-test fixing agent session, shared by every unit-test loop of this conformance phase.
|
|
144
211
|
self.unit_tests_agent_session = UnitTestsAgentSession()
|
|
212
|
+
# The fixer and reviewer of the conformance tests currently failing; None when no fix is in progress.
|
|
213
|
+
self.conformance_fix_agents: Optional[ConformanceFixAgents] = None
|
|
145
214
|
|
|
146
215
|
def get_conformance_tests_json(self, module_name: str) -> dict:
|
|
147
216
|
return self._conformance_tests_json[module_name]
|
render_machine/render_utils.py
CHANGED
|
@@ -19,6 +19,15 @@ from plain2code_utils import with_position
|
|
|
19
19
|
|
|
20
20
|
SCRIPT_EXECUTION_TIMEOUT = 120
|
|
21
21
|
TIMEOUT_ERROR_EXIT_CODE = 124
|
|
22
|
+
# Told to the fixers with a timed-out run: the run was stopped before it reported, so its output often shows no test
|
|
23
|
+
# result, and guessing at the implementation from it repeats the same timeout.
|
|
24
|
+
TIMEOUT_HINT = (
|
|
25
|
+
" It was stopped, so the output may show no test result at all. A run that does not finish usually has a test"
|
|
26
|
+
" that blocks: it waits for input that never comes (a prompt, a terminal, stdin), for a subprocess or server"
|
|
27
|
+
" that does not exit, or on a lock or a network call. Find the test that blocks and why before changing anything"
|
|
28
|
+
" else: a timeout on each blocking call in the tests turns the hang into a reported failure whose message shows"
|
|
29
|
+
" what to fix."
|
|
30
|
+
)
|
|
22
31
|
POLL_INTERVAL_SECONDS = 0.2
|
|
23
32
|
SIGTERM_GRACE_PERIOD_SECONDS = 0.2
|
|
24
33
|
STDOUT_READ_TIMEOUT_SECONDS = 2
|
|
@@ -232,6 +241,6 @@ def execute_script( # noqa: C901
|
|
|
232
241
|
partial_output = f"\nPartial test script output:\n{sanitized}"
|
|
233
242
|
return (
|
|
234
243
|
TIMEOUT_ERROR_EXIT_CODE,
|
|
235
|
-
f"{script_type} script did not finish in {script_timeout} seconds.{partial_output}",
|
|
244
|
+
f"{script_type} script did not finish in {script_timeout} seconds.{TIMEOUT_HINT}{partial_output}",
|
|
236
245
|
temp_file_path,
|
|
237
246
|
)
|