codeplain 0.4.3.dev60__py3-none-any.whl → 0.4.3.dev62__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {codeplain-0.4.3.dev60.dist-info → codeplain-0.4.3.dev62.dist-info}/METADATA +1 -1
- {codeplain-0.4.3.dev60.dist-info → codeplain-0.4.3.dev62.dist-info}/RECORD +25 -20
- codeplain_REST_api.py +51 -47
- plain2code.py +2 -0
- render_machine/actions/fix_conformance_test.py +347 -145
- render_machine/actions/fix_unit_tests.py +1 -0
- render_machine/actions/prepare_testing_environment.py +89 -15
- render_machine/actions/render_conformance_tests.py +2 -0
- render_machine/actions/run_conformance_tests.py +5 -0
- render_machine/agent_tools.py +414 -83
- render_machine/conformance_fix_conversation.py +409 -0
- render_machine/render_context.py +3 -1
- render_machine/render_types.py +69 -0
- render_machine/render_utils.py +10 -1
- tests/test_agent_tools.py +180 -0
- tests/test_api_client_retries.py +52 -0
- tests/test_background_preparation.py +73 -0
- tests/test_conformance_fix_conversation.py +677 -0
- tests/test_fix_unit_tests_action.py +7 -0
- tests/test_fix_unit_tests_conformance_context.py +8 -106
- tests/test_render_finished_report.py +14 -0
- tests/test_script_timeout_hint.py +24 -0
- {codeplain-0.4.3.dev60.dist-info → codeplain-0.4.3.dev62.dist-info}/WHEEL +0 -0
- {codeplain-0.4.3.dev60.dist-info → codeplain-0.4.3.dev62.dist-info}/entry_points.txt +0 -0
- {codeplain-0.4.3.dev60.dist-info → codeplain-0.4.3.dev62.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,36 +1,52 @@
|
|
|
1
|
-
|
|
1
|
+
import os
|
|
2
|
+
from typing import Any, Optional
|
|
2
3
|
|
|
3
|
-
import diff_utils
|
|
4
4
|
import file_utils
|
|
5
5
|
import plain_spec
|
|
6
6
|
from memory_management import MemoryManager
|
|
7
7
|
from plain2code_console import RETRY_COLOR, console
|
|
8
8
|
from plain2code_exceptions import InternalClientError
|
|
9
|
+
from plain2code_utils import qualified_frid
|
|
10
|
+
from render_machine import agent_tools
|
|
9
11
|
from render_machine.actions.base_action import BaseAction
|
|
12
|
+
from render_machine.actions.fix_unit_tests import FixUnitTests
|
|
13
|
+
from render_machine.conformance_fix_conversation import (
|
|
14
|
+
CONFLICTING_REQUIREMENTS,
|
|
15
|
+
ConformanceFixConversation,
|
|
16
|
+
)
|
|
10
17
|
from render_machine.implementation_code_helpers import ImplementationCodeHelpers
|
|
11
18
|
from render_machine.render_context import RenderContext
|
|
12
|
-
from render_machine.render_types import RenderError, TestExecutionPhase
|
|
19
|
+
from render_machine.render_types import ConformanceFixAgents, RenderError, TestExecutionPhase
|
|
13
20
|
|
|
14
21
|
MAX_CONFORMANCE_TEST_FIX_ATTEMPTS = 20
|
|
15
22
|
MAX_CONFORMANCE_TEST_RERENDER_ATTEMPTS = 1
|
|
23
|
+
# What is recorded for the changes of a fix the reviewer did not approve (see FixConformanceTest.execute).
|
|
24
|
+
UNAPPROVED_FIX_ARGS = {
|
|
25
|
+
"root_cause": None,
|
|
26
|
+
"changes_made": "Changes the conformance tests fixer made without getting them approved by the reviewer.",
|
|
27
|
+
}
|
|
28
|
+
# Bound on the conformance tests files a new session is seeded with.
|
|
29
|
+
MAX_CONFORMANCE_TESTS_FILES_CHARS = 60_000
|
|
16
30
|
|
|
17
31
|
|
|
18
32
|
class FixConformanceTest(BaseAction):
|
|
33
|
+
"""Fix failing conformance tests through a conversation between a fixer agent and a reviewer agent (see
|
|
34
|
+
ConformanceFixConversation). The action returns once the reviewer approved a fix; the state machine then
|
|
35
|
+
runs the tests again (after the unit tests, when the implementation code changed). The two agents live as
|
|
36
|
+
long as the conformance tests of the same functionality keep failing, so each next execution continues
|
|
37
|
+
them with the new failure output."""
|
|
38
|
+
|
|
19
39
|
IMPLEMENTATION_CODE_NOT_UPDATED = "implementation_code_not_updated"
|
|
20
40
|
IMPLEMENTATION_CODE_UPDATED = "implementation_code_updated"
|
|
21
41
|
LIMIT_EXCEEDED_OUTCOME = "conformance_test_fix_limit_exceeded"
|
|
22
42
|
REGENERATE_CONFORMANCE_TESTS_OUTCOME = "regenerate_conformance_tests"
|
|
23
43
|
|
|
24
|
-
ISSUE_REASON_CODE_CONFORMANCE_TESTS = 0
|
|
25
|
-
ISSUE_REASON_CODE_IMPLEMENTATION_CODE = 1
|
|
26
|
-
ISSUE_REASON_CODE_CONFLICTING_REQUIREMENTS = 2
|
|
27
|
-
ISSUE_REASON_CODE_CONFLICTING_ACCEPTANCE_TESTS = 3
|
|
28
|
-
|
|
29
44
|
def execute(self, render_context: RenderContext, previous_action_payload: Any | None):
|
|
30
45
|
ctx = render_context.conformance_tests_running_context
|
|
31
46
|
ctx.fix_attempts += 1
|
|
32
47
|
|
|
33
48
|
if ctx.fix_attempts >= MAX_CONFORMANCE_TEST_FIX_ATTEMPTS:
|
|
49
|
+
ctx.conformance_fix_agents = None
|
|
34
50
|
if ctx.conformance_tests_render_attempts >= MAX_CONFORMANCE_TEST_RERENDER_ATTEMPTS:
|
|
35
51
|
error_msg = f"The renderer was unable to produce an implementation that passes conformance tests for functionality '{render_context.frid_context.frid}' after many attempts. Please review and rewrite the specification. (Render ID: {render_context.run_state.render_id})"
|
|
36
52
|
render_context.last_error_message = error_msg
|
|
@@ -43,179 +59,365 @@ class FixConformanceTest(BaseAction):
|
|
|
43
59
|
return self.REGENERATE_CONFORMANCE_TESTS_OUTCOME, None
|
|
44
60
|
|
|
45
61
|
console.info(f"Running conformance tests attempt {ctx.fix_attempts + 1}.")
|
|
46
|
-
|
|
47
62
|
console.info(f"Fixing conformance test for functionality {render_context.testing_functionality_label()}.")
|
|
48
63
|
|
|
49
|
-
if not previous_action_payload.get("previous_conformance_tests_issue"):
|
|
64
|
+
if not previous_action_payload or not previous_action_payload.get("previous_conformance_tests_issue"):
|
|
50
65
|
raise InternalClientError(
|
|
51
66
|
"Internal client error: Previous action payload does not contain previous conformance tests issue."
|
|
52
67
|
)
|
|
53
|
-
|
|
68
|
+
issue = previous_action_payload["previous_conformance_tests_issue"]
|
|
54
69
|
|
|
55
70
|
journal = render_context.memory_manager.journal
|
|
56
71
|
journal.record_result(ctx.current_testing_module_name, ctx.current_testing_frid, passed=False)
|
|
57
|
-
journal.open_attempt(
|
|
72
|
+
journal.open_attempt(ctx.current_testing_module_name, ctx.current_testing_frid, ctx.fix_attempts, issue)
|
|
73
|
+
|
|
74
|
+
# Reset the conflicting requirement count if the current testing functionality is not the same as the
|
|
75
|
+
# previously conflicting functionality.
|
|
76
|
+
if (ctx.conflicting_module_name, ctx.conflicting_frid) != (
|
|
58
77
|
ctx.current_testing_module_name,
|
|
59
78
|
ctx.current_testing_frid,
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
)
|
|
79
|
+
):
|
|
80
|
+
ctx.conflicting_requirement_count = 0
|
|
63
81
|
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
(
|
|
69
|
-
existing_conformance_test_files,
|
|
70
|
-
existing_conformance_test_files_content,
|
|
71
|
-
) = render_context.conformance_tests.fetch_existing_conformance_test_files(
|
|
72
|
-
render_context.module_name,
|
|
73
|
-
render_context.required_modules,
|
|
74
|
-
render_context.conformance_tests_running_context.current_testing_module_name,
|
|
75
|
-
render_context.conformance_tests_running_context.get_current_conformance_test_folder_name(),
|
|
82
|
+
agents = self._agents_for_current_tests(render_context)
|
|
83
|
+
common_params = self._common_params(render_context, agents)
|
|
84
|
+
conversation = ConformanceFixConversation(
|
|
85
|
+
render_context, agents, self._fixer_params(render_context, common_params), common_params
|
|
76
86
|
)
|
|
77
|
-
|
|
78
|
-
|
|
87
|
+
log_path = render_context.script_execution_history.latest_conformance_test_output_path
|
|
88
|
+
fix = conversation.fix(issue, log_path)
|
|
89
|
+
if fix is not None:
|
|
90
|
+
return self._apply_outcome(render_context, conversation, fix.args, fix.changed_files)
|
|
91
|
+
|
|
92
|
+
# No fix was approved. Its changes stay (see ConformanceFixConversation.fix) and the tests decide; changed
|
|
93
|
+
# implementation code is then tested like an approved change, after the unit tests and a fresh preparation,
|
|
94
|
+
# so it cannot be committed with broken unit tests or tested on an environment prepared for older code.
|
|
95
|
+
changed_files = conversation.unapproved_changes()
|
|
96
|
+
if any(path.startswith(conversation.build_folder + os.sep) for path in changed_files):
|
|
97
|
+
console.warning(
|
|
98
|
+
"No fix was approved; the implementation code the fixer changed is kept and tested: the unit tests "
|
|
99
|
+
"first, then the conformance tests."
|
|
100
|
+
)
|
|
101
|
+
return self._apply_outcome(render_context, conversation, UNAPPROVED_FIX_ARGS, changed_files)
|
|
102
|
+
journal.record_fix(
|
|
103
|
+
ctx.current_testing_module_name, ctx.current_testing_frid, None, [], "IMPLEMENTATION_CODE", None, None
|
|
79
104
|
)
|
|
105
|
+
return self.IMPLEMENTATION_CODE_NOT_UPDATED, None
|
|
80
106
|
|
|
81
|
-
|
|
82
|
-
conflicting_frid = render_context.conformance_tests_running_context.conflicting_frid
|
|
83
|
-
current_testing_module_name = render_context.conformance_tests_running_context.current_testing_module_name
|
|
84
|
-
current_testing_frid = render_context.conformance_tests_running_context.current_testing_frid
|
|
107
|
+
# ---------------------------------------------------------------------------------------------------------
|
|
85
108
|
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
109
|
+
def _apply_outcome(
|
|
110
|
+
self,
|
|
111
|
+
render_context: RenderContext,
|
|
112
|
+
conversation: ConformanceFixConversation,
|
|
113
|
+
args: dict,
|
|
114
|
+
changed_files: dict[str, Optional[str]],
|
|
115
|
+
):
|
|
116
|
+
"""Record the fix (the fixer's submit_fix `args` and the files it changed) and pick the next step: the unit
|
|
117
|
+
tests when the implementation code changed, otherwise the conformance tests right away. A conflict confirmed
|
|
118
|
+
a second time ends the render."""
|
|
119
|
+
ctx = render_context.conformance_tests_running_context
|
|
120
|
+
build_folder = conversation.build_folder
|
|
121
|
+
implementation_files = {path for path in changed_files if path.startswith(build_folder + os.sep)}
|
|
122
|
+
code_diff = conversation.diff(changed_files)
|
|
123
|
+
implementation_code_diff = {
|
|
124
|
+
name: diff for name, diff in code_diff.items() if os.path.join(build_folder, name) in implementation_files
|
|
125
|
+
}
|
|
126
|
+
summary = {"hypothesis": args.get("root_cause"), "approach": args.get("changes_made")}
|
|
89
127
|
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
render_context.plain_source_tree,
|
|
93
|
-
tmp_resources_list,
|
|
94
|
-
None,
|
|
95
|
-
False,
|
|
96
|
-
render_context.frid_context.frid,
|
|
97
|
-
)
|
|
98
|
-
console.print_resources(tmp_resources_list, render_context.frid_context.linked_resources)
|
|
128
|
+
if args.get("failing_reason") == CONFLICTING_REQUIREMENTS:
|
|
129
|
+
self._handle_conflict(render_context, args)
|
|
99
130
|
|
|
100
131
|
console.print_files(
|
|
101
|
-
"
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
style=console.
|
|
132
|
+
"Files changed while fixing conformance tests:",
|
|
133
|
+
build_folder,
|
|
134
|
+
{conversation.display_name(path): "" for path in sorted(changed_files)},
|
|
135
|
+
style=console.OUTPUT_STYLE,
|
|
136
|
+
)
|
|
137
|
+
target = "IMPLEMENTATION_CODE" if implementation_files else "CONFORMANCE_TESTS"
|
|
138
|
+
render_context.memory_manager.journal.record_fix(
|
|
139
|
+
ctx.current_testing_module_name,
|
|
140
|
+
ctx.current_testing_frid,
|
|
141
|
+
summary,
|
|
142
|
+
sorted(conversation.display_name(path) for path in changed_files),
|
|
143
|
+
target,
|
|
144
|
+
code_diff,
|
|
145
|
+
None,
|
|
105
146
|
)
|
|
106
147
|
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
148
|
+
if not implementation_files:
|
|
149
|
+
return self.IMPLEMENTATION_CODE_NOT_UPDATED, None
|
|
150
|
+
|
|
151
|
+
ctx.should_prepare_testing_environment = True
|
|
152
|
+
ctx.test_that_triggered_code_change = (ctx.current_testing_module_name, ctx.current_testing_frid)
|
|
153
|
+
ctx.execution_phase = TestExecutionPhase.RETRYING_AFTER_CODE_CHANGE
|
|
154
|
+
# Remember the change so that the unit tests fixer (which runs next) preserves it instead of reverting
|
|
155
|
+
# it - see FixUnitTests.
|
|
156
|
+
ctx.implementation_code_fixes.append(
|
|
157
|
+
{
|
|
158
|
+
**summary,
|
|
159
|
+
"code_diff": implementation_code_diff,
|
|
160
|
+
"tested": (ctx.current_testing_module_name, str(ctx.current_testing_frid)),
|
|
161
|
+
}
|
|
112
162
|
)
|
|
163
|
+
return self.IMPLEMENTATION_CODE_UPDATED, None
|
|
113
164
|
|
|
114
|
-
|
|
115
|
-
|
|
165
|
+
def _handle_conflict(self, render_context: RenderContext, args: dict) -> None:
|
|
166
|
+
ctx = render_context.conformance_tests_running_context
|
|
167
|
+
failing_reason = (
|
|
168
|
+
"CONFLICTING_REQUIREMENTS" if self._is_regression(render_context) else "CONFLICTING_ACCEPTANCE_TESTS"
|
|
169
|
+
)
|
|
170
|
+
conflicting_module, conflicting_frid = ctx.current_testing_module_name, str(ctx.current_testing_frid)
|
|
171
|
+
other_tested = self._other_functionality_of_conflict(render_context)
|
|
172
|
+
if other_tested is not None:
|
|
173
|
+
# The functionality has no acceptance tests to conflict with: its tests conflict with the functionality
|
|
174
|
+
# whose tests an earlier fix of this phase made pass (see _other_functionalities_fixes).
|
|
175
|
+
failing_reason = "CONFLICTING_REQUIREMENTS"
|
|
176
|
+
conflicting_module, conflicting_frid = other_tested
|
|
177
|
+
if ctx.conflicting_requirement_count >= 1:
|
|
178
|
+
# Declared before, a fix did not resolve it, and the reviewer agreed again: the server reports it as
|
|
179
|
+
# the ConflictingRequirements error, which ends the render.
|
|
180
|
+
render_context.codeplain_api.report_conflicting_requirements(
|
|
116
181
|
render_context.frid_context.frid,
|
|
117
|
-
|
|
182
|
+
conflicting_frid,
|
|
118
183
|
render_context.plain_source_tree,
|
|
119
|
-
render_context.frid_context.linked_resources,
|
|
120
|
-
existing_files_content,
|
|
121
|
-
memory_files_content,
|
|
122
184
|
render_context.module_name,
|
|
123
|
-
|
|
185
|
+
conflicting_module,
|
|
124
186
|
render_context.get_required_modules_functionalities(),
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
previous_conformance_tests_issue,
|
|
128
|
-
render_context.conformance_tests_running_context.fix_attempts,
|
|
129
|
-
render_context.conformance_tests_running_context.get_current_conformance_test_folder_name(),
|
|
130
|
-
render_context.conformance_tests_running_context.conflicting_requirement_count,
|
|
131
|
-
journal.build_digest(ctx.current_testing_module_name, ctx.current_testing_frid),
|
|
187
|
+
failing_reason,
|
|
188
|
+
args.get("conflict_reason", ""),
|
|
132
189
|
run_state=render_context.run_state,
|
|
133
190
|
)
|
|
191
|
+
raise InternalClientError("The server did not report the confirmed conflict.")
|
|
192
|
+
ctx.conflicting_requirement_count += 1
|
|
193
|
+
ctx.conflicting_module_name = ctx.current_testing_module_name
|
|
194
|
+
ctx.conflicting_frid = ctx.current_testing_frid
|
|
195
|
+
console.info(
|
|
196
|
+
f"↻ Potential conflicting functionalities detected while fixing conformance tests "
|
|
197
|
+
f"for functionality {render_context.testing_functionality_label()}.",
|
|
198
|
+
color=RETRY_COLOR,
|
|
134
199
|
)
|
|
135
|
-
code_diff_files_content = {}
|
|
136
200
|
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
201
|
+
@staticmethod
|
|
202
|
+
def _other_functionality_of_conflict(render_context: RenderContext) -> Optional[tuple[str, str]]:
|
|
203
|
+
"""The functionality a conflict declared for the current functionality's own tests is with, when that
|
|
204
|
+
functionality has no acceptance tests: the one whose tests the most recent fix of this phase for other tests
|
|
205
|
+
made pass. None for regression tests, with acceptance tests, or without such a fix."""
|
|
206
|
+
if FixConformanceTest._is_regression(render_context):
|
|
207
|
+
return None
|
|
208
|
+
specifications, _ = plain_spec.get_specifications_for_frid(
|
|
209
|
+
render_context.plain_source_tree, render_context.frid_context.frid
|
|
210
|
+
)
|
|
211
|
+
if specifications.get(plain_spec.ACCEPTANCE_TESTS):
|
|
212
|
+
return None
|
|
213
|
+
ctx = render_context.conformance_tests_running_context
|
|
214
|
+
tested = (ctx.current_testing_module_name, str(ctx.current_testing_frid))
|
|
215
|
+
for fix in reversed(ctx.implementation_code_fixes):
|
|
216
|
+
if fix.get("tested") and tuple(fix["tested"]) != tested:
|
|
217
|
+
module_name, frid = fix["tested"]
|
|
218
|
+
return module_name, frid
|
|
219
|
+
return None
|
|
220
|
+
|
|
221
|
+
# ---------------------------------------------------------------------------------------------------------
|
|
222
|
+
|
|
223
|
+
@staticmethod
|
|
224
|
+
def _agents_for_current_tests(render_context: RenderContext) -> ConformanceFixAgents:
|
|
225
|
+
"""The fixer and reviewer of the failing tests; new ones when a different functionality's tests fail."""
|
|
226
|
+
ctx = render_context.conformance_tests_running_context
|
|
227
|
+
tested = (ctx.current_testing_module_name, str(ctx.current_testing_frid))
|
|
228
|
+
if ctx.conformance_fix_agents is None or ctx.conformance_fix_agents.tested != tested:
|
|
229
|
+
ctx.conformance_fix_agents = ConformanceFixAgents(
|
|
230
|
+
tested=tested,
|
|
231
|
+
conformance_tests_folder=FixConformanceTest._writable_conformance_tests_folder(render_context),
|
|
232
|
+
readable_tests_folders=FixConformanceTest._readable_conformance_tests_folders(render_context),
|
|
233
|
+
plain_modules_folder=agent_tools.rendered_modules_folder(render_context),
|
|
234
|
+
writable_module_tests_folder=(
|
|
235
|
+
os.path.normpath(
|
|
236
|
+
os.path.abspath(
|
|
237
|
+
render_context.conformance_tests.get_module_conformance_tests_folder(
|
|
238
|
+
render_context.module_name
|
|
239
|
+
)
|
|
240
|
+
)
|
|
241
|
+
)
|
|
242
|
+
if FixConformanceTest._is_regression(render_context)
|
|
243
|
+
else ""
|
|
244
|
+
),
|
|
148
245
|
)
|
|
246
|
+
if FixConformanceTest._is_regression(render_context):
|
|
247
|
+
ctx.conformance_fix_agents.other_tests_folders = FixConformanceTest._other_tests_folders(
|
|
248
|
+
render_context, ctx.conformance_fix_agents.conformance_tests_folder
|
|
249
|
+
)
|
|
250
|
+
return ctx.conformance_fix_agents
|
|
149
251
|
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
252
|
+
@staticmethod
|
|
253
|
+
def _readable_conformance_tests_folders(render_context: RenderContext) -> list[str]:
|
|
254
|
+
"""The existing conformance tests folders of the current module and of every module it requires that lie
|
|
255
|
+
outside the rendered modules folder (those inside it are readable with it)."""
|
|
256
|
+
root = agent_tools.rendered_modules_folder(render_context)
|
|
257
|
+
module_names = [render_context.module_name] + [
|
|
258
|
+
module.module_name for module in render_context.plain_module.all_required_modules
|
|
259
|
+
]
|
|
260
|
+
folders: list[str] = []
|
|
261
|
+
for module_name in module_names:
|
|
262
|
+
folder = os.path.normpath(
|
|
263
|
+
os.path.abspath(render_context.conformance_tests.get_module_conformance_tests_folder(module_name))
|
|
158
264
|
)
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
265
|
+
if os.path.isdir(folder) and folder not in folders and not folder.startswith(root + os.sep):
|
|
266
|
+
folders.append(folder)
|
|
267
|
+
return folders
|
|
268
|
+
|
|
269
|
+
@staticmethod
|
|
270
|
+
def _writable_conformance_tests_folder(render_context: RenderContext) -> str:
|
|
271
|
+
"""The folder of the failing conformance tests. A required module's tests are fixed in their copy inside
|
|
272
|
+
the current module (made here if missing), which is the copy the conformance tests then run from. The copies
|
|
273
|
+
of that module's other functionalities' tests are made too, so a fix of a cause they share can reach them
|
|
274
|
+
(they run from their copies later)."""
|
|
275
|
+
ctx = render_context.conformance_tests_running_context
|
|
276
|
+
folder = ctx.get_current_conformance_test_folder_name()
|
|
277
|
+
if render_context.module_name == ctx.current_testing_module_name:
|
|
278
|
+
return os.path.normpath(os.path.abspath(folder))
|
|
279
|
+
testing_module = ctx.current_testing_module_name
|
|
280
|
+
other_folders = [
|
|
281
|
+
entry.get("folder_name")
|
|
282
|
+
for entry in render_context.conformance_tests.get_conformance_tests_json(testing_module).values()
|
|
283
|
+
if entry.get("folder_name")
|
|
284
|
+
]
|
|
285
|
+
writable = folder
|
|
286
|
+
for tests_folder in [folder, *other_folders]:
|
|
287
|
+
source, copy = render_context.conformance_tests.get_source_conformance_test_folder_name(
|
|
288
|
+
render_context.module_name, render_context.required_modules, testing_module, tests_folder
|
|
168
289
|
)
|
|
290
|
+
if not os.path.exists(copy) and os.path.isdir(source):
|
|
291
|
+
file_utils.copy_folder_content(source, copy)
|
|
292
|
+
if tests_folder == folder:
|
|
293
|
+
writable = copy
|
|
294
|
+
return os.path.normpath(os.path.abspath(writable))
|
|
169
295
|
|
|
170
|
-
|
|
296
|
+
@staticmethod
|
|
297
|
+
def _other_tests_folders(render_context: RenderContext, tests_folder: str) -> list[str]:
|
|
298
|
+
"""The existing folders of the other functionalities' conformance tests the failing tests' module has, where
|
|
299
|
+
a regression fix runs them from: the current module's own folders, or a required module's copies in it."""
|
|
300
|
+
ctx = render_context.conformance_tests_running_context
|
|
301
|
+
testing_module = ctx.current_testing_module_name
|
|
302
|
+
if testing_module == render_context.module_name:
|
|
303
|
+
folders = [entry.get("folder_name") for entry in ctx.get_conformance_tests_json(testing_module).values()]
|
|
171
304
|
else:
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
render_context.conformance_tests_running_context.should_prepare_testing_environment = True
|
|
182
|
-
|
|
183
|
-
# Record which test triggered the change and transition to retry phase
|
|
184
|
-
ctx = render_context.conformance_tests_running_context
|
|
185
|
-
ctx.test_that_triggered_code_change = (ctx.current_testing_module_name, ctx.current_testing_frid)
|
|
186
|
-
ctx.execution_phase = TestExecutionPhase.RETRYING_AFTER_CODE_CHANGE
|
|
187
|
-
|
|
188
|
-
# Remember the change so that the unit tests fixer (which runs next) preserves it instead of
|
|
189
|
-
# reverting it - see FixUnitTests.
|
|
190
|
-
summary = fix_attempt_summary if isinstance(fix_attempt_summary, dict) else {}
|
|
191
|
-
ctx.implementation_code_fixes.append(
|
|
192
|
-
{
|
|
193
|
-
"hypothesis": summary.get("hypothesis"),
|
|
194
|
-
"approach": summary.get("approach"),
|
|
195
|
-
"code_diff": code_diff_files_content,
|
|
196
|
-
}
|
|
197
|
-
)
|
|
305
|
+
folders = [
|
|
306
|
+
render_context.conformance_tests.get_source_conformance_test_folder_name(
|
|
307
|
+
render_context.module_name, render_context.required_modules, testing_module, entry["folder_name"]
|
|
308
|
+
)[1]
|
|
309
|
+
for entry in render_context.conformance_tests.get_conformance_tests_json(testing_module).values()
|
|
310
|
+
if entry.get("folder_name")
|
|
311
|
+
]
|
|
312
|
+
normalized = [os.path.normpath(os.path.abspath(f)) for f in folders if f]
|
|
313
|
+
return sorted({f for f in normalized if os.path.isdir(f) and f != tests_folder})
|
|
198
314
|
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
315
|
+
@staticmethod
|
|
316
|
+
def _is_regression(render_context: RenderContext) -> bool:
|
|
317
|
+
"""Whether the failing tests belong to a functionality implemented before the current one."""
|
|
318
|
+
ctx = render_context.conformance_tests_running_context
|
|
319
|
+
return (
|
|
320
|
+
ctx.current_testing_module_name != render_context.module_name
|
|
321
|
+
or ctx.current_testing_frid != render_context.frid_context.frid
|
|
322
|
+
)
|
|
323
|
+
|
|
324
|
+
@staticmethod
|
|
325
|
+
def _common_params(render_context: RenderContext, agents: ConformanceFixAgents) -> dict:
|
|
326
|
+
"""The task params both agents start a session with (besides the issue)."""
|
|
327
|
+
ctx = render_context.conformance_tests_running_context
|
|
328
|
+
frid = render_context.frid_context.frid
|
|
329
|
+
specifications, _ = plain_spec.get_specifications_for_frid(render_context.plain_source_tree, frid)
|
|
330
|
+
tested_text, tested_scenarios = FixConformanceTest._tested_functionality(
|
|
331
|
+
ctx.current_testing_frid_specifications
|
|
332
|
+
)
|
|
333
|
+
is_regression = FixConformanceTest._is_regression(render_context)
|
|
334
|
+
_, memory_files = MemoryManager.fetch_memory_files(render_context.memory_manager.memory_folder)
|
|
335
|
+
params = {
|
|
336
|
+
"definitions": "\n".join(specifications.get(plain_spec.DEFINITIONS, [])),
|
|
337
|
+
"non_functional_requirements": "\n".join(specifications.get(plain_spec.NON_FUNCTIONAL_REQUIREMENTS, [])),
|
|
338
|
+
"test_requirements": "\n".join(specifications.get(plain_spec.TEST_REQUIREMENTS, [])),
|
|
339
|
+
"functional_requirements": FixUnitTests._functional_requirements_section(render_context, specifications),
|
|
340
|
+
"linked_resources": render_context.frid_context.linked_resources,
|
|
341
|
+
"module_name": render_context.module_name,
|
|
342
|
+
"build_folder": os.path.normpath(os.path.abspath(render_context.build_folder)),
|
|
343
|
+
"conformance_tests_folder": agents.conformance_tests_folder,
|
|
344
|
+
**agent_tools.modules_layout_params(render_context),
|
|
345
|
+
"writable_module_tests_folder": agents.writable_module_tests_folder,
|
|
346
|
+
"other_conformance_tests_folders": agents.other_tests_folders,
|
|
347
|
+
"readable_conformance_tests_folders": agents.readable_tests_folders,
|
|
348
|
+
"conformance_tests_script_content": FixUnitTests._read_script(render_context.conformance_tests_script),
|
|
349
|
+
"prepare_environment_script_content": FixUnitTests._read_script(render_context.prepare_environment_script),
|
|
350
|
+
"tested_functionality_id": qualified_frid(ctx.current_testing_module_name, ctx.current_testing_frid),
|
|
351
|
+
"tested_functionality": tested_text,
|
|
352
|
+
"is_regression": is_regression,
|
|
353
|
+
"conformance_scenarios": _stringify_scenarios(tested_scenarios),
|
|
354
|
+
"acceptance_tests": _stringify_scenarios(specifications.get(plain_spec.ACCEPTANCE_TESTS, [])),
|
|
355
|
+
"memory_files": memory_files,
|
|
356
|
+
"conformance_tests_files": _folder_files(agents.conformance_tests_folder),
|
|
357
|
+
}
|
|
358
|
+
if is_regression:
|
|
359
|
+
params["code_diff"] = ImplementationCodeHelpers.get_code_diff(
|
|
360
|
+
render_context.build_folder, render_context.plain_source_tree, frid
|
|
361
|
+
)
|
|
362
|
+
return params
|
|
363
|
+
|
|
364
|
+
@staticmethod
|
|
365
|
+
def _fixer_params(render_context: RenderContext, common_params: dict) -> dict:
|
|
366
|
+
ctx = render_context.conformance_tests_running_context
|
|
367
|
+
params = dict(common_params)
|
|
368
|
+
params["file_tree"] = FixUnitTests._file_tree(render_context.build_folder)
|
|
369
|
+
params["relevant_files"] = FixUnitTests._relevant_files(
|
|
370
|
+
render_context.build_folder, set(render_context.frid_context.changed_files)
|
|
371
|
+
)
|
|
372
|
+
if ctx.conflicting_requirement_count >= 1:
|
|
373
|
+
params["conflict_already_reported"] = True
|
|
374
|
+
other_fixes = FixConformanceTest._other_functionalities_fixes(render_context)
|
|
375
|
+
if other_fixes:
|
|
376
|
+
params["other_functionalities_fixes"] = other_fixes
|
|
377
|
+
return params
|
|
378
|
+
|
|
379
|
+
@staticmethod
|
|
380
|
+
def _other_functionalities_fixes(render_context: RenderContext) -> list[dict]:
|
|
381
|
+
"""The implementation changes made earlier in this conformance phase to make another functionality's tests
|
|
382
|
+
pass, oldest first. A fixer that undoes one makes those tests fail again, and the two fixes then alternate,
|
|
383
|
+
so the fixer is shown them and told to declare the conflict instead."""
|
|
384
|
+
ctx = render_context.conformance_tests_running_context
|
|
385
|
+
tested = (ctx.current_testing_module_name, str(ctx.current_testing_frid))
|
|
386
|
+
return [
|
|
387
|
+
{
|
|
388
|
+
"functionality": qualified_frid(*fix["tested"]),
|
|
389
|
+
"hypothesis": fix.get("hypothesis"),
|
|
390
|
+
"approach": fix.get("approach"),
|
|
391
|
+
"code_diff": fix.get("code_diff") or {},
|
|
392
|
+
}
|
|
393
|
+
for fix in ctx.implementation_code_fixes
|
|
394
|
+
if fix.get("tested") and tuple(fix["tested"]) != tested
|
|
395
|
+
]
|
|
396
|
+
|
|
397
|
+
@staticmethod
|
|
398
|
+
def _tested_functionality(specifications: Optional[dict | str]) -> tuple[str, list]:
|
|
399
|
+
"""The text and the conformance scenarios of the functionality whose tests fail. A required module's
|
|
400
|
+
functionality is known only by its text."""
|
|
401
|
+
if isinstance(specifications, str):
|
|
402
|
+
return specifications, []
|
|
403
|
+
specifications = specifications or {}
|
|
404
|
+
functional_requirements = specifications.get(plain_spec.FUNCTIONAL_REQUIREMENTS) or [""]
|
|
405
|
+
return functional_requirements[-1], specifications.get(plain_spec.CONFORMANCE_TESTS) or []
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
def _stringify_scenarios(scenarios: list) -> str:
|
|
409
|
+
return "\n\n".join(f"#### Scenario `{scenario['id']}`\n\n{scenario['text']}" for scenario in scenarios or [])
|
|
208
410
|
|
|
209
|
-
return self.IMPLEMENTATION_CODE_UPDATED, None
|
|
210
|
-
else:
|
|
211
|
-
journal.record_fix(
|
|
212
|
-
ctx.current_testing_module_name,
|
|
213
|
-
ctx.current_testing_frid,
|
|
214
|
-
fix_attempt_summary,
|
|
215
|
-
[],
|
|
216
|
-
"IMPLEMENTATION_CODE",
|
|
217
|
-
None,
|
|
218
|
-
prepared_conformance_tests_issue,
|
|
219
|
-
)
|
|
220
411
|
|
|
221
|
-
|
|
412
|
+
def _folder_files(folder: str) -> dict[str, str]:
|
|
413
|
+
"""The text files of `folder` by full path, smallest first, within MAX_CONFORMANCE_TESTS_FILES_CHARS."""
|
|
414
|
+
if not os.path.isdir(folder):
|
|
415
|
+
return {}
|
|
416
|
+
contents = file_utils.get_existing_files_content(folder, file_utils.list_all_text_files(folder))
|
|
417
|
+
selected, total = {}, 0
|
|
418
|
+
for name in sorted(contents, key=lambda n: (len(contents[n]), n)):
|
|
419
|
+
if total + len(contents[name]) > MAX_CONFORMANCE_TESTS_FILES_CHARS:
|
|
420
|
+
break
|
|
421
|
+
selected[os.path.join(folder, name)] = contents[name]
|
|
422
|
+
total += len(contents[name])
|
|
423
|
+
return dict(sorted(selected.items()))
|
|
@@ -185,6 +185,7 @@ class FixUnitTests(BaseAction):
|
|
|
185
185
|
"linked_resources": render_context.frid_context.linked_resources,
|
|
186
186
|
"build_folder": render_context.build_folder,
|
|
187
187
|
"module_name": render_context.module_name,
|
|
188
|
+
**agent_tools.modules_layout_params(render_context),
|
|
188
189
|
"unittests_script_content": FixUnitTests._read_script(render_context.unittests_script),
|
|
189
190
|
"unittests_issue": unittests_issue,
|
|
190
191
|
"unittests_log_path": render_context.script_execution_history.latest_unit_test_output_path,
|