codeplain 0.3.13.dev49__py3-none-any.whl → 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- change_detection.py +65 -94
- cli_output/dry_run.py +7 -3
- {codeplain-0.3.13.dev49.dist-info → codeplain-0.4.0.dist-info}/METADATA +1 -1
- {codeplain-0.3.13.dev49.dist-info → codeplain-0.4.0.dist-info}/RECORD +49 -43
- codeplain_REST_api.py +9 -14
- docs/generate_cli.py +3 -2
- partial_rendering.py +32 -2
- plain2code.py +42 -14
- plain2code_arguments.py +69 -12
- plain2code_telemetry.py +2 -1
- plain2code_utils.py +14 -0
- plain_modules.py +73 -14
- render_machine/actions/analyze_specification_ambiguity.py +5 -3
- render_machine/actions/distill_conformance_test_memory.py +3 -1
- render_machine/actions/fix_conformance_test.py +4 -4
- render_machine/actions/fix_unit_tests.py +3 -1
- render_machine/actions/refactor_code.py +6 -1
- render_machine/actions/render_conformance_tests.py +52 -20
- render_machine/actions/render_functional_requirement.py +5 -2
- render_machine/actions/run_conformance_tests.py +3 -2
- render_machine/actions/summarize_conformance_tests.py +3 -1
- render_machine/render_context.py +59 -20
- render_machine/render_types.py +6 -15
- standard_template_library/typescript-react-app-boilerplate.plain +2 -1
- tests/test_acceptance_scenario_rendered_once.py +169 -0
- tests/test_acceptance_scenarios_after_a_code_fix.py +110 -0
- tests/test_change_detection.py +138 -127
- tests/test_conformance_fix_journal.py +46 -46
- tests/test_dry_run_link_validation.py +2 -1
- tests/test_fix_unit_tests_action.py +1 -1
- tests/test_frid_arguments.py +40 -0
- tests/test_functionality_without_scenarios.py +88 -0
- tests/test_git_utils.py +13 -13
- tests/test_partial_rendering.py +82 -47
- tests/test_plain2code.py +39 -41
- tests/test_plain2code_utils.py +12 -1
- tests/test_plain_modules.py +50 -44
- tests/test_render_finished_report.py +2 -0
- tests/test_render_range.py +18 -9
- tests/test_render_stop_reason.py +5 -5
- tests/test_telemetry.py +2 -2
- tests/test_terminal_attachment.py +2 -1
- tests/test_tests_folders.py +91 -0
- tests/test_traceability_warnings.py +69 -0
- tui/plain_module_render_choice_tui.py +4 -6
- tui/state_handlers.py +19 -28
- {codeplain-0.3.13.dev49.dist-info → codeplain-0.4.0.dist-info}/WHEEL +0 -0
- {codeplain-0.3.13.dev49.dist-info → codeplain-0.4.0.dist-info}/entry_points.txt +0 -0
- {codeplain-0.3.13.dev49.dist-info → codeplain-0.4.0.dist-info}/licenses/LICENSE +0 -0
|
@@ -109,9 +109,7 @@ class RenderConformanceTests(BaseAction):
|
|
|
109
109
|
style=console.INPUT_STYLE,
|
|
110
110
|
)
|
|
111
111
|
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
response_files, implementation_plan_summary = render_context.codeplain_api.render_conformance_tests(
|
|
112
|
+
response_files, traceability = render_context.codeplain_api.render_conformance_tests(
|
|
115
113
|
render_context.frid_context.frid,
|
|
116
114
|
render_context.conformance_tests_running_context.current_testing_frid,
|
|
117
115
|
render_context.plain_source_tree,
|
|
@@ -124,13 +122,10 @@ class RenderConformanceTests(BaseAction):
|
|
|
124
122
|
render_context.conformance_tests_running_context.get_conformance_tests_json(
|
|
125
123
|
render_context.conformance_tests_running_context.current_testing_module_name
|
|
126
124
|
),
|
|
127
|
-
all_acceptance_tests,
|
|
128
125
|
run_state=render_context.run_state,
|
|
129
126
|
)
|
|
130
127
|
|
|
131
|
-
render_context
|
|
132
|
-
implementation_plan_summary
|
|
133
|
-
)
|
|
128
|
+
self._warn_about_untraced_scenarios(render_context, traceability)
|
|
134
129
|
|
|
135
130
|
file_utils.store_response_files(conformance_tests_folder_name, response_files, [])
|
|
136
131
|
|
|
@@ -143,11 +138,51 @@ class RenderConformanceTests(BaseAction):
|
|
|
143
138
|
|
|
144
139
|
return self.SUCCESSFUL_OUTCOME, None
|
|
145
140
|
|
|
141
|
+
@staticmethod
|
|
142
|
+
def _warn_about_untraced_scenarios(render_context: RenderContext, traceability: dict) -> None:
|
|
143
|
+
"""Warn about a scenario with no test, a test that implements no scenario, and a scenario
|
|
144
|
+
with more than one test.
|
|
145
|
+
|
|
146
|
+
Both test renders report the same traceability, so both are checked the same way. A render
|
|
147
|
+
that writes a test the specification did not ask for, or leaves one it did ask for unwritten,
|
|
148
|
+
must say so instead of reporting success.
|
|
149
|
+
"""
|
|
150
|
+
frid = render_context.conformance_tests_running_context.current_testing_frid
|
|
151
|
+
|
|
152
|
+
missing = traceability.get("missing") or []
|
|
153
|
+
if missing:
|
|
154
|
+
console.warning(f"No conformance test was generated for {frid} scenarios: {', '.join(missing)}.")
|
|
155
|
+
|
|
156
|
+
extra = traceability.get("extra") or []
|
|
157
|
+
if extra:
|
|
158
|
+
console.warning(f"Conformance tests of {frid} that implement no scenario: {', '.join(extra)}.")
|
|
159
|
+
|
|
160
|
+
duplicated = traceability.get("duplicated") or []
|
|
161
|
+
if duplicated:
|
|
162
|
+
console.warning(
|
|
163
|
+
f"More than one conformance test of {frid} implements the scenarios: {', '.join(duplicated)}."
|
|
164
|
+
)
|
|
165
|
+
|
|
146
166
|
def _render_acceptance_test(self, render_context: RenderContext):
|
|
147
167
|
if plain_spec.ACCEPTANCE_TESTS not in render_context.frid_context.specifications:
|
|
148
168
|
# If there are no acceptance tests defined, continue.
|
|
149
169
|
return self.SUCCESSFUL_OUTCOME, None
|
|
150
170
|
|
|
171
|
+
ctx = render_context.conformance_tests_running_context
|
|
172
|
+
|
|
173
|
+
# Get the current acceptance scenario being rendered (completed count is 1-based)
|
|
174
|
+
scenario = render_context.frid_context.specifications[plain_spec.ACCEPTANCE_TESTS][
|
|
175
|
+
ctx.acceptance_tests_completed - 1
|
|
176
|
+
]
|
|
177
|
+
|
|
178
|
+
# This action is entered through more than one path, and the counter above belongs to the
|
|
179
|
+
# state machine, not to the action. A path that enters it again without the counter moving
|
|
180
|
+
# asks for a scenario that already has a test. Rendering it again costs a call and lets the
|
|
181
|
+
# render write a second test for the one scenario.
|
|
182
|
+
if scenario["id"] in ctx.rendered_acceptance_scenarios:
|
|
183
|
+
console.warning(f"The acceptance scenario {scenario['id']} already has a test. It is not rendered again.")
|
|
184
|
+
return self.SUCCESSFUL_OUTCOME, None
|
|
185
|
+
|
|
151
186
|
_, existing_files_content = ImplementationCodeHelpers.fetch_existing_files(render_context.build_folder)
|
|
152
187
|
_, memory_files_content = MemoryManager.fetch_memory_files(render_context.memory_manager.memory_folder)
|
|
153
188
|
(
|
|
@@ -156,18 +191,13 @@ class RenderConformanceTests(BaseAction):
|
|
|
156
191
|
) = render_context.conformance_tests.fetch_existing_conformance_test_files(
|
|
157
192
|
render_context.module_name,
|
|
158
193
|
render_context.required_modules,
|
|
159
|
-
|
|
160
|
-
|
|
194
|
+
ctx.current_testing_module_name,
|
|
195
|
+
ctx.get_current_conformance_test_folder_name(),
|
|
161
196
|
)
|
|
162
197
|
|
|
163
|
-
|
|
164
|
-
acceptance_test = render_context.frid_context.specifications[plain_spec.ACCEPTANCE_TESTS][
|
|
165
|
-
render_context.conformance_tests_running_context.acceptance_tests_completed - 1
|
|
166
|
-
]
|
|
167
|
-
|
|
168
|
-
console.info(f"Generating acceptance test:\n {acceptance_test}")
|
|
198
|
+
console.info(f"Generating acceptance test for scenario {scenario['id']}.")
|
|
169
199
|
|
|
170
|
-
response_files = render_context.codeplain_api.render_acceptance_tests(
|
|
200
|
+
response_files, traceability = render_context.codeplain_api.render_acceptance_tests(
|
|
171
201
|
render_context.frid_context.frid,
|
|
172
202
|
render_context.plain_source_tree,
|
|
173
203
|
render_context.frid_context.linked_resources,
|
|
@@ -176,12 +206,14 @@ class RenderConformanceTests(BaseAction):
|
|
|
176
206
|
conformance_tests_files_content,
|
|
177
207
|
render_context.module_name,
|
|
178
208
|
render_context.get_required_modules_functionalities(),
|
|
179
|
-
|
|
209
|
+
scenario["id"],
|
|
180
210
|
run_state=render_context.run_state,
|
|
181
211
|
)
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
)
|
|
212
|
+
|
|
213
|
+
self._warn_about_untraced_scenarios(render_context, traceability)
|
|
214
|
+
ctx.rendered_acceptance_scenarios.append(scenario["id"])
|
|
215
|
+
|
|
216
|
+
conformance_tests_folder_name = ctx.get_current_conformance_test_folder_name()
|
|
185
217
|
|
|
186
218
|
file_utils.store_response_files(conformance_tests_folder_name, response_files, conformance_tests_files)
|
|
187
219
|
console.print_files(
|
|
@@ -5,6 +5,7 @@ import render_machine.render_utils as render_utils
|
|
|
5
5
|
from memory_management import MemoryManager
|
|
6
6
|
from plain2code_console import RETRY_COLOR, console
|
|
7
7
|
from plain2code_exceptions import FunctionalRequirementTooComplex
|
|
8
|
+
from plain2code_utils import qualified_frid
|
|
8
9
|
from render_machine.actions.base_action import BaseAction
|
|
9
10
|
from render_machine.implementation_code_helpers import ImplementationCodeHelpers
|
|
10
11
|
from render_machine.render_context import MAX_UNITTEST_FIX_ATTEMPTS, RenderContext
|
|
@@ -21,8 +22,9 @@ class RenderFunctionalRequirement(BaseAction):
|
|
|
21
22
|
def execute(self, render_context: RenderContext, _previous_action_payload: Any | None):
|
|
22
23
|
if render_context.frid_context.functional_requirement_render_attempts >= MAX_CODE_GENERATION_RETRIES:
|
|
23
24
|
# A render attempt is only restarted once its unit tests used up every fix round.
|
|
25
|
+
functionality = qualified_frid(render_context.module_name, render_context.frid_context.frid)
|
|
24
26
|
error_msg = (
|
|
25
|
-
f"Functionality {
|
|
27
|
+
f"Functionality {functionality}: unit tests still fail after "
|
|
26
28
|
f"{MAX_CODE_GENERATION_RETRIES} render attempts, {MAX_UNITTEST_FIX_ATTEMPTS} fix rounds each."
|
|
27
29
|
)
|
|
28
30
|
last_output_path = render_context.script_execution_history.latest_unit_test_output_path
|
|
@@ -36,7 +38,8 @@ class RenderFunctionalRequirement(BaseAction):
|
|
|
36
38
|
if render_context.frid_context.functional_requirement_render_attempts > 1:
|
|
37
39
|
console.info(
|
|
38
40
|
f"↻ Unittests could not be fixed after rendering the functionality. "
|
|
39
|
-
f"Restarting rendering functionality
|
|
41
|
+
f"Restarting rendering functionality "
|
|
42
|
+
f"{qualified_frid(render_context.module_name, render_context.frid_context.frid)} from scratch.",
|
|
40
43
|
color=RETRY_COLOR,
|
|
41
44
|
)
|
|
42
45
|
|
|
@@ -3,6 +3,7 @@ from typing import Any
|
|
|
3
3
|
|
|
4
4
|
import render_machine.render_utils as render_utils
|
|
5
5
|
from plain2code_console import console
|
|
6
|
+
from plain2code_utils import qualified_frid
|
|
6
7
|
from render_machine.actions.base_action import BaseAction
|
|
7
8
|
from render_machine.render_context import RenderContext
|
|
8
9
|
from render_machine.render_types import RenderError
|
|
@@ -33,11 +34,11 @@ class RunConformanceTests(BaseAction):
|
|
|
33
34
|
)
|
|
34
35
|
)
|
|
35
36
|
|
|
37
|
+
ctx = render_context.conformance_tests_running_context
|
|
36
38
|
console.info(
|
|
37
39
|
f"Running conformance tests script {conformance_tests_script} "
|
|
38
40
|
+ f"for {conformance_tests_folder_name} ("
|
|
39
|
-
+ f"functionality {
|
|
40
|
-
+ f"in module {render_context.conformance_tests_running_context.current_testing_module_name}"
|
|
41
|
+
+ f"functionality {qualified_frid(ctx.current_testing_module_name, ctx.current_testing_frid)}"
|
|
41
42
|
+ ")."
|
|
42
43
|
)
|
|
43
44
|
exit_code, conformance_tests_issue, conformance_tests_temp_log_file_path = render_utils.execute_script(
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from typing import Any
|
|
2
2
|
|
|
3
3
|
from plain2code_console import console
|
|
4
|
+
from plain2code_utils import qualified_frid
|
|
4
5
|
from render_machine.actions.base_action import BaseAction
|
|
5
6
|
from render_machine.render_context import RenderContext
|
|
6
7
|
|
|
@@ -9,7 +10,8 @@ class SummarizeConformanceTests(BaseAction):
|
|
|
9
10
|
SUCCESSFUL_OUTCOME = "conformance_tests_summarized"
|
|
10
11
|
|
|
11
12
|
def execute(self, render_context: RenderContext, _previous_action_payload: Any | None):
|
|
12
|
-
|
|
13
|
+
functionality = qualified_frid(render_context.module_name, render_context.frid_context.frid)
|
|
14
|
+
console.info(f"Summarizing conformance tests for functionality {functionality}.")
|
|
13
15
|
|
|
14
16
|
_, existing_conformance_test_files_content = (
|
|
15
17
|
render_context.conformance_tests.fetch_existing_conformance_test_files(
|
render_machine/render_context.py
CHANGED
|
@@ -10,6 +10,7 @@ from event_bus import EventBus
|
|
|
10
10
|
from plain2code_console import RETRY_COLOR, console
|
|
11
11
|
from plain2code_events import RenderContextSnapshot
|
|
12
12
|
from plain2code_state import RunState
|
|
13
|
+
from plain2code_utils import qualified_frid
|
|
13
14
|
from plain_modules import PlainModule
|
|
14
15
|
from render_machine import triggers
|
|
15
16
|
from render_machine.conformance_tests import CONFORMANCE_TESTS_DEFINITION_FILE_NAME, ConformanceTests
|
|
@@ -176,7 +177,7 @@ class RenderContext:
|
|
|
176
177
|
if self.render_range is None or len(self.render_range) == 0:
|
|
177
178
|
return next_frid is not None
|
|
178
179
|
|
|
179
|
-
return next_frid is not None and
|
|
180
|
+
return next_frid is not None and next_frid in self.render_range
|
|
180
181
|
|
|
181
182
|
def finish_implementing_frid(self):
|
|
182
183
|
self.functional_requirements_render_attempts_failed_unit_during_conformance_tests = 0
|
|
@@ -297,13 +298,15 @@ class RenderContext:
|
|
|
297
298
|
self.functional_requirements_render_attempts_failed_unit_during_conformance_tests
|
|
298
299
|
>= MAX_FUNCTIONAL_REQUIREMENT_RENDER_ATTEMPTS_FAILED_UNIT_DURING_CONFORMANCE_TESTS
|
|
299
300
|
):
|
|
300
|
-
|
|
301
|
+
functionality = qualified_frid(self.module_name, self.frid_context.frid)
|
|
302
|
+
error_msg = f"Failed to adjust the unit tests after implementation code was update while fixing the conformance tests for functionality {functionality} for the {MAX_FUNCTIONAL_REQUIREMENT_RENDER_ATTEMPTS_FAILED_UNIT_DURING_CONFORMANCE_TESTS} times."
|
|
301
303
|
self.dispatch_error(error_msg)
|
|
302
304
|
else:
|
|
305
|
+
functionality = qualified_frid(self.module_name, self.frid_context.frid)
|
|
303
306
|
console.info(
|
|
304
307
|
f"↻ Failed to adjust the unit tests after implementation code was updated while fixing the "
|
|
305
|
-
f"conformance tests for functionality {
|
|
306
|
-
f"Restarting rendering the functionality {
|
|
308
|
+
f"conformance tests for functionality {functionality}. "
|
|
309
|
+
f"Restarting rendering the functionality {functionality} from scratch.",
|
|
307
310
|
color=RETRY_COLOR,
|
|
308
311
|
)
|
|
309
312
|
self.machine.dispatch(triggers.RESTART_FRID_PROCESSING)
|
|
@@ -313,10 +316,13 @@ class RenderContext:
|
|
|
313
316
|
self.machine.dispatch(triggers.START_NEW_REFACTORING_ITERATION)
|
|
314
317
|
|
|
315
318
|
def start_conformance_tests_processing(self):
|
|
316
|
-
console.info("Implementing conformance tests...")
|
|
317
319
|
current_frid_specifications, _ = plain_spec.get_specifications_for_frid(
|
|
318
320
|
self.plain_source_tree, self.frid_context.frid
|
|
319
321
|
)
|
|
322
|
+
has_own_scenarios = bool(current_frid_specifications.get(plain_spec.CONFORMANCE_TESTS))
|
|
323
|
+
if has_own_scenarios:
|
|
324
|
+
console.info("Implementing conformance tests...")
|
|
325
|
+
|
|
320
326
|
self.conformance_tests_running_context = ConformanceTestsRunningContext(
|
|
321
327
|
current_testing_module_name=self.module_name,
|
|
322
328
|
current_testing_frid=self.frid_context.frid,
|
|
@@ -328,6 +334,30 @@ class RenderContext:
|
|
|
328
334
|
frid_being_implemented=self.frid_context.frid,
|
|
329
335
|
)
|
|
330
336
|
|
|
337
|
+
if has_own_scenarios:
|
|
338
|
+
return
|
|
339
|
+
|
|
340
|
+
# No scenario was written for this functionality, so no conformance test is rendered or run
|
|
341
|
+
# for it. The tests of the previous functionalities still run, so a regression it causes is
|
|
342
|
+
# still caught.
|
|
343
|
+
functionality = qualified_frid(self.module_name, self.frid_context.frid)
|
|
344
|
+
if self._has_conformance_tests_to_run_in_regression():
|
|
345
|
+
console.info(
|
|
346
|
+
f"No conformance scenario is written for {functionality}. "
|
|
347
|
+
"Running the conformance tests of the previous functionalities."
|
|
348
|
+
)
|
|
349
|
+
self._start_regression_phase()
|
|
350
|
+
else:
|
|
351
|
+
console.info(f"No conformance scenario is written for {functionality}, and there is none to re-run.")
|
|
352
|
+
self.conformance_tests_running_context.execution_phase = TestExecutionPhase.COMPLETED
|
|
353
|
+
|
|
354
|
+
def _has_conformance_tests_to_run_in_regression(self) -> bool:
|
|
355
|
+
"""Whether any functionality rendered before this one has conformance tests to re-run."""
|
|
356
|
+
for module in self.required_modules or []:
|
|
357
|
+
if self.conformance_tests.get_conformance_tests_json(module.module_name):
|
|
358
|
+
return True
|
|
359
|
+
return bool(self.conformance_tests.get_conformance_tests_json(self.module_name))
|
|
360
|
+
|
|
331
361
|
def finish_conformance_tests_processing(self):
|
|
332
362
|
self.conformance_tests_running_context = None
|
|
333
363
|
|
|
@@ -402,7 +432,10 @@ class RenderContext:
|
|
|
402
432
|
"""Handle regeneration of conformance tests after too many failures."""
|
|
403
433
|
ctx = self.conformance_tests_running_context
|
|
404
434
|
|
|
405
|
-
console.info(
|
|
435
|
+
console.info(
|
|
436
|
+
f"Recreating conformance tests for functionality "
|
|
437
|
+
f"{qualified_frid(ctx.current_testing_module_name, ctx.current_testing_frid)}."
|
|
438
|
+
)
|
|
406
439
|
|
|
407
440
|
existing_folder = ctx.get_conformance_tests_json(ctx.current_testing_module_name).pop(ctx.current_testing_frid)
|
|
408
441
|
file_utils.delete_folder(existing_folder["folder_name"])
|
|
@@ -411,6 +444,12 @@ class RenderContext:
|
|
|
411
444
|
ctx.fix_attempts = 0
|
|
412
445
|
ctx.regenerating_conformance_tests = False
|
|
413
446
|
|
|
447
|
+
# The folder held the acceptance tests of the functionality as well, so they are gone with
|
|
448
|
+
# it. The functionality writes all of its tests again, from the first scenario.
|
|
449
|
+
ctx.acceptance_test_phase = AcceptanceTestPhase.NOT_STARTED
|
|
450
|
+
ctx.acceptance_tests_completed = 0
|
|
451
|
+
ctx.rendered_acceptance_scenarios = []
|
|
452
|
+
|
|
414
453
|
def _handle_retry_after_code_change(self):
|
|
415
454
|
"""Re-run the test that failed and triggered a code change."""
|
|
416
455
|
ctx = self.conformance_tests_running_context
|
|
@@ -425,11 +464,20 @@ class RenderContext:
|
|
|
425
464
|
"""Called when a test passes after being retried due to code changes."""
|
|
426
465
|
ctx = self.conformance_tests_running_context
|
|
427
466
|
|
|
428
|
-
if ctx.execution_phase
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
467
|
+
if ctx.execution_phase != TestExecutionPhase.RETRYING_AFTER_CODE_CHANGE:
|
|
468
|
+
return
|
|
469
|
+
|
|
470
|
+
# Test passed after code change - mark that code changed
|
|
471
|
+
ctx.code_changed_during_regression = True
|
|
472
|
+
ctx.test_that_triggered_code_change = None
|
|
473
|
+
|
|
474
|
+
if self._has_more_acceptance_test_phases():
|
|
475
|
+
# The functionality still has acceptance scenarios with no test. Go back to its own testing
|
|
476
|
+
# phase, so that the scenarios that remain still get one. Regression starts after them.
|
|
477
|
+
ctx.execution_phase = TestExecutionPhase.TESTING_CURRENT_FRID
|
|
478
|
+
return
|
|
479
|
+
|
|
480
|
+
self._start_regression_phase()
|
|
433
481
|
|
|
434
482
|
def _handle_current_frid_testing(self):
|
|
435
483
|
"""Handle incremental testing of the current FRID being implemented."""
|
|
@@ -467,16 +515,7 @@ class RenderContext:
|
|
|
467
515
|
if ctx.acceptance_test_phase == AcceptanceTestPhase.IN_PROGRESS:
|
|
468
516
|
if self._has_more_acceptance_test_phases():
|
|
469
517
|
# Run next phase
|
|
470
|
-
if ctx.acceptance_tests_completed == 0:
|
|
471
|
-
ctx.current_testing_frid_high_level_implementation_plan = None
|
|
472
|
-
|
|
473
518
|
ctx.acceptance_tests_completed += 1
|
|
474
|
-
acceptance_tests = self.frid_context.specifications[plain_spec.ACCEPTANCE_TESTS][
|
|
475
|
-
: ctx.acceptance_tests_completed
|
|
476
|
-
]
|
|
477
|
-
ctx.get_conformance_tests_json(ctx.current_testing_module_name)[ctx.current_testing_frid][
|
|
478
|
-
plain_spec.ACCEPTANCE_TESTS
|
|
479
|
-
] = acceptance_tests
|
|
480
519
|
return
|
|
481
520
|
else:
|
|
482
521
|
# All phases done - move to regression
|
render_machine/render_types.py
CHANGED
|
@@ -126,14 +126,16 @@ class ConformanceTestsRunningContext:
|
|
|
126
126
|
self.execution_phase: TestExecutionPhase = TestExecutionPhase.TESTING_CURRENT_FRID
|
|
127
127
|
self.acceptance_test_phase: AcceptanceTestPhase = AcceptanceTestPhase.NOT_STARTED
|
|
128
128
|
self.acceptance_tests_completed: int = 0
|
|
129
|
+
# The acceptance scenarios of this functionality that already have a test, in render order.
|
|
130
|
+
# The action that renders them is entered through more than one path, so it reads this
|
|
131
|
+
# instead of trusting the counter above, which a repeated entry leaves where it was.
|
|
132
|
+
self.rendered_acceptance_scenarios: list[str] = []
|
|
129
133
|
self.frid_being_implemented: Optional[str] = frid_being_implemented
|
|
130
134
|
self.test_that_triggered_code_change: Optional[tuple[str, str]] = None
|
|
131
135
|
self.code_changed_during_regression: bool = False
|
|
132
136
|
|
|
133
137
|
self.regenerating_conformance_tests: bool = False
|
|
134
138
|
|
|
135
|
-
self.current_testing_frid_high_level_implementation_plan: Optional[str] = None
|
|
136
|
-
|
|
137
139
|
# Implementation code changes made by the conformance tests fixer during this conformance phase, in
|
|
138
140
|
# order. Each entry is {"hypothesis": str | None, "approach": str | None, "code_diff": {file: diff}}.
|
|
139
141
|
# Handed to the unit tests fixer so it adjusts the unit tests instead of reverting these changes.
|
|
@@ -160,19 +162,8 @@ class ConformanceTestsRunningContext:
|
|
|
160
162
|
self.get_conformance_tests_json(self.current_testing_module_name).get(self.current_testing_frid) is not None
|
|
161
163
|
)
|
|
162
164
|
|
|
163
|
-
def
|
|
164
|
-
|
|
165
|
-
plain_spec.ACCEPTANCE_TESTS
|
|
166
|
-
in self.get_conformance_tests_json(self.current_testing_module_name)[self.current_testing_frid]
|
|
167
|
-
):
|
|
168
|
-
return self.get_conformance_tests_json(self.current_testing_module_name)[self.current_testing_frid][
|
|
169
|
-
plain_spec.ACCEPTANCE_TESTS
|
|
170
|
-
]
|
|
171
|
-
|
|
172
|
-
return []
|
|
173
|
-
|
|
174
|
-
def get_current_acceptance_test(self) -> Optional[str]:
|
|
175
|
-
"""Get the current acceptance test text (raw, unformatted)."""
|
|
165
|
+
def get_current_acceptance_scenario(self) -> Optional[dict]:
|
|
166
|
+
"""The acceptance scenario being rendered, as the functionality's tree entry holds it."""
|
|
176
167
|
if plain_spec.ACCEPTANCE_TESTS not in self.current_testing_frid_specifications:
|
|
177
168
|
return None
|
|
178
169
|
acceptance_tests = self.current_testing_frid_specifications[plain_spec.ACCEPTANCE_TESTS]
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
"""An acceptance scenario is rendered once, whatever path enters the render action.
|
|
2
|
+
|
|
3
|
+
The action reads the scenario from `acceptance_tests_completed`, a counter the state machine owns.
|
|
4
|
+
A path that enters the action again without moving the counter asked for the same scenario a second
|
|
5
|
+
time, which cost a call and let the render write a second test for the one scenario.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from types import SimpleNamespace
|
|
9
|
+
from unittest.mock import MagicMock, patch
|
|
10
|
+
|
|
11
|
+
import pytest
|
|
12
|
+
|
|
13
|
+
import plain_spec
|
|
14
|
+
from render_machine.actions.render_conformance_tests import RenderConformanceTests
|
|
15
|
+
from render_machine.render_types import AcceptanceTestPhase, ConformanceTestsRunningContext
|
|
16
|
+
|
|
17
|
+
MODULE = "vault_functions"
|
|
18
|
+
FRID = "change-encryption-key"
|
|
19
|
+
SCENARIOS = [
|
|
20
|
+
{"id": "the-old-key-no-longer-opens-the-vault", "text": "..."},
|
|
21
|
+
{"id": "change-with-a-wrong-current-key", "text": "..."},
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@pytest.fixture
|
|
26
|
+
def render_context(tmp_path):
|
|
27
|
+
ctx = ConformanceTestsRunningContext(
|
|
28
|
+
current_testing_module_name=MODULE,
|
|
29
|
+
current_testing_frid=FRID,
|
|
30
|
+
fix_attempts=0,
|
|
31
|
+
conformance_tests_json={FRID: {"folder_name": str(tmp_path)}},
|
|
32
|
+
conformance_tests_render_attempts=0,
|
|
33
|
+
current_testing_frid_specifications={plain_spec.ACCEPTANCE_TESTS: SCENARIOS},
|
|
34
|
+
should_prepare_testing_environment=False,
|
|
35
|
+
frid_being_implemented=FRID,
|
|
36
|
+
)
|
|
37
|
+
ctx.acceptance_tests_completed = 1
|
|
38
|
+
|
|
39
|
+
api = MagicMock()
|
|
40
|
+
api.render_acceptance_tests.return_value = ({}, {"implemented": {}, "missing": [], "extra": []})
|
|
41
|
+
|
|
42
|
+
conformance_tests = MagicMock()
|
|
43
|
+
conformance_tests.fetch_existing_conformance_test_files.return_value = ({}, {})
|
|
44
|
+
|
|
45
|
+
return SimpleNamespace(
|
|
46
|
+
module_name=MODULE,
|
|
47
|
+
build_folder=str(tmp_path),
|
|
48
|
+
plain_source_tree={},
|
|
49
|
+
required_modules=[],
|
|
50
|
+
run_state=SimpleNamespace(),
|
|
51
|
+
codeplain_api=api,
|
|
52
|
+
conformance_tests=conformance_tests,
|
|
53
|
+
conformance_tests_running_context=ctx,
|
|
54
|
+
memory_manager=SimpleNamespace(memory_folder=str(tmp_path)),
|
|
55
|
+
frid_context=SimpleNamespace(
|
|
56
|
+
frid=FRID, linked_resources={}, specifications={plain_spec.ACCEPTANCE_TESTS: SCENARIOS}
|
|
57
|
+
),
|
|
58
|
+
get_required_modules_functionalities=lambda: {},
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _render(render_context):
|
|
63
|
+
with (
|
|
64
|
+
patch("render_machine.actions.render_conformance_tests.ImplementationCodeHelpers") as code_helpers,
|
|
65
|
+
patch("render_machine.actions.render_conformance_tests.MemoryManager") as memory,
|
|
66
|
+
patch("render_machine.actions.render_conformance_tests.file_utils"),
|
|
67
|
+
patch("render_machine.actions.render_conformance_tests.console") as console,
|
|
68
|
+
):
|
|
69
|
+
code_helpers.fetch_existing_files.return_value = ({}, {})
|
|
70
|
+
memory.fetch_memory_files.return_value = ({}, {})
|
|
71
|
+
outcome, _ = RenderConformanceTests()._render_acceptance_test(render_context)
|
|
72
|
+
return outcome, [call.args[0] for call in console.warning.call_args_list]
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def test_the_scenario_is_rendered_and_recorded(render_context):
|
|
76
|
+
outcome, warnings = _render(render_context)
|
|
77
|
+
|
|
78
|
+
assert outcome == RenderConformanceTests.SUCCESSFUL_OUTCOME
|
|
79
|
+
assert render_context.codeplain_api.render_acceptance_tests.call_count == 1
|
|
80
|
+
assert render_context.conformance_tests_running_context.rendered_acceptance_scenarios == [SCENARIOS[0]["id"]]
|
|
81
|
+
assert warnings == []
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def test_a_second_entry_without_the_counter_moving_renders_nothing(render_context):
|
|
85
|
+
_render(render_context)
|
|
86
|
+
|
|
87
|
+
outcome, warnings = _render(render_context)
|
|
88
|
+
|
|
89
|
+
assert outcome == RenderConformanceTests.SUCCESSFUL_OUTCOME
|
|
90
|
+
# The call that would have written a second test for the scenario is not made.
|
|
91
|
+
assert render_context.codeplain_api.render_acceptance_tests.call_count == 1
|
|
92
|
+
assert len(warnings) == 1
|
|
93
|
+
assert SCENARIOS[0]["id"] in warnings[0]
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def test_the_next_scenario_is_rendered_once_the_counter_moves(render_context):
|
|
97
|
+
_render(render_context)
|
|
98
|
+
render_context.conformance_tests_running_context.acceptance_tests_completed = 2
|
|
99
|
+
|
|
100
|
+
outcome, warnings = _render(render_context)
|
|
101
|
+
|
|
102
|
+
assert outcome == RenderConformanceTests.SUCCESSFUL_OUTCOME
|
|
103
|
+
assert render_context.codeplain_api.render_acceptance_tests.call_count == 2
|
|
104
|
+
assert [call.args[8] for call in render_context.codeplain_api.render_acceptance_tests.call_args_list] == [
|
|
105
|
+
SCENARIOS[0]["id"],
|
|
106
|
+
SCENARIOS[1]["id"],
|
|
107
|
+
]
|
|
108
|
+
assert warnings == []
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def test_a_functionality_without_acceptance_scenarios_renders_nothing(render_context):
|
|
112
|
+
render_context.frid_context.specifications = {}
|
|
113
|
+
|
|
114
|
+
outcome, _ = _render(render_context)
|
|
115
|
+
|
|
116
|
+
assert outcome == RenderConformanceTests.SUCCESSFUL_OUTCOME
|
|
117
|
+
assert render_context.codeplain_api.render_acceptance_tests.call_count == 0
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
class TestRegeneratedConformanceTests:
|
|
121
|
+
"""Regeneration deletes the tests folder of the functionality, so the acceptance tests in it are
|
|
122
|
+
gone too. The functionality must then write all of its tests again, from the first scenario."""
|
|
123
|
+
|
|
124
|
+
@staticmethod
|
|
125
|
+
def make_render_context(tmp_path, acceptance_tests_completed, rendered):
|
|
126
|
+
from render_machine.render_context import RenderContext
|
|
127
|
+
|
|
128
|
+
render_context = RenderContext.__new__(RenderContext)
|
|
129
|
+
render_context.module_name = MODULE
|
|
130
|
+
folder = tmp_path / "tests_folder"
|
|
131
|
+
folder.mkdir()
|
|
132
|
+
|
|
133
|
+
ctx = ConformanceTestsRunningContext(
|
|
134
|
+
current_testing_module_name=MODULE,
|
|
135
|
+
current_testing_frid=FRID,
|
|
136
|
+
fix_attempts=5,
|
|
137
|
+
conformance_tests_json={FRID: {"folder_name": str(folder)}},
|
|
138
|
+
conformance_tests_render_attempts=0,
|
|
139
|
+
current_testing_frid_specifications={plain_spec.ACCEPTANCE_TESTS: SCENARIOS},
|
|
140
|
+
should_prepare_testing_environment=False,
|
|
141
|
+
frid_being_implemented=FRID,
|
|
142
|
+
)
|
|
143
|
+
ctx.acceptance_test_phase = AcceptanceTestPhase.IN_PROGRESS
|
|
144
|
+
ctx.acceptance_tests_completed = acceptance_tests_completed
|
|
145
|
+
ctx.rendered_acceptance_scenarios = list(rendered)
|
|
146
|
+
ctx.regenerating_conformance_tests = True
|
|
147
|
+
render_context.conformance_tests_running_context = ctx
|
|
148
|
+
|
|
149
|
+
return render_context
|
|
150
|
+
|
|
151
|
+
def test_the_acceptance_phase_starts_again(self, tmp_path):
|
|
152
|
+
render_context = self.make_render_context(tmp_path, 1, [SCENARIOS[0]["id"]])
|
|
153
|
+
|
|
154
|
+
with patch("render_machine.render_context.console"):
|
|
155
|
+
render_context._handle_test_regeneration()
|
|
156
|
+
|
|
157
|
+
ctx = render_context.conformance_tests_running_context
|
|
158
|
+
assert ctx.acceptance_test_phase == AcceptanceTestPhase.NOT_STARTED
|
|
159
|
+
assert ctx.acceptance_tests_completed == 0
|
|
160
|
+
assert ctx.rendered_acceptance_scenarios == []
|
|
161
|
+
|
|
162
|
+
def test_the_deleted_folder_is_no_longer_claimed(self, tmp_path):
|
|
163
|
+
render_context = self.make_render_context(tmp_path, 1, [SCENARIOS[0]["id"]])
|
|
164
|
+
|
|
165
|
+
with patch("render_machine.render_context.console"):
|
|
166
|
+
render_context._handle_test_regeneration()
|
|
167
|
+
|
|
168
|
+
ctx = render_context.conformance_tests_running_context
|
|
169
|
+
assert not ctx.current_conformance_tests_exist()
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"""A conformance fix that changes implementation code must not cancel the acceptance scenarios.
|
|
2
|
+
|
|
3
|
+
The fix moves the execution phase to RETRYING_AFTER_CODE_CHANGE. When the test then passes, the
|
|
4
|
+
render must go back to the testing phase of the functionality while acceptance scenarios remain,
|
|
5
|
+
because that phase holds the only code that renders them. Regression starts after the last one.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from types import SimpleNamespace
|
|
9
|
+
|
|
10
|
+
import pytest
|
|
11
|
+
|
|
12
|
+
import plain_spec
|
|
13
|
+
from render_machine.render_context import RenderContext
|
|
14
|
+
from render_machine.render_types import (
|
|
15
|
+
AcceptanceTestPhase,
|
|
16
|
+
ConformanceTestsRunningContext,
|
|
17
|
+
)
|
|
18
|
+
from render_machine.render_types import TestExecutionPhase as ExecutionPhase
|
|
19
|
+
|
|
20
|
+
MODULE = "vault-cli"
|
|
21
|
+
FRID = "get-command"
|
|
22
|
+
SCENARIOS = [
|
|
23
|
+
{"id": "get-with-an-id-that-is-not-in-the-vault", "text": "..."},
|
|
24
|
+
{"id": "get-with-a-wrong-master-password", "text": "..."},
|
|
25
|
+
{"id": "get-from-a-vault-that-is-not-there", "text": "..."},
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def make_render_context(acceptance_scenarios, acceptance_test_phase, acceptance_tests_completed):
|
|
30
|
+
"""A render context stopped right after a code fix, with its test passing again."""
|
|
31
|
+
render_context = RenderContext.__new__(RenderContext)
|
|
32
|
+
render_context.module_name = MODULE
|
|
33
|
+
render_context.frid_context = SimpleNamespace(
|
|
34
|
+
frid=FRID, specifications={plain_spec.ACCEPTANCE_TESTS: acceptance_scenarios}
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
ctx = ConformanceTestsRunningContext(
|
|
38
|
+
current_testing_module_name=MODULE,
|
|
39
|
+
current_testing_frid=FRID,
|
|
40
|
+
fix_attempts=0,
|
|
41
|
+
conformance_tests_json={FRID: {"folder_name": "get_command"}},
|
|
42
|
+
conformance_tests_render_attempts=0,
|
|
43
|
+
current_testing_frid_specifications={plain_spec.ACCEPTANCE_TESTS: acceptance_scenarios},
|
|
44
|
+
should_prepare_testing_environment=False,
|
|
45
|
+
frid_being_implemented=FRID,
|
|
46
|
+
)
|
|
47
|
+
ctx.execution_phase = ExecutionPhase.RETRYING_AFTER_CODE_CHANGE
|
|
48
|
+
ctx.test_that_triggered_code_change = (MODULE, FRID)
|
|
49
|
+
ctx.acceptance_test_phase = acceptance_test_phase
|
|
50
|
+
ctx.acceptance_tests_completed = acceptance_tests_completed
|
|
51
|
+
render_context.conformance_tests_running_context = ctx
|
|
52
|
+
|
|
53
|
+
return render_context
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def test_a_scenario_that_remains_gets_a_test_after_a_code_fix():
|
|
57
|
+
"""The first scenario has a test, the fix changed code, two scenarios remain."""
|
|
58
|
+
render_context = make_render_context(SCENARIOS, AcceptanceTestPhase.IN_PROGRESS, 1)
|
|
59
|
+
|
|
60
|
+
render_context.start_conformance_tests_for_frid()
|
|
61
|
+
|
|
62
|
+
ctx = render_context.conformance_tests_running_context
|
|
63
|
+
# The render asks for the second scenario instead of moving on to regression.
|
|
64
|
+
assert ctx.acceptance_tests_completed == 2
|
|
65
|
+
assert ctx.execution_phase == ExecutionPhase.TESTING_CURRENT_FRID
|
|
66
|
+
assert ctx.acceptance_test_phase == AcceptanceTestPhase.IN_PROGRESS
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def test_every_scenario_is_reached_when_a_code_fix_follows_each_one():
|
|
70
|
+
"""A code fix after every scenario still leaves no scenario without a test."""
|
|
71
|
+
render_context = make_render_context(SCENARIOS, AcceptanceTestPhase.IN_PROGRESS, 1)
|
|
72
|
+
ctx = render_context.conformance_tests_running_context
|
|
73
|
+
|
|
74
|
+
for _ in range(len(SCENARIOS) - 1):
|
|
75
|
+
ctx.execution_phase = ExecutionPhase.RETRYING_AFTER_CODE_CHANGE
|
|
76
|
+
ctx.test_that_triggered_code_change = (MODULE, FRID)
|
|
77
|
+
render_context.start_conformance_tests_for_frid()
|
|
78
|
+
|
|
79
|
+
assert ctx.acceptance_tests_completed == len(SCENARIOS)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def test_regression_starts_when_every_scenario_has_a_test():
|
|
83
|
+
render_context = make_render_context(SCENARIOS, AcceptanceTestPhase.IN_PROGRESS, len(SCENARIOS))
|
|
84
|
+
|
|
85
|
+
render_context._on_conformance_test_passed_after_retry()
|
|
86
|
+
|
|
87
|
+
ctx = render_context.conformance_tests_running_context
|
|
88
|
+
assert ctx.execution_phase == ExecutionPhase.RUNNING_REGRESSION
|
|
89
|
+
assert ctx.code_changed_during_regression is True
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
@pytest.mark.parametrize(
|
|
93
|
+
"acceptance_scenarios, acceptance_test_phase, acceptance_tests_completed",
|
|
94
|
+
[
|
|
95
|
+
([], AcceptanceTestPhase.NOT_APPLICABLE, 1),
|
|
96
|
+
(SCENARIOS, AcceptanceTestPhase.COMPLETED, len(SCENARIOS)),
|
|
97
|
+
],
|
|
98
|
+
ids=["no acceptance scenario", "acceptance phase completed"],
|
|
99
|
+
)
|
|
100
|
+
def test_regression_starts_when_the_functionality_has_no_scenario_left(
|
|
101
|
+
acceptance_scenarios, acceptance_test_phase, acceptance_tests_completed
|
|
102
|
+
):
|
|
103
|
+
"""A code fix during regression keeps the behaviour it had before."""
|
|
104
|
+
render_context = make_render_context(acceptance_scenarios, acceptance_test_phase, acceptance_tests_completed)
|
|
105
|
+
|
|
106
|
+
render_context._on_conformance_test_passed_after_retry()
|
|
107
|
+
|
|
108
|
+
ctx = render_context.conformance_tests_running_context
|
|
109
|
+
assert ctx.execution_phase == ExecutionPhase.RUNNING_REGRESSION
|
|
110
|
+
assert ctx.current_testing_frid is None
|