prettyplay 0.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prettyplay/.usages/lifecycle.md +117 -0
- prettyplay/.usages/steps.md +169 -0
- prettyplay/__init__.py +18 -0
- prettyplay/cache/.usages/addressing.md +44 -0
- prettyplay/cache/.usages/budgets.md +51 -0
- prettyplay/cache/.usages/storage.md +32 -0
- prettyplay/cache/__init__.py +8 -0
- prettyplay/cache/budgets.py +93 -0
- prettyplay/cache/models.py +55 -0
- prettyplay/cache/store.py +276 -0
- prettyplay/cache/text.py +36 -0
- prettyplay/config/.usages/configuration.md +246 -0
- prettyplay/config/__init__.py +6 -0
- prettyplay/config/loader.py +336 -0
- prettyplay/config/models.py +288 -0
- prettyplay/driver/.usages/error_kinds.md +26 -0
- prettyplay/driver/.usages/plumbing.md +52 -0
- prettyplay/driver/__init__.py +11 -0
- prettyplay/driver/errors.py +76 -0
- prettyplay/driver/page.py +175 -0
- prettyplay/driver/session.py +330 -0
- prettyplay/engine/.usages/generation.md +157 -0
- prettyplay/engine/.usages/healing.md +61 -0
- prettyplay/engine/__init__.py +19 -0
- prettyplay/engine/attempts.py +80 -0
- prettyplay/engine/classification.py +70 -0
- prettyplay/engine/compliance.py +125 -0
- prettyplay/engine/execution.py +48 -0
- prettyplay/engine/generator.py +1110 -0
- prettyplay/engine/groups/.usages/recovery.md +71 -0
- prettyplay/engine/groups/__init__.py +7 -0
- prettyplay/engine/groups/diagnosis.py +128 -0
- prettyplay/engine/groups/outcome.py +55 -0
- prettyplay/engine/groups/recovery.py +491 -0
- prettyplay/engine/healer.py +149 -0
- prettyplay/engine/polling/.usages/settle.md +55 -0
- prettyplay/engine/polling/__init__.py +6 -0
- prettyplay/engine/polling/settle.py +98 -0
- prettyplay/engine/polling/window.py +72 -0
- prettyplay/engine/renderer/.usages/memory.md +24 -0
- prettyplay/engine/renderer/.usages/rendering.md +44 -0
- prettyplay/engine/renderer/.usages/validation.md +20 -0
- prettyplay/engine/renderer/__init__.py +7 -0
- prettyplay/engine/renderer/memory.py +37 -0
- prettyplay/engine/renderer/render.py +184 -0
- prettyplay/engine/renderer/validation.py +76 -0
- prettyplay/engine/steering/.usages/steering.md +95 -0
- prettyplay/engine/steering/__init__.py +5 -0
- prettyplay/engine/steering/steering.py +765 -0
- prettyplay/engine/text.py +41 -0
- prettyplay/executor.py +654 -0
- prettyplay/failures/.usages/taxonomy.md +131 -0
- prettyplay/failures/__init__.py +25 -0
- prettyplay/failures/errors.py +359 -0
- prettyplay/groups.py +276 -0
- prettyplay/llm/.usages/classification.md +31 -0
- prettyplay/llm/.usages/providers.md +127 -0
- prettyplay/llm/.usages/transport.md +81 -0
- prettyplay/llm/__init__.py +38 -0
- prettyplay/llm/_request.py +823 -0
- prettyplay/llm/anthropic_provider.py +468 -0
- prettyplay/llm/models.py +273 -0
- prettyplay/llm/openai_provider.py +444 -0
- prettyplay/llm/provider.py +279 -0
- prettyplay/reporting/.usages/hooks.md +61 -0
- prettyplay/reporting/__init__.py +6 -0
- prettyplay/reporting/hooks.py +47 -0
- prettyplay/reporting/reporter.py +70 -0
- prettyplay/runtime.py +81 -0
- prettyplay/scenario.py +469 -0
- prettyplay-0.0.2.dist-info/METADATA +515 -0
- prettyplay-0.0.2.dist-info/RECORD +75 -0
- prettyplay-0.0.2.dist-info/WHEEL +5 -0
- prettyplay-0.0.2.dist-info/licenses/LICENSE +28 -0
- prettyplay-0.0.2.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1110 @@
|
|
|
1
|
+
"""Generation of working step code: LLM candidates executed against the live page under the settle window."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from datetime import date
|
|
5
|
+
|
|
6
|
+
from ..cache import CachedStep, RunBudgets, StepCache, StepIdentity
|
|
7
|
+
from ..config import Config
|
|
8
|
+
from ..driver import PageFacade
|
|
9
|
+
from ..failures import FailureVerdict, IncurableStepError, LLMUnavailableError, ProductDefectError
|
|
10
|
+
from ..llm import ComplianceFinding, FailureClassification, LLMProvider, ScenarioStep
|
|
11
|
+
from ..reporting import StepReporter
|
|
12
|
+
from .attempts import (
|
|
13
|
+
OUTCOME_COMPLIANCE_BLOCKED,
|
|
14
|
+
OUTCOME_EXECUTION_FAILED,
|
|
15
|
+
OUTCOME_FAILED_CHECK,
|
|
16
|
+
StepAttempt,
|
|
17
|
+
_read_url,
|
|
18
|
+
)
|
|
19
|
+
from .classification import classify_step_failure
|
|
20
|
+
from .compliance import check_step_compliance
|
|
21
|
+
from .execution import bind_step_inputs, run_step_code
|
|
22
|
+
from .polling import SettleWindow, settle
|
|
23
|
+
from .renderer import PreparedStep, StepMemory, validate_step_result
|
|
24
|
+
from .text import format_step_error
|
|
25
|
+
|
|
26
|
+
logger = logging.getLogger("prettyplay")
|
|
27
|
+
|
|
28
|
+
#: System prompt of every generation and regeneration request; applied verbatim by the provider.
|
|
29
|
+
#: Frozen mirror of ``.goga/usages/prompts/step_generation.md`` (the section after the ``---``
|
|
30
|
+
#: separator) — the single source of the prompt; the constant changes only together with the file.
|
|
31
|
+
SYSTEM_PROMPT = """You generate executable Python code for one step of a web UI test.
|
|
32
|
+
|
|
33
|
+
Input you receive:
|
|
34
|
+
- STEP TYPE: action or assertion — the kind of the step
|
|
35
|
+
- STEP: the prepared instruction of the step — the plain-text sentence with actual values embedded
|
|
36
|
+
- INPUTS: the call input bindings, one name = value line each — present when the step carries inputs
|
|
37
|
+
- RESULTS: the declared result names — present when the step declares captures; the code must return a dictionary of exactly these names to non-blank strings observed on the page
|
|
38
|
+
- PREVIOUS STEPS: the sentences of the previous steps of the test, in order
|
|
39
|
+
- PAGE SNAPSHOT: the accessibility snapshot of the current page
|
|
40
|
+
- PAGE URL: the current URL of the page, when present
|
|
41
|
+
- SCREENSHOT: an image of the page, when attached
|
|
42
|
+
- CHEAT SHEET: a compact reference of useful Playwright sync API idioms — guidance, not an allowlist; everything standard stays allowed
|
|
43
|
+
- USER INSTRUCTIONS: the project's binding code style guidance, when configured
|
|
44
|
+
- HISTORY: the verbatim record of every attempt of this step so far, when present — the original cached code first when it exists; each record carries the attempt outcome, the URL before -> after line, the complete candidate code and the complete error
|
|
45
|
+
- RECOMMENDATION: the diagnosis of the classification that preceded this regeneration, when present
|
|
46
|
+
- USER GUIDANCE: the engineer guidance message of the interactive steering, when present
|
|
47
|
+
|
|
48
|
+
Output exactly one Python code block with one function of the fixed form:
|
|
49
|
+
|
|
50
|
+
- without RESULTS: def step(page) -> None: ...
|
|
51
|
+
- with RESULTS: def step(page) -> dict[str, str] | None: ... — return the dictionary of exactly the declared names to values actually read from the current page
|
|
52
|
+
|
|
53
|
+
Rules:
|
|
54
|
+
- The function receives exactly one argument: the page — the genuine Playwright sync Page; the whole step runs inside the driver worker thread
|
|
55
|
+
- The global `step_inputs` is supplied fresh on every execution: captured values are available by name and call-local INPUTS under `step_inputs["vars"]`. Read changing values from it instead of hardcoding the current STEP or INPUTS values into cached code
|
|
56
|
+
- Import from playwright.sync_api and the Python standard library only — no third-party
|
|
57
|
+
libraries; imports are global only: at the top level of the code block, before `def
|
|
58
|
+
step`, never inside the function body
|
|
59
|
+
- Work through the standard Playwright sync API: locator factories, actions, waits, expect chains, plain asserts on immediate reads — everything standard is allowed; the CHEAT SHEET is guidance, never a boundary
|
|
60
|
+
- Assertions: for an assertion sentence end with a check — a waiting expect(...) chain for dynamic content, or an immediate read with a plain Python assert (assert locator.count() > 1)
|
|
61
|
+
- When the request carries a RESULTS block: read every declared value from the current page — never fabricate, never reuse values from HISTORY; return all declared names in one dictionary; every value is a non-blank string; a missing observation fails the step rather than fabricating a value
|
|
62
|
+
- No fixed delays, no sleeps, no wait_for_timeout — locators and expect chains auto-wait
|
|
63
|
+
- The runtime owns the page lifecycle: never call page.close() or context.close()
|
|
64
|
+
- No stateful actions that outlive the step on the page shared by the whole test: page.route, page.clock, add_init_script, tracing, HAR, CDP — excluded from generated code; a cached step would poison every later step far from the cause
|
|
65
|
+
- Dialogs: capture with the stock means — with page.expect_event("dialog") as info: — perform the triggering action inside the block, read info.value.type, info.value.message, info.value.default_value, then info.value.accept() or info.value.dismiss()
|
|
66
|
+
- Popups and new tabs: capture with with page.expect_popup() as popup_info: — trigger the opening action inside the block, work through popup_info.value; page.bring_to_front() raises a page above the others
|
|
67
|
+
- Content inside an iframe goes through page.frame_locator(selector) — locate elements within the returned scope; nested frames chain
|
|
68
|
+
- Scrolling: locator.scroll_into_view_if_needed() and page.mouse.wheel(dx, dy) are the standard means
|
|
69
|
+
- The page state may already include the effects of prior attempts or manual intervention — the HISTORY records and their URL before -> after lines show what already happened. Regeneration after failed attempts never rides the leftover state of those attempts: an action step repeats its action through the page even when the page looks done. A first attempt — no HISTORY — works on the page the previous steps produced: the PAGE URL line and the snapshot show where they landed
|
|
70
|
+
- An action step performs its action and ends there — no trailing check that confirms the action's own completion; the following steps of the test carry their own expectations
|
|
71
|
+
- An assertion step only observes the current page — never navigate, click, type or fill; passive observation such as scroll_into_view_if_needed() stays allowed
|
|
72
|
+
- RECOMMENDATION and USER GUIDANCE carry the diagnosis and the engineer's intent — follow them when they conflict with your first instinct
|
|
73
|
+
- USER INSTRUCTIONS are binding for everything below the safety core of these Rules: follow them when configured; silently ignoring an instruction is a violation
|
|
74
|
+
- The safety core of these Rules always outranks the instructions: the fixed function form, the import rule, the lifecycle rule, the stateful-action exclusions, no fixed delays. An instruction conflicting with a Rule or demanding a stateful action is unfollowable: never implement it silently — raise in the step code with the message "instruction conflicts with rule Y" naming the conflict, so the failure surfaces loudly
|
|
75
|
+
- Prefer-type instructions are conditional by their own wording: follow them when the page offers the option — best-effort with a graceful fallback is compliance
|
|
76
|
+
- The step must complete exactly what STEP says — nothing more, nothing less
|
|
77
|
+
- Output only the code block, no explanations"""
|
|
78
|
+
|
|
79
|
+
#: The compact standard Playwright sync API reference of every generation request;
|
|
80
|
+
#: guidance, not an allowlist — everything standard stays allowed.
|
|
81
|
+
#: Frozen mirror of ``.goga/usages/prompts/step_cheatsheet.md`` — the whole file, verbatim
|
|
82
|
+
#: (the practice has no ``---`` separator, so the whole-file rule is the only mirror
|
|
83
|
+
#: rule with no extraction logic to drift); the constant changes only together with the file.
|
|
84
|
+
CHEAT_SHEET = """# Playwright cheat sheet
|
|
85
|
+
|
|
86
|
+
The compact standard Playwright sync API reference carried by every step-code generation and regeneration request
|
|
87
|
+
of prettyplay — referenced by the engine and steering cells as the `cheat_sheet` practice; rendered by the provider
|
|
88
|
+
implementations as the leading CHEAT SHEET block of the user content. Guidance, not an allowlist: everything
|
|
89
|
+
standard stays allowed — the error-driven regeneration loop is the second line of defense against hallucinated
|
|
90
|
+
calls. Target audience: the generation model — a model that knows Playwright weakly writes a correct step from
|
|
91
|
+
this reference alone.
|
|
92
|
+
|
|
93
|
+
## Locator factories
|
|
94
|
+
|
|
95
|
+
page.get_by_role("button", name="Sign in")
|
|
96
|
+
page.get_by_label("Username")
|
|
97
|
+
page.get_by_text("Welcome back")
|
|
98
|
+
page.get_by_placeholder("Search")
|
|
99
|
+
page.get_by_alt_text("Logo")
|
|
100
|
+
page.get_by_title("Close")
|
|
101
|
+
page.get_by_test_id("submit")
|
|
102
|
+
page.locator("css selector | //xpath | [data-qa=row]")
|
|
103
|
+
|
|
104
|
+
## Narrowing
|
|
105
|
+
|
|
106
|
+
locator.first / locator.last / locator.nth(2)
|
|
107
|
+
locator.filter(has_text="Product X")
|
|
108
|
+
locator.and_(other) / locator.or_(other)
|
|
109
|
+
|
|
110
|
+
## Actions
|
|
111
|
+
|
|
112
|
+
.click() / .dblclick() / .click(button="right")
|
|
113
|
+
.fill("text") / .clear() / .press("Enter") / .press("Control+A")
|
|
114
|
+
.check() / .uncheck() / .hover() / .select_option("v")
|
|
115
|
+
.drag_to(target) / .set_input_files("path.png")
|
|
116
|
+
|
|
117
|
+
## Navigation and waits
|
|
118
|
+
|
|
119
|
+
page.goto(url) / page.go_back() / page.go_forward() / page.reload()
|
|
120
|
+
page.wait_for_url("**/dashboard") / page.wait_for_load_state("networkidle")
|
|
121
|
+
page.url # the current URL — an immediate read beside the waiting forms
|
|
122
|
+
|
|
123
|
+
## Waiting assertions — expect chains
|
|
124
|
+
|
|
125
|
+
from playwright.sync_api import expect
|
|
126
|
+
|
|
127
|
+
expect(locator).to_be_visible() / to_be_hidden()
|
|
128
|
+
expect(locator).to_be_enabled() / to_be_checked()
|
|
129
|
+
expect(locator).to_have_text("...") / to_contain_text("...")
|
|
130
|
+
expect(locator).to_have_value("v") / to_have_attribute("href", "/docs")
|
|
131
|
+
expect(page).to_have_url("**/dashboard") / to_have_title("Dashboard")
|
|
132
|
+
|
|
133
|
+
## Count forms — "the page shows a list of X"
|
|
134
|
+
|
|
135
|
+
items = page.get_by_role("listitem")
|
|
136
|
+
assert items.count() > 1
|
|
137
|
+
|
|
138
|
+
An exact count: `expect(items).to_have_count(3)`. One check per meaning — a visibility expect
|
|
139
|
+
plus a count assert on the same locator verifies one fact twice.
|
|
140
|
+
|
|
141
|
+
## Immediate reads with plain asserts
|
|
142
|
+
|
|
143
|
+
assert locator.count() >= 1
|
|
144
|
+
assert "Dashboard" in page.title()
|
|
145
|
+
assert "/dashboard" in page.url
|
|
146
|
+
|
|
147
|
+
## Dialogs
|
|
148
|
+
|
|
149
|
+
with page.expect_event("dialog") as info:
|
|
150
|
+
page.get_by_role("button", name="Delete").click()
|
|
151
|
+
dialog = info.value
|
|
152
|
+
dialog.type / dialog.message / dialog.default_value
|
|
153
|
+
dialog.accept() / dialog.dismiss() / dialog.accept("the answer")
|
|
154
|
+
|
|
155
|
+
## Popups and new tabs
|
|
156
|
+
|
|
157
|
+
with page.expect_popup() as popup_info:
|
|
158
|
+
page.get_by_role("link", name="Open docs").click()
|
|
159
|
+
popup = popup_info.value
|
|
160
|
+
popup.bring_to_front()
|
|
161
|
+
|
|
162
|
+
## Frames
|
|
163
|
+
|
|
164
|
+
frame = page.frame_locator("#checkout")
|
|
165
|
+
frame.get_by_role("button", name="Pay").click()
|
|
166
|
+
|
|
167
|
+
## Scrolling
|
|
168
|
+
|
|
169
|
+
locator.scroll_into_view_if_needed()
|
|
170
|
+
page.mouse.wheel(0, 600)
|
|
171
|
+
"""
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
class StepGenerator:
|
|
175
|
+
"""Generates working step code by executing LLM candidates against the live page; only proven candidates are cached.
|
|
176
|
+
|
|
177
|
+
Attributes:
|
|
178
|
+
_config: project settings; the screenshot flag feeds the requests.
|
|
179
|
+
_provider: the LLM port implementation doing the requests.
|
|
180
|
+
_cache: the store where working steps are saved.
|
|
181
|
+
_budgets: the per-test attempt registry of the engine.
|
|
182
|
+
_reporter: the visibility point for generation and cache events.
|
|
183
|
+
"""
|
|
184
|
+
|
|
185
|
+
def __init__(
|
|
186
|
+
self,
|
|
187
|
+
config: Config,
|
|
188
|
+
provider: LLMProvider,
|
|
189
|
+
cache: StepCache,
|
|
190
|
+
budgets: RunBudgets,
|
|
191
|
+
reporter: StepReporter,
|
|
192
|
+
) -> None:
|
|
193
|
+
"""Keep the collaborators of the generation loop.
|
|
194
|
+
|
|
195
|
+
Args:
|
|
196
|
+
config: project settings; ``send_screenshots`` attaches page images.
|
|
197
|
+
provider: the LLM port implementation.
|
|
198
|
+
cache: the store of working steps.
|
|
199
|
+
budgets: the per-test attempt registry.
|
|
200
|
+
reporter: the visibility point for engine events.
|
|
201
|
+
"""
|
|
202
|
+
self._config = config
|
|
203
|
+
self._provider = provider
|
|
204
|
+
self._cache = cache
|
|
205
|
+
self._budgets = budgets
|
|
206
|
+
self._reporter = reporter
|
|
207
|
+
|
|
208
|
+
def generate( # noqa: PLR0913, PLR0917 — the signature is fixed by the engine contract
|
|
209
|
+
self,
|
|
210
|
+
identity: StepIdentity,
|
|
211
|
+
prepared: PreparedStep,
|
|
212
|
+
step_type: str,
|
|
213
|
+
previous_steps: list[ScenarioStep],
|
|
214
|
+
group_prompt: str | None,
|
|
215
|
+
page: PageFacade,
|
|
216
|
+
attempt_history: list[StepAttempt],
|
|
217
|
+
window: SettleWindow,
|
|
218
|
+
memory: StepMemory,
|
|
219
|
+
) -> CachedStep:
|
|
220
|
+
"""Generate step code until a candidate works, then cache it.
|
|
221
|
+
|
|
222
|
+
Args:
|
|
223
|
+
identity: the address of the step.
|
|
224
|
+
prepared: the render product of the step — instruction, input
|
|
225
|
+
bindings, result declarations; every request renders the
|
|
226
|
+
instruction with its INPUTS and RESULTS blocks; declarations
|
|
227
|
+
present — the candidate code must return the result
|
|
228
|
+
dictionary of exactly the declared names.
|
|
229
|
+
step_type: action or assertion — carried into every request.
|
|
230
|
+
previous_steps: the typed scenario records of the previous steps
|
|
231
|
+
of the test, in execution order — each the prepared
|
|
232
|
+
instruction plus its permanent group membership.
|
|
233
|
+
group_prompt: the group prompt of the current step's group;
|
|
234
|
+
None — an ordinary step: every path stays byte-identical to
|
|
235
|
+
today, the same requests, the same classification points,
|
|
236
|
+
the same bounded healing; non-empty — a group step: every
|
|
237
|
+
request carries the group framing and the internal
|
|
238
|
+
classification points are suppressed — the calling executor
|
|
239
|
+
routes the unclassified failures to the group recovery.
|
|
240
|
+
page: the live page facade the candidates run against.
|
|
241
|
+
attempt_history: the per-step attempt history created by the
|
|
242
|
+
executor — this loop appends a record after every attempt
|
|
243
|
+
that does not produce a cached step.
|
|
244
|
+
window: the settle window of the current step execution — every
|
|
245
|
+
candidate execution absorbs transient failures inside it.
|
|
246
|
+
memory: the test memory — the accepted candidate's validated
|
|
247
|
+
captures publish here after validation and the gate; a
|
|
248
|
+
failed or gate-blocked attempt publishes nothing.
|
|
249
|
+
|
|
250
|
+
Returns:
|
|
251
|
+
The cached step holding the proven code.
|
|
252
|
+
|
|
253
|
+
Raises:
|
|
254
|
+
ProductDefectError: a failed candidate check classified as a
|
|
255
|
+
genuine product defect — by the entry or the final
|
|
256
|
+
classification; never raised for a group step inside this
|
|
257
|
+
loop.
|
|
258
|
+
IncurableStepError: a check, result-contract violation or budget
|
|
259
|
+
outcome classified incurable, or the healing funding
|
|
260
|
+
refused; the code field carries the failed step code; a
|
|
261
|
+
group step raises the unclassified variant at its failed
|
|
262
|
+
check and its budget exhaustion — the group recovery
|
|
263
|
+
decides.
|
|
264
|
+
LLMUnavailableError: the provider service failed after the
|
|
265
|
+
bounded transport retries; no engine retry.
|
|
266
|
+
ComplianceVerdictError: the compliance verdict of a green
|
|
267
|
+
candidate did not parse; nothing is cached.
|
|
268
|
+
"""
|
|
269
|
+
return self._generation_loop(
|
|
270
|
+
identity, prepared, step_type, previous_steps, group_prompt, page, attempt_history, window, memory
|
|
271
|
+
)
|
|
272
|
+
|
|
273
|
+
def regenerate( # noqa: PLR0913, PLR0917 — the signature is fixed by the engine contract
|
|
274
|
+
self,
|
|
275
|
+
identity: StepIdentity,
|
|
276
|
+
prepared: PreparedStep,
|
|
277
|
+
step_type: str,
|
|
278
|
+
previous_steps: list[ScenarioStep],
|
|
279
|
+
group_prompt: str | None,
|
|
280
|
+
page: PageFacade,
|
|
281
|
+
attempt_history: list[StepAttempt],
|
|
282
|
+
recommendation: str,
|
|
283
|
+
window: SettleWindow,
|
|
284
|
+
memory: StepMemory,
|
|
285
|
+
) -> CachedStep:
|
|
286
|
+
"""Regenerate step code starting from the anchored history and the diagnosis.
|
|
287
|
+
|
|
288
|
+
Args:
|
|
289
|
+
identity: the address of the step.
|
|
290
|
+
prepared: the render product of the row or healed step — carried
|
|
291
|
+
into every request with its INPUTS and RESULTS blocks;
|
|
292
|
+
passed by the calling path: the healer threads the
|
|
293
|
+
executor's product, the recovery re-renders the row step
|
|
294
|
+
against the current context.
|
|
295
|
+
step_type: action or assertion — carried into every request.
|
|
296
|
+
previous_steps: the typed scenario records of the previous steps
|
|
297
|
+
of the test, in execution order.
|
|
298
|
+
group_prompt: the group prompt of the row step's group — the
|
|
299
|
+
recovery row passes it; non-empty — the request carries the
|
|
300
|
+
group framing; None — an ordinary regeneration.
|
|
301
|
+
page: the live page facade the candidates run against.
|
|
302
|
+
attempt_history: the anchored per-step attempt history — record 0
|
|
303
|
+
carries the original cached code composed by the caller; this
|
|
304
|
+
loop appends after every attempt that does not produce a
|
|
305
|
+
cached step; the original is never lost to a re-binding.
|
|
306
|
+
recommendation: the diagnosis of the classification that launched
|
|
307
|
+
the healing; rendered into every request — the entry
|
|
308
|
+
classification guards the anti-masking, no per-attempt
|
|
309
|
+
classification happens inside the loop.
|
|
310
|
+
window: the settle window of the current step execution.
|
|
311
|
+
memory: the test memory — publication at the acceptance point,
|
|
312
|
+
as in generate.
|
|
313
|
+
|
|
314
|
+
Returns:
|
|
315
|
+
The cached step holding the proven regenerated code.
|
|
316
|
+
|
|
317
|
+
Raises:
|
|
318
|
+
IncurableStepError: the healing attempt budget is exhausted; the
|
|
319
|
+
verdict stays None — the calling healer attaches its entry
|
|
320
|
+
verdict — and the code field carries the last record's code.
|
|
321
|
+
LLMUnavailableError: the provider service failed after the
|
|
322
|
+
bounded transport retries; no engine retry.
|
|
323
|
+
ComplianceVerdictError: the compliance verdict of a green
|
|
324
|
+
candidate did not parse; nothing is cached.
|
|
325
|
+
"""
|
|
326
|
+
return self._healing_loop(
|
|
327
|
+
identity,
|
|
328
|
+
prepared,
|
|
329
|
+
step_type,
|
|
330
|
+
previous_steps,
|
|
331
|
+
group_prompt,
|
|
332
|
+
page,
|
|
333
|
+
attempt_history,
|
|
334
|
+
recommendation,
|
|
335
|
+
window,
|
|
336
|
+
memory,
|
|
337
|
+
)
|
|
338
|
+
|
|
339
|
+
def _generation_loop( # noqa: PLR0913, PLR0917 — the shared attempt loop with its fixed inputs
|
|
340
|
+
self,
|
|
341
|
+
identity: StepIdentity,
|
|
342
|
+
prepared: PreparedStep,
|
|
343
|
+
step_type: str,
|
|
344
|
+
previous_steps: list[ScenarioStep],
|
|
345
|
+
group_prompt: str | None,
|
|
346
|
+
page: PageFacade,
|
|
347
|
+
history: list[StepAttempt],
|
|
348
|
+
window: SettleWindow,
|
|
349
|
+
memory: StepMemory,
|
|
350
|
+
) -> CachedStep:
|
|
351
|
+
"""Run the generation pool loop until a candidate works or the budget runs out.
|
|
352
|
+
|
|
353
|
+
Args:
|
|
354
|
+
identity: the address of the step.
|
|
355
|
+
prepared: the render product of the step — instruction, input
|
|
356
|
+
bindings, result declarations.
|
|
357
|
+
step_type: action or assertion — carried into every request.
|
|
358
|
+
previous_steps: the typed scenario records of the previous steps
|
|
359
|
+
of the test, in execution order.
|
|
360
|
+
group_prompt: the group prompt of the current step's group;
|
|
361
|
+
None — an ordinary step, the ordinary paths byte-identical;
|
|
362
|
+
non-empty — a group step, the classification points
|
|
363
|
+
suppressed.
|
|
364
|
+
page: the live page facade the candidates run against.
|
|
365
|
+
history: the per-step attempt history this loop grows — every
|
|
366
|
+
attempt that does not produce a cached step appends its
|
|
367
|
+
record; the green attempt never records.
|
|
368
|
+
window: the settle window of the current step execution.
|
|
369
|
+
memory: the test memory — the accepted candidate's validated
|
|
370
|
+
captures publish after validation and the gate.
|
|
371
|
+
|
|
372
|
+
Returns:
|
|
373
|
+
The cached step holding the proven code.
|
|
374
|
+
|
|
375
|
+
Raises:
|
|
376
|
+
ProductDefectError: a failed candidate check classified as a
|
|
377
|
+
genuine product defect.
|
|
378
|
+
IncurableStepError: the generation attempt budget is exhausted,
|
|
379
|
+
or a failure classified incurable.
|
|
380
|
+
LLMUnavailableError: the provider service failed after the
|
|
381
|
+
bounded transport retries; no engine retry.
|
|
382
|
+
ComplianceVerdictError: the compliance verdict of a green
|
|
383
|
+
candidate did not parse; nothing is cached.
|
|
384
|
+
"""
|
|
385
|
+
attempt = 0 # every LLM request of this pool run — loop attempts and funded regenerations
|
|
386
|
+
standing: ComplianceFinding | None = None # the high finding the retries still carry
|
|
387
|
+
|
|
388
|
+
while True:
|
|
389
|
+
if not self._budgets.try_generation(identity):
|
|
390
|
+
return self._exhaustion_outcome(
|
|
391
|
+
identity,
|
|
392
|
+
prepared,
|
|
393
|
+
step_type,
|
|
394
|
+
previous_steps,
|
|
395
|
+
group_prompt,
|
|
396
|
+
page,
|
|
397
|
+
history,
|
|
398
|
+
window,
|
|
399
|
+
memory,
|
|
400
|
+
attempt,
|
|
401
|
+
standing,
|
|
402
|
+
)
|
|
403
|
+
|
|
404
|
+
attempt += 1
|
|
405
|
+
self._emit_generation_started(prepared.instruction, attempt)
|
|
406
|
+
|
|
407
|
+
code = self._request(prepared, step_type, previous_steps, group_prompt, page, history, recommendation=None)
|
|
408
|
+
|
|
409
|
+
url_before = _read_url(page)
|
|
410
|
+
try:
|
|
411
|
+
with bind_step_inputs(memory.snapshot(), prepared.inputs):
|
|
412
|
+
result = settle(run_step_code, code, page, window)
|
|
413
|
+
except AssertionError as check_failure:
|
|
414
|
+
# failed check survived the window — the decision table, never blind retries
|
|
415
|
+
url_after = _read_url(page)
|
|
416
|
+
# full text, no prefix — the type is the semantics
|
|
417
|
+
history.append(
|
|
418
|
+
_record(code, format_step_error(check_failure), OUTCOME_FAILED_CHECK, url_before, url_after)
|
|
419
|
+
)
|
|
420
|
+
return self._failed_check_outcome(
|
|
421
|
+
identity,
|
|
422
|
+
prepared,
|
|
423
|
+
step_type,
|
|
424
|
+
previous_steps,
|
|
425
|
+
group_prompt,
|
|
426
|
+
page,
|
|
427
|
+
history,
|
|
428
|
+
code,
|
|
429
|
+
format_step_error(check_failure),
|
|
430
|
+
window,
|
|
431
|
+
memory,
|
|
432
|
+
attempt,
|
|
433
|
+
)
|
|
434
|
+
except Exception as candidate_error: # other candidate failures heal via retry
|
|
435
|
+
url_after = _read_url(page)
|
|
436
|
+
history.append(
|
|
437
|
+
_record(code, format_step_error(candidate_error), OUTCOME_EXECUTION_FAILED, url_before, url_after)
|
|
438
|
+
)
|
|
439
|
+
standing = None # a real candidate failure replaces the standing violation
|
|
440
|
+
else:
|
|
441
|
+
url_after = _read_url(page)
|
|
442
|
+
captures, violation = _validated_captures(prepared, result)
|
|
443
|
+
|
|
444
|
+
if violation is not None: # the result contract failed — the identical failed-check channel
|
|
445
|
+
history.append(_record(code, violation, OUTCOME_FAILED_CHECK, url_before, url_after))
|
|
446
|
+
return self._failed_check_outcome(
|
|
447
|
+
identity,
|
|
448
|
+
prepared,
|
|
449
|
+
step_type,
|
|
450
|
+
previous_steps,
|
|
451
|
+
group_prompt,
|
|
452
|
+
page,
|
|
453
|
+
history,
|
|
454
|
+
code,
|
|
455
|
+
violation,
|
|
456
|
+
window,
|
|
457
|
+
memory,
|
|
458
|
+
attempt,
|
|
459
|
+
)
|
|
460
|
+
|
|
461
|
+
# the gate sits outside every exception-swallowing try: its hard failures propagate
|
|
462
|
+
findings = check_step_compliance(self._config, self._provider, prepared, step_type, code, history)
|
|
463
|
+
high = _high_finding(findings)
|
|
464
|
+
if high is not None: # the attempt failed on the violation — retry targeted at it
|
|
465
|
+
history.append(
|
|
466
|
+
_record(code, _violation_text(high), OUTCOME_COMPLIANCE_BLOCKED, url_before, url_after)
|
|
467
|
+
)
|
|
468
|
+
standing = high
|
|
469
|
+
continue
|
|
470
|
+
if findings:
|
|
471
|
+
_medium_warning(prepared.instruction, findings)
|
|
472
|
+
|
|
473
|
+
memory.publish(captures) # publication exactly once — after validation and the gate
|
|
474
|
+
return self._store(identity, code)
|
|
475
|
+
|
|
476
|
+
def _healing_loop( # noqa: PLR0913, PLR0917 — the shared attempt loop with its fixed inputs
|
|
477
|
+
self,
|
|
478
|
+
identity: StepIdentity,
|
|
479
|
+
prepared: PreparedStep,
|
|
480
|
+
step_type: str,
|
|
481
|
+
previous_steps: list[ScenarioStep],
|
|
482
|
+
group_prompt: str | None,
|
|
483
|
+
page: PageFacade,
|
|
484
|
+
history: list[StepAttempt],
|
|
485
|
+
recommendation: str,
|
|
486
|
+
window: SettleWindow,
|
|
487
|
+
memory: StepMemory,
|
|
488
|
+
) -> CachedStep:
|
|
489
|
+
"""Run the healing pool loop until a candidate works or the budget runs out.
|
|
490
|
+
|
|
491
|
+
Args:
|
|
492
|
+
identity: the address of the step.
|
|
493
|
+
prepared: the render product of the row or healed step —
|
|
494
|
+
instruction, input bindings, result declarations.
|
|
495
|
+
step_type: action or assertion — carried into every request.
|
|
496
|
+
previous_steps: the typed scenario records of the previous steps
|
|
497
|
+
of the test, in execution order.
|
|
498
|
+
group_prompt: the group prompt of the row step's group — the
|
|
499
|
+
recovery row passes it; non-empty — the request carries the
|
|
500
|
+
framing; None — an ordinary regeneration.
|
|
501
|
+
page: the live page facade the candidates run against.
|
|
502
|
+
history: the anchored per-step attempt history — record 0 stays
|
|
503
|
+
untouched at index 0; every failed attempt, a failed check
|
|
504
|
+
included, appends its record and retries.
|
|
505
|
+
recommendation: the diagnosis of the classification that launched
|
|
506
|
+
the healing; rendered into every request.
|
|
507
|
+
window: the settle window of the current step execution.
|
|
508
|
+
memory: the test memory — the accepted candidate's validated
|
|
509
|
+
captures publish after validation and the gate.
|
|
510
|
+
|
|
511
|
+
Returns:
|
|
512
|
+
The cached step holding the proven code.
|
|
513
|
+
|
|
514
|
+
Raises:
|
|
515
|
+
IncurableStepError: the healing attempt budget is exhausted; no
|
|
516
|
+
classification inside the loop — the calling healer attaches
|
|
517
|
+
the verdict it already holds.
|
|
518
|
+
LLMUnavailableError: the provider service failed after the
|
|
519
|
+
bounded transport retries; no engine retry.
|
|
520
|
+
ComplianceVerdictError: the compliance verdict of a green
|
|
521
|
+
candidate did not parse; nothing is cached.
|
|
522
|
+
"""
|
|
523
|
+
attempt = 0
|
|
524
|
+
|
|
525
|
+
while True:
|
|
526
|
+
if not self._budgets.try_healing(identity):
|
|
527
|
+
# colon-free authored reason; the last record carries the terminal-failure facts
|
|
528
|
+
code, error = _last_facts(history)
|
|
529
|
+
raise IncurableStepError(prepared.instruction, "healing attempt budget exhausted", error, code=code)
|
|
530
|
+
|
|
531
|
+
attempt += 1
|
|
532
|
+
self._emit_generation_started(prepared.instruction, attempt)
|
|
533
|
+
|
|
534
|
+
code = self._request(prepared, step_type, previous_steps, group_prompt, page, history, recommendation)
|
|
535
|
+
|
|
536
|
+
url_before = _read_url(page)
|
|
537
|
+
try:
|
|
538
|
+
with bind_step_inputs(memory.snapshot(), prepared.inputs):
|
|
539
|
+
result = settle(run_step_code, code, page, window)
|
|
540
|
+
except AssertionError as check_failure: # failed checks included — the entry classification guards
|
|
541
|
+
url_after = _read_url(page)
|
|
542
|
+
history.append(
|
|
543
|
+
_record(code, format_step_error(check_failure), OUTCOME_FAILED_CHECK, url_before, url_after)
|
|
544
|
+
)
|
|
545
|
+
continue
|
|
546
|
+
except Exception as candidate_error:
|
|
547
|
+
url_after = _read_url(page)
|
|
548
|
+
history.append(
|
|
549
|
+
_record(code, format_step_error(candidate_error), OUTCOME_EXECUTION_FAILED, url_before, url_after)
|
|
550
|
+
)
|
|
551
|
+
continue
|
|
552
|
+
url_after = _read_url(page)
|
|
553
|
+
|
|
554
|
+
captures, violation = _validated_captures(prepared, result)
|
|
555
|
+
|
|
556
|
+
if violation is not None: # a result-contract violation is a failed attempt like any other
|
|
557
|
+
history.append(_record(code, violation, OUTCOME_FAILED_CHECK, url_before, url_after))
|
|
558
|
+
continue
|
|
559
|
+
|
|
560
|
+
# the gate sits outside every exception-swallowing try: its hard failures propagate
|
|
561
|
+
findings = check_step_compliance(self._config, self._provider, prepared, step_type, code, history)
|
|
562
|
+
high = _high_finding(findings)
|
|
563
|
+
if high is not None: # the attempt failed on the violation — retry targeted at it
|
|
564
|
+
history.append(_record(code, _violation_text(high), OUTCOME_COMPLIANCE_BLOCKED, url_before, url_after))
|
|
565
|
+
continue
|
|
566
|
+
if findings:
|
|
567
|
+
_medium_warning(prepared.instruction, findings)
|
|
568
|
+
|
|
569
|
+
memory.publish(captures) # publication exactly once — after validation and the gate
|
|
570
|
+
return self._store(identity, code)
|
|
571
|
+
|
|
572
|
+
def _failed_check_outcome( # noqa: PLR0913, PLR0917 — the decision table of one failed candidate check
|
|
573
|
+
self,
|
|
574
|
+
identity: StepIdentity,
|
|
575
|
+
prepared: PreparedStep,
|
|
576
|
+
step_type: str,
|
|
577
|
+
previous_steps: list[ScenarioStep],
|
|
578
|
+
group_prompt: str | None,
|
|
579
|
+
page: PageFacade,
|
|
580
|
+
history: list[StepAttempt],
|
|
581
|
+
code: str,
|
|
582
|
+
error_field: str,
|
|
583
|
+
window: SettleWindow,
|
|
584
|
+
memory: StepMemory,
|
|
585
|
+
attempt: int,
|
|
586
|
+
) -> CachedStep:
|
|
587
|
+
"""Decide the bounded-healing outcome of a failed candidate check.
|
|
588
|
+
|
|
589
|
+
Args:
|
|
590
|
+
identity: the address of the step — the healing funding key.
|
|
591
|
+
prepared: the render product of the failed step.
|
|
592
|
+
step_type: action or assertion — carried into the funded request.
|
|
593
|
+
previous_steps: the typed scenario records of the previous steps
|
|
594
|
+
of the test, in execution order.
|
|
595
|
+
group_prompt: the group prompt of the current step's group;
|
|
596
|
+
non-empty — the suppressed-classification group raise;
|
|
597
|
+
None — the ordinary decision table.
|
|
598
|
+
page: the live page facade of the test.
|
|
599
|
+
history: the per-step attempt history — the failed check's
|
|
600
|
+
record already appended; the funded request carries it grown.
|
|
601
|
+
code: the code of the failed candidate.
|
|
602
|
+
error_field: the full failure text of the failed check — a page
|
|
603
|
+
assertion or the deterministic result-contract violation.
|
|
604
|
+
window: the settle window of the current step execution.
|
|
605
|
+
memory: the test memory — the funded regeneration publishes at
|
|
606
|
+
its own acceptance point.
|
|
607
|
+
attempt: the number of LLM requests the pool run made so far.
|
|
608
|
+
|
|
609
|
+
Returns:
|
|
610
|
+
The healed step when the funded regeneration worked.
|
|
611
|
+
|
|
612
|
+
Raises:
|
|
613
|
+
ProductDefectError: the entry or final verdict says product defect.
|
|
614
|
+
IncurableStepError: the verdict says incurable, the healing
|
|
615
|
+
funding is refused, or the repeat failure stays terminal —
|
|
616
|
+
one final classification decides the terminal kind only; a
|
|
617
|
+
group step raises the unclassified variant — the group
|
|
618
|
+
recovery decides.
|
|
619
|
+
"""
|
|
620
|
+
if group_prompt:
|
|
621
|
+
# a group step suppresses the classification and the funded regeneration — the
|
|
622
|
+
# group recovery decides; colon-free authored reason, no verdict attached
|
|
623
|
+
raise IncurableStepError(
|
|
624
|
+
prepared.instruction,
|
|
625
|
+
"group step check failed — the group recovery decides",
|
|
626
|
+
error_field,
|
|
627
|
+
code=code,
|
|
628
|
+
) from None
|
|
629
|
+
|
|
630
|
+
reason = f"candidate check failed — {_first_line(error_field)}"
|
|
631
|
+
verdict = self._classify(prepared.instruction, code, error_field, page)
|
|
632
|
+
if verdict is None: # quiet skip — the failed check itself is the primary signal
|
|
633
|
+
raise IncurableStepError(prepared.instruction, reason, error_field, code=code) from None
|
|
634
|
+
if verdict.category == "product_defect":
|
|
635
|
+
raise ProductDefectError(prepared.instruction, verdict.explanation, error_field, verdict) from None
|
|
636
|
+
if verdict.category == "incurable":
|
|
637
|
+
raise IncurableStepError(prepared.instruction, reason, error_field, code=code, verdict=verdict) from None
|
|
638
|
+
|
|
639
|
+
# rot | fixable — exactly one healing-funded regeneration
|
|
640
|
+
if not self._budgets.try_healing(identity):
|
|
641
|
+
raise IncurableStepError(
|
|
642
|
+
prepared.instruction, "healing attempt budget exhausted", error_field, code=code, verdict=verdict
|
|
643
|
+
) from None
|
|
644
|
+
|
|
645
|
+
healed, failed_code, failure_text, repeat_was_check = self._funded_regeneration(
|
|
646
|
+
identity,
|
|
647
|
+
prepared,
|
|
648
|
+
step_type,
|
|
649
|
+
previous_steps,
|
|
650
|
+
page,
|
|
651
|
+
history,
|
|
652
|
+
verdict.recommendation,
|
|
653
|
+
window,
|
|
654
|
+
memory,
|
|
655
|
+
attempt,
|
|
656
|
+
)
|
|
657
|
+
if healed is not None:
|
|
658
|
+
return healed
|
|
659
|
+
|
|
660
|
+
# repeat failure — one final classification deciding the terminal kind only
|
|
661
|
+
final = self._classify(prepared.instruction, failed_code, failure_text, page)
|
|
662
|
+
if final is not None and final.category == "product_defect":
|
|
663
|
+
raise ProductDefectError(prepared.instruction, final.explanation, failure_text, final) from None
|
|
664
|
+
if repeat_was_check:
|
|
665
|
+
# an assertion repeat names the failed check
|
|
666
|
+
repeat_reason = f"candidate check failed — {_first_line(failure_text)}"
|
|
667
|
+
else:
|
|
668
|
+
# a compliance block or a failed execution is a candidate failure, quiet skip or
|
|
669
|
+
# not — the violation text carries colons; the first-line contract forbids them
|
|
670
|
+
repeat_reason = f"candidate failed — {_reason_safe(failure_text)}"
|
|
671
|
+
raise IncurableStepError(
|
|
672
|
+
prepared.instruction, repeat_reason, failure_text, code=failed_code, verdict=final
|
|
673
|
+
) from None
|
|
674
|
+
|
|
675
|
+
def _exhaustion_outcome( # noqa: PLR0913, PLR0917 — the decision table of the refused generation pool
|
|
676
|
+
self,
|
|
677
|
+
identity: StepIdentity,
|
|
678
|
+
prepared: PreparedStep,
|
|
679
|
+
step_type: str,
|
|
680
|
+
previous_steps: list[ScenarioStep],
|
|
681
|
+
group_prompt: str | None,
|
|
682
|
+
page: PageFacade,
|
|
683
|
+
history: list[StepAttempt],
|
|
684
|
+
window: SettleWindow,
|
|
685
|
+
memory: StepMemory,
|
|
686
|
+
attempt: int,
|
|
687
|
+
standing: ComplianceFinding | None,
|
|
688
|
+
) -> CachedStep:
|
|
689
|
+
"""Decide the outcome of a refused generation attempt.
|
|
690
|
+
|
|
691
|
+
Args:
|
|
692
|
+
identity: the address of the step — the healing funding key.
|
|
693
|
+
prepared: the render product of the failed step.
|
|
694
|
+
step_type: action or assertion — carried into the funded request.
|
|
695
|
+
previous_steps: the typed scenario records of the previous steps
|
|
696
|
+
of the test, in execution order.
|
|
697
|
+
group_prompt: the group prompt of the current step's group;
|
|
698
|
+
non-empty — the suppressed-classification group raise;
|
|
699
|
+
None — the ordinary exhaustion table.
|
|
700
|
+
page: the live page facade of the test.
|
|
701
|
+
history: the per-step attempt history of the pool run — its last
|
|
702
|
+
record carries the last candidate's facts; empty — no
|
|
703
|
+
candidate ever existed.
|
|
704
|
+
window: the settle window of the current step execution.
|
|
705
|
+
memory: the test memory — the funded regeneration publishes at
|
|
706
|
+
its own acceptance point.
|
|
707
|
+
attempt: the number of LLM requests the pool run made so far.
|
|
708
|
+
standing: the high compliance finding the last retries carried;
|
|
709
|
+
None — the exhaustion is not a standing violation.
|
|
710
|
+
|
|
711
|
+
Returns:
|
|
712
|
+
The healed step when the funded regeneration worked.
|
|
713
|
+
|
|
714
|
+
Raises:
|
|
715
|
+
ProductDefectError: the verdict says product defect.
|
|
716
|
+
IncurableStepError: the generation attempt budget is exhausted —
|
|
717
|
+
the finding named when a high one stands, the verdict says
|
|
718
|
+
incurable, or the healing funding is refused; a repeat
|
|
719
|
+
failure is terminal without reclassification; a group step
|
|
720
|
+
raises the unclassified variant — the group recovery decides.
|
|
721
|
+
"""
|
|
722
|
+
if group_prompt:
|
|
723
|
+
# a group step suppresses the exhaustion classification and the funded regeneration —
|
|
724
|
+
# the group recovery decides; the last record carries the terminal-failure facts
|
|
725
|
+
code, error = _last_facts(history)
|
|
726
|
+
raise IncurableStepError(
|
|
727
|
+
prepared.instruction,
|
|
728
|
+
"group step generation budget exhausted — the group recovery decides",
|
|
729
|
+
error,
|
|
730
|
+
code=code,
|
|
731
|
+
) from None
|
|
732
|
+
|
|
733
|
+
if standing is not None:
|
|
734
|
+
# the verdict is built from the finding — no classification, no regrant of a funded attempt
|
|
735
|
+
verdict = FailureVerdict(
|
|
736
|
+
category="incurable",
|
|
737
|
+
explanation=f"{standing.instruction} — {standing.explanation}",
|
|
738
|
+
recommendation=f"satisfy the finding in the step code: {standing.instruction}",
|
|
739
|
+
)
|
|
740
|
+
raise IncurableStepError(
|
|
741
|
+
prepared.instruction,
|
|
742
|
+
f"generation attempt budget exhausted — {_reason_safe(standing.instruction)}",
|
|
743
|
+
_violation_text(standing),
|
|
744
|
+
code=_last_facts(history)[0],
|
|
745
|
+
verdict=verdict,
|
|
746
|
+
) from None
|
|
747
|
+
|
|
748
|
+
reason = "generation attempt budget exhausted"
|
|
749
|
+
if not history:
|
|
750
|
+
raise IncurableStepError(prepared.instruction, reason, "", code="") from None # nothing to classify
|
|
751
|
+
|
|
752
|
+
code, error = _last_facts(history) # the last failed attempt's record carries the facts
|
|
753
|
+
verdict = self._classify(prepared.instruction, code, error, page)
|
|
754
|
+
if verdict is None: # quiet skip — the budget failure is the primary signal
|
|
755
|
+
raise IncurableStepError(prepared.instruction, reason, error, code=code) from None
|
|
756
|
+
if verdict.category == "product_defect":
|
|
757
|
+
raise ProductDefectError(prepared.instruction, verdict.explanation, error, verdict) from None
|
|
758
|
+
if verdict.category == "incurable":
|
|
759
|
+
raise IncurableStepError(prepared.instruction, reason, error, code=code, verdict=verdict) from None
|
|
760
|
+
|
|
761
|
+
# rot | fixable — one extra healing-funded regeneration
|
|
762
|
+
if not self._budgets.try_healing(identity):
|
|
763
|
+
raise IncurableStepError(
|
|
764
|
+
prepared.instruction, "healing attempt budget exhausted", error, code=code, verdict=verdict
|
|
765
|
+
) from None
|
|
766
|
+
|
|
767
|
+
healed, failed_code, failure_text, _repeat_was_check = self._funded_regeneration(
|
|
768
|
+
identity,
|
|
769
|
+
prepared,
|
|
770
|
+
step_type,
|
|
771
|
+
previous_steps,
|
|
772
|
+
page,
|
|
773
|
+
history,
|
|
774
|
+
verdict.recommendation,
|
|
775
|
+
window,
|
|
776
|
+
memory,
|
|
777
|
+
attempt,
|
|
778
|
+
)
|
|
779
|
+
if healed is not None:
|
|
780
|
+
return healed
|
|
781
|
+
|
|
782
|
+
# repeat failure — terminal, no reclassification; the verdict of the entry classification travels
|
|
783
|
+
raise IncurableStepError(
|
|
784
|
+
prepared.instruction, reason, failure_text, code=failed_code, verdict=verdict
|
|
785
|
+
) from None
|
|
786
|
+
|
|
787
|
+
def _funded_regeneration( # noqa: PLR0913, PLR0917 — the single healing-funded request of the bounded healing
|
|
788
|
+
self,
|
|
789
|
+
identity: StepIdentity,
|
|
790
|
+
prepared: PreparedStep,
|
|
791
|
+
step_type: str,
|
|
792
|
+
previous_steps: list[ScenarioStep],
|
|
793
|
+
page: PageFacade,
|
|
794
|
+
history: list[StepAttempt],
|
|
795
|
+
recommendation: str,
|
|
796
|
+
window: SettleWindow,
|
|
797
|
+
memory: StepMemory,
|
|
798
|
+
attempt: int,
|
|
799
|
+
) -> tuple[CachedStep | None, str, str, bool]:
|
|
800
|
+
"""Run the one healing-funded regeneration request of the bounded healing.
|
|
801
|
+
|
|
802
|
+
Args:
|
|
803
|
+
identity: the address of the step — the identity of the stored step.
|
|
804
|
+
prepared: the render product of the failed step.
|
|
805
|
+
step_type: action or assertion — carried into the request.
|
|
806
|
+
previous_steps: the typed scenario records of the previous steps
|
|
807
|
+
of the test, in execution order.
|
|
808
|
+
page: the live page facade the candidate runs against.
|
|
809
|
+
history: the per-step attempt history — the request carries it
|
|
810
|
+
grown; the funded attempt's own failure appends its record.
|
|
811
|
+
recommendation: the classification diagnosis carried by the request.
|
|
812
|
+
window: the settle window of the current step execution — shared
|
|
813
|
+
with the loop, the same step execution.
|
|
814
|
+
memory: the test memory — the funded acceptance publishes its
|
|
815
|
+
validated captures like any acceptance.
|
|
816
|
+
attempt: the number of LLM requests the pool run made so far.
|
|
817
|
+
|
|
818
|
+
Returns:
|
|
819
|
+
The stored healed step with empty failure facts on success; None
|
|
820
|
+
with the failed code, the formatted failure text and whether the
|
|
821
|
+
failure was a failed check. A single inline request — never the
|
|
822
|
+
retrying regenerate loop; a high compliance finding counts as a
|
|
823
|
+
candidate failure, never a check; a result-contract violation is
|
|
824
|
+
a check.
|
|
825
|
+
|
|
826
|
+
Raises:
|
|
827
|
+
LLMUnavailableError: the provider request failed after the
|
|
828
|
+
bounded transport retries; no engine retry, no final
|
|
829
|
+
classification.
|
|
830
|
+
ComplianceVerdictError: the compliance verdict of a green funded
|
|
831
|
+
candidate did not parse; nothing is cached.
|
|
832
|
+
"""
|
|
833
|
+
attempt += 1
|
|
834
|
+
self._emit_generation_started(prepared.instruction, attempt)
|
|
835
|
+
|
|
836
|
+
# the funded path only runs for an ordinary step — the group branches raise before it
|
|
837
|
+
code = self._request(prepared, step_type, previous_steps, None, page, history, recommendation)
|
|
838
|
+
url_before = _read_url(page)
|
|
839
|
+
try:
|
|
840
|
+
with bind_step_inputs(memory.snapshot(), prepared.inputs):
|
|
841
|
+
result = settle(run_step_code, code, page, window)
|
|
842
|
+
except AssertionError as check_failure:
|
|
843
|
+
url_after = _read_url(page)
|
|
844
|
+
history.append(_record(code, format_step_error(check_failure), OUTCOME_FAILED_CHECK, url_before, url_after))
|
|
845
|
+
return None, code, format_step_error(check_failure), True
|
|
846
|
+
except Exception as failure:
|
|
847
|
+
url_after = _read_url(page)
|
|
848
|
+
history.append(_record(code, format_step_error(failure), OUTCOME_EXECUTION_FAILED, url_before, url_after))
|
|
849
|
+
return None, code, format_step_error(failure), False
|
|
850
|
+
url_after = _read_url(page)
|
|
851
|
+
|
|
852
|
+
captures, violation = _validated_captures(prepared, result)
|
|
853
|
+
|
|
854
|
+
if violation is not None: # the result contract failed — a failed check like a page assertion
|
|
855
|
+
history.append(_record(code, violation, OUTCOME_FAILED_CHECK, url_before, url_after))
|
|
856
|
+
return None, code, violation, True
|
|
857
|
+
|
|
858
|
+
# the gate sits outside the settle try: its hard failures propagate
|
|
859
|
+
findings = check_step_compliance(self._config, self._provider, prepared, step_type, code, history)
|
|
860
|
+
high = _high_finding(findings)
|
|
861
|
+
if high is not None: # a compliance block is a candidate failure, not a check
|
|
862
|
+
history.append(_record(code, _violation_text(high), OUTCOME_COMPLIANCE_BLOCKED, url_before, url_after))
|
|
863
|
+
return None, code, _violation_text(high), False
|
|
864
|
+
if findings:
|
|
865
|
+
_medium_warning(prepared.instruction, findings)
|
|
866
|
+
|
|
867
|
+
memory.publish(captures) # publication exactly once — after validation and the gate
|
|
868
|
+
return self._store(identity, code), code, "", False
|
|
869
|
+
|
|
870
|
+
def _request( # noqa: PLR0913, PLR0917 — the fixed request inputs of the port signature
|
|
871
|
+
self,
|
|
872
|
+
prepared: PreparedStep,
|
|
873
|
+
step_type: str,
|
|
874
|
+
previous_steps: list[ScenarioStep],
|
|
875
|
+
group_prompt: str | None,
|
|
876
|
+
page: PageFacade,
|
|
877
|
+
history: list[StepAttempt],
|
|
878
|
+
recommendation: str | None,
|
|
879
|
+
) -> str:
|
|
880
|
+
"""Collect the request inputs and ask the provider for one candidate.
|
|
881
|
+
|
|
882
|
+
Args:
|
|
883
|
+
prepared: the render product of the step — the instruction with
|
|
884
|
+
its INPUTS and RESULTS blocks; the raw template sentence
|
|
885
|
+
never reaches the request.
|
|
886
|
+
step_type: action or assertion — carried into every request.
|
|
887
|
+
previous_steps: the typed scenario records of the previous steps
|
|
888
|
+
of the test, in execution order — the provider renders the
|
|
889
|
+
PREVIOUS STEPS block from them, the group entries marked.
|
|
890
|
+
group_prompt: the group prompt of the current step's group;
|
|
891
|
+
non-empty — the provider renders the GROUP PROMPT block
|
|
892
|
+
immediately before PREVIOUS STEPS; None — no block, the
|
|
893
|
+
ordinary request byte-identical to today.
|
|
894
|
+
page: the live page facade of the test.
|
|
895
|
+
history: the per-step attempt history — rendered record by
|
|
896
|
+
record into the HISTORY block of the request.
|
|
897
|
+
recommendation: the classification diagnosis of the request, if any.
|
|
898
|
+
|
|
899
|
+
Returns:
|
|
900
|
+
The generated step code of the fixed form.
|
|
901
|
+
"""
|
|
902
|
+
snapshot = page.aria_snapshot()
|
|
903
|
+
screenshot = page.screenshot() if self._config.send_screenshots else None
|
|
904
|
+
|
|
905
|
+
return self._provider.generate_step_code(
|
|
906
|
+
prompt=SYSTEM_PROMPT,
|
|
907
|
+
user_instructions=self._config.generation_prompt,
|
|
908
|
+
instruction=prepared.instruction,
|
|
909
|
+
step_type=step_type,
|
|
910
|
+
previous_steps=previous_steps,
|
|
911
|
+
group_prompt=group_prompt,
|
|
912
|
+
inputs=prepared.inputs,
|
|
913
|
+
declarations=prepared.declarations,
|
|
914
|
+
snapshot=snapshot,
|
|
915
|
+
page_url=_read_url(page), # the guarded read — an empty string renders no PAGE URL line
|
|
916
|
+
screenshot=screenshot,
|
|
917
|
+
cheat_sheet=CHEAT_SHEET,
|
|
918
|
+
attempt_history=[record.render() for record in history],
|
|
919
|
+
recommendation=recommendation,
|
|
920
|
+
guidance=None, # steering-only input — the engine never carries guidance
|
|
921
|
+
)
|
|
922
|
+
|
|
923
|
+
def _emit_generation_started(self, step_text: str, attempt: int) -> None:
|
|
924
|
+
"""Report the start of one LLM attempt — once per provider request, never per settle re-execution.
|
|
925
|
+
|
|
926
|
+
Args:
|
|
927
|
+
step_text: the sentence of the step.
|
|
928
|
+
attempt: the 1-based ordinal of the LLM request in the pool run.
|
|
929
|
+
"""
|
|
930
|
+
self._reporter.emit("on_generation_started", {"step_text": step_text, "attempt": attempt})
|
|
931
|
+
|
|
932
|
+
def _store(self, identity: StepIdentity, code: str) -> CachedStep:
|
|
933
|
+
"""Save the proven candidate as the cached step of the identity.
|
|
934
|
+
|
|
935
|
+
Args:
|
|
936
|
+
identity: the address of the step.
|
|
937
|
+
code: the proven step code.
|
|
938
|
+
|
|
939
|
+
Returns:
|
|
940
|
+
The stored cached step.
|
|
941
|
+
"""
|
|
942
|
+
step = CachedStep(
|
|
943
|
+
identity=identity,
|
|
944
|
+
code=code,
|
|
945
|
+
created_at=date.today().isoformat(), # noqa: DTZ011 — calendar date of step creation
|
|
946
|
+
)
|
|
947
|
+
self._cache.save(step)
|
|
948
|
+
|
|
949
|
+
return step
|
|
950
|
+
|
|
951
|
+
def _classify(self, step_text: str, code: str, error: str, page: PageFacade) -> FailureVerdict | None:
|
|
952
|
+
"""Classify a failure through the shared routine with the quiet skip.
|
|
953
|
+
|
|
954
|
+
Args:
|
|
955
|
+
step_text: the sentence of the failed step.
|
|
956
|
+
code: the code of the last candidate.
|
|
957
|
+
error: the failure description of the candidate.
|
|
958
|
+
page: the live page facade of the test.
|
|
959
|
+
|
|
960
|
+
Returns:
|
|
961
|
+
The verdict of the classification, or ``None`` when the LLM was
|
|
962
|
+
unavailable — the quiet skip logs a WARNING; an already-decided
|
|
963
|
+
failure never waits for the LLM.
|
|
964
|
+
"""
|
|
965
|
+
try:
|
|
966
|
+
classification = classify_step_failure(self._config, self._provider, step_text, code, error, page)
|
|
967
|
+
except LLMUnavailableError:
|
|
968
|
+
logger.warning("verdict skipped: llm unavailable")
|
|
969
|
+
return None
|
|
970
|
+
|
|
971
|
+
return _verdict(classification)
|
|
972
|
+
|
|
973
|
+
|
|
974
|
+
def _first_line(text: str) -> str:
|
|
975
|
+
"""Return the first line of a failure text — the authored-reason tail.
|
|
976
|
+
|
|
977
|
+
Args:
|
|
978
|
+
text: the full failure text.
|
|
979
|
+
|
|
980
|
+
Returns:
|
|
981
|
+
The text up to the first newline; the whole text when single-line.
|
|
982
|
+
"""
|
|
983
|
+
return text.partition("\n")[0]
|
|
984
|
+
|
|
985
|
+
|
|
986
|
+
def _reason_safe(instruction: str) -> str:
|
|
987
|
+
"""Make an instruction quote safe for an authored reason — the first-line contract.
|
|
988
|
+
|
|
989
|
+
Args:
|
|
990
|
+
instruction: the violated instruction text, quoted from the project's
|
|
991
|
+
user configuration; may contain colons and newlines.
|
|
992
|
+
|
|
993
|
+
Returns:
|
|
994
|
+
The first line of the instruction with every colon replaced by a
|
|
995
|
+
space — the reason line stays colon-free.
|
|
996
|
+
"""
|
|
997
|
+
return _first_line(instruction).replace(":", " ")
|
|
998
|
+
|
|
999
|
+
|
|
1000
|
+
def _high_finding(findings: list[ComplianceFinding]) -> ComplianceFinding | None:
|
|
1001
|
+
"""Return the first high finding of a compliance verdict — either dimension blocks.
|
|
1002
|
+
|
|
1003
|
+
Args:
|
|
1004
|
+
findings: the findings of the verdict, in the verdict's own ordering.
|
|
1005
|
+
|
|
1006
|
+
Returns:
|
|
1007
|
+
The first finding of priority high, or ``None`` when the verdict
|
|
1008
|
+
holds none — the gate re-runs on the next candidate and catches any
|
|
1009
|
+
remaining violation.
|
|
1010
|
+
"""
|
|
1011
|
+
return next((finding for finding in findings if finding.priority == "high"), None)
|
|
1012
|
+
|
|
1013
|
+
|
|
1014
|
+
def _violation_text(finding: ComplianceFinding) -> str:
|
|
1015
|
+
"""Render the violation text of a high finding — the record's error field.
|
|
1016
|
+
|
|
1017
|
+
Args:
|
|
1018
|
+
finding: the high finding of either dimension that blocked the candidate.
|
|
1019
|
+
|
|
1020
|
+
Returns:
|
|
1021
|
+
The dimension, the named instruction or step fragment, and its
|
|
1022
|
+
explanation in the fixed violation wording.
|
|
1023
|
+
"""
|
|
1024
|
+
return f"{finding.dimension} violation: {finding.instruction} — {finding.explanation}"
|
|
1025
|
+
|
|
1026
|
+
|
|
1027
|
+
def _medium_warning(step_text: str, findings: list[ComplianceFinding]) -> None:
|
|
1028
|
+
"""Log the medium and low findings a green candidate passed with.
|
|
1029
|
+
|
|
1030
|
+
Args:
|
|
1031
|
+
step_text: the sentence of the gated step.
|
|
1032
|
+
findings: the non-blocking findings of both dimensions of the verdict.
|
|
1033
|
+
"""
|
|
1034
|
+
logger.warning(
|
|
1035
|
+
"compliance findings passed",
|
|
1036
|
+
extra={
|
|
1037
|
+
"step_text": step_text,
|
|
1038
|
+
"findings": [
|
|
1039
|
+
f"{finding.priority} {finding.dimension}: {finding.instruction} — {finding.explanation}"
|
|
1040
|
+
for finding in findings
|
|
1041
|
+
],
|
|
1042
|
+
},
|
|
1043
|
+
)
|
|
1044
|
+
|
|
1045
|
+
|
|
1046
|
+
def _validated_captures(prepared: PreparedStep, result: object) -> tuple[dict[str, str], str | None]:
|
|
1047
|
+
"""Validate the settled result of a green candidate against the render product.
|
|
1048
|
+
|
|
1049
|
+
Args:
|
|
1050
|
+
prepared: the render product of the step — the declaration set the
|
|
1051
|
+
returned result must satisfy.
|
|
1052
|
+
result: the step function's return, passed through by the run
|
|
1053
|
+
primitive and the settle window.
|
|
1054
|
+
|
|
1055
|
+
Returns:
|
|
1056
|
+
The validated captures with no violation; an empty captures with the
|
|
1057
|
+
formatted deterministic violation text when the result contract
|
|
1058
|
+
failed — the violation joins the failed-check channel of the loops.
|
|
1059
|
+
"""
|
|
1060
|
+
try:
|
|
1061
|
+
return validate_step_result(prepared, result), None
|
|
1062
|
+
except AssertionError as violation:
|
|
1063
|
+
return {}, format_step_error(violation)
|
|
1064
|
+
|
|
1065
|
+
|
|
1066
|
+
def _record(code: str, error: str, outcome: str, url_before: str, url_after: str) -> StepAttempt:
|
|
1067
|
+
"""Compose one verbatim attempt record — the shape every append site of the loops shares.
|
|
1068
|
+
|
|
1069
|
+
Args:
|
|
1070
|
+
code: the complete candidate code of the attempt.
|
|
1071
|
+
error: the complete failure text of the attempt; empty on no error.
|
|
1072
|
+
outcome: the outcome label constant of the attempt.
|
|
1073
|
+
url_before: the page URL read immediately before the attempt's execution.
|
|
1074
|
+
url_after: the page URL read immediately after the attempt's execution.
|
|
1075
|
+
|
|
1076
|
+
Returns:
|
|
1077
|
+
The immutable record appended to the per-step attempt history.
|
|
1078
|
+
"""
|
|
1079
|
+
return StepAttempt(code=code, error=error, outcome=outcome, url_before=url_before, url_after=url_after)
|
|
1080
|
+
|
|
1081
|
+
|
|
1082
|
+
def _last_facts(history: list[StepAttempt]) -> tuple[str, str]:
|
|
1083
|
+
"""Derive the terminal-failure facts from the last record of the history.
|
|
1084
|
+
|
|
1085
|
+
Args:
|
|
1086
|
+
history: the per-step attempt history of the exhausted pool run.
|
|
1087
|
+
|
|
1088
|
+
Returns:
|
|
1089
|
+
The code and the error of the last record; the empty pair when the
|
|
1090
|
+
history holds no record — no candidate ever existed.
|
|
1091
|
+
"""
|
|
1092
|
+
if not history:
|
|
1093
|
+
return "", ""
|
|
1094
|
+
return history[-1].code, history[-1].error
|
|
1095
|
+
|
|
1096
|
+
|
|
1097
|
+
def _verdict(classification: FailureClassification) -> FailureVerdict:
|
|
1098
|
+
"""Build the verdict value object carried by the terminal errors.
|
|
1099
|
+
|
|
1100
|
+
Args:
|
|
1101
|
+
classification: the classification verdict of the provider.
|
|
1102
|
+
|
|
1103
|
+
Returns:
|
|
1104
|
+
The frozen verdict of the terminal failure.
|
|
1105
|
+
"""
|
|
1106
|
+
return FailureVerdict(
|
|
1107
|
+
category=classification.category,
|
|
1108
|
+
explanation=classification.explanation,
|
|
1109
|
+
recommendation=classification.recommendation,
|
|
1110
|
+
)
|