prettyplay 0.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prettyplay/.usages/lifecycle.md +117 -0
- prettyplay/.usages/steps.md +169 -0
- prettyplay/__init__.py +18 -0
- prettyplay/cache/.usages/addressing.md +44 -0
- prettyplay/cache/.usages/budgets.md +51 -0
- prettyplay/cache/.usages/storage.md +32 -0
- prettyplay/cache/__init__.py +8 -0
- prettyplay/cache/budgets.py +93 -0
- prettyplay/cache/models.py +55 -0
- prettyplay/cache/store.py +276 -0
- prettyplay/cache/text.py +36 -0
- prettyplay/config/.usages/configuration.md +246 -0
- prettyplay/config/__init__.py +6 -0
- prettyplay/config/loader.py +336 -0
- prettyplay/config/models.py +288 -0
- prettyplay/driver/.usages/error_kinds.md +26 -0
- prettyplay/driver/.usages/plumbing.md +52 -0
- prettyplay/driver/__init__.py +11 -0
- prettyplay/driver/errors.py +76 -0
- prettyplay/driver/page.py +175 -0
- prettyplay/driver/session.py +330 -0
- prettyplay/engine/.usages/generation.md +157 -0
- prettyplay/engine/.usages/healing.md +61 -0
- prettyplay/engine/__init__.py +19 -0
- prettyplay/engine/attempts.py +80 -0
- prettyplay/engine/classification.py +70 -0
- prettyplay/engine/compliance.py +125 -0
- prettyplay/engine/execution.py +48 -0
- prettyplay/engine/generator.py +1110 -0
- prettyplay/engine/groups/.usages/recovery.md +71 -0
- prettyplay/engine/groups/__init__.py +7 -0
- prettyplay/engine/groups/diagnosis.py +128 -0
- prettyplay/engine/groups/outcome.py +55 -0
- prettyplay/engine/groups/recovery.py +491 -0
- prettyplay/engine/healer.py +149 -0
- prettyplay/engine/polling/.usages/settle.md +55 -0
- prettyplay/engine/polling/__init__.py +6 -0
- prettyplay/engine/polling/settle.py +98 -0
- prettyplay/engine/polling/window.py +72 -0
- prettyplay/engine/renderer/.usages/memory.md +24 -0
- prettyplay/engine/renderer/.usages/rendering.md +44 -0
- prettyplay/engine/renderer/.usages/validation.md +20 -0
- prettyplay/engine/renderer/__init__.py +7 -0
- prettyplay/engine/renderer/memory.py +37 -0
- prettyplay/engine/renderer/render.py +184 -0
- prettyplay/engine/renderer/validation.py +76 -0
- prettyplay/engine/steering/.usages/steering.md +95 -0
- prettyplay/engine/steering/__init__.py +5 -0
- prettyplay/engine/steering/steering.py +765 -0
- prettyplay/engine/text.py +41 -0
- prettyplay/executor.py +654 -0
- prettyplay/failures/.usages/taxonomy.md +131 -0
- prettyplay/failures/__init__.py +25 -0
- prettyplay/failures/errors.py +359 -0
- prettyplay/groups.py +276 -0
- prettyplay/llm/.usages/classification.md +31 -0
- prettyplay/llm/.usages/providers.md +127 -0
- prettyplay/llm/.usages/transport.md +81 -0
- prettyplay/llm/__init__.py +38 -0
- prettyplay/llm/_request.py +823 -0
- prettyplay/llm/anthropic_provider.py +468 -0
- prettyplay/llm/models.py +273 -0
- prettyplay/llm/openai_provider.py +444 -0
- prettyplay/llm/provider.py +279 -0
- prettyplay/reporting/.usages/hooks.md +61 -0
- prettyplay/reporting/__init__.py +6 -0
- prettyplay/reporting/hooks.py +47 -0
- prettyplay/reporting/reporter.py +70 -0
- prettyplay/runtime.py +81 -0
- prettyplay/scenario.py +469 -0
- prettyplay-0.0.2.dist-info/METADATA +515 -0
- prettyplay-0.0.2.dist-info/RECORD +75 -0
- prettyplay-0.0.2.dist-info/WHEEL +5 -0
- prettyplay-0.0.2.dist-info/licenses/LICENSE +28 -0
- prettyplay-0.0.2.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
# Step generation
|
|
2
|
+
|
|
3
|
+
Domain: generating executable code for an unknown step. Audience: library internals and engineers debugging a first run.
|
|
4
|
+
|
|
5
|
+
## Generate a step
|
|
6
|
+
|
|
7
|
+
```python
|
|
8
|
+
from prettyplay.engine.renderer import PreparedStep
|
|
9
|
+
from prettyplay.llm import ScenarioStep
|
|
10
|
+
|
|
11
|
+
step = generator.generate(
|
|
12
|
+
identity=identity,
|
|
13
|
+
prepared=PreparedStep(
|
|
14
|
+
instruction="click the Sign in button"
|
|
15
|
+
), # the render product of the step — instruction, inputs, declarations
|
|
16
|
+
step_type="action",
|
|
17
|
+
previous_steps=[
|
|
18
|
+
ScenarioStep(sentence="open the login page", instruction="open the login page", group_prompt=""),
|
|
19
|
+
ScenarioStep(
|
|
20
|
+
sentence="enter the login and password", instruction="enter the login and password", group_prompt=""
|
|
21
|
+
),
|
|
22
|
+
],
|
|
23
|
+
group_prompt=None, # None — an ordinary step; the group prompt of the step's group inside one
|
|
24
|
+
page=page,
|
|
25
|
+
attempt_history=history,
|
|
26
|
+
window=window,
|
|
27
|
+
memory=memory,
|
|
28
|
+
)
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
- The loop: request code → execute against the live page → validate the returned result → on failure append the full attempt record and re-request with the fresh snapshot and the grown history
|
|
32
|
+
- The attempt history is one continuous verbatim list: every record carries the outcome, the `URL before -> after` line, the complete candidate code and the complete error; no collapsing, no size limits — the attempt budgets are the only bound. The error text follows the engine error-text policy: the bulky aria-snapshot section Playwright appends to a failed check message is excluded — the fresh page state reaches the loop through the separately captured snapshot
|
|
33
|
+
- Every request carries the honest inputs: the step type (action or assertion) and the prepared instruction of the step (with its INPUTS/RESULTS blocks when present) — the raw template sentence never reaches a request; the casefolded normalization is an addressing key only
|
|
34
|
+
- The page may carry side effects of failed candidates and manual intervention — the replayability requirement of the system prompt tells the model a regeneration never rides that leftover state: an action step repeats its action; a first attempt works on the page the previous steps produced. The structural separation rides the same prompt: an action step ends at its action, an assertion step observes without changing the page
|
|
35
|
+
- Every candidate execution runs under the settle window: transient failures re-execute the same code inside the window (settle_retry log records), no LLM budget consumed; deterministic failures go to the next request or classification; the URL pair brackets the whole attempt, settle re-executions included
|
|
36
|
+
- A non-empty generation_prompt setting adds a USER INSTRUCTIONS block to every generation and regeneration request; classification requests never carry it; changing the instructions never invalidates the cache — cached steps run as stored
|
|
37
|
+
- A non-empty classification_prompt setting adds a USER INSTRUCTIONS block to classification requests only; generation requests never carry it
|
|
38
|
+
- Every generation and regeneration request carries the CHEAT SHEET block right before the USER INSTRUCTIONS block — the compact standard Playwright sync API reference carried by every request; guidance, not an allowlist: everything standard stays allowed, the error-driven regeneration loop is the second line of defense
|
|
39
|
+
|
|
40
|
+
## Result contract
|
|
41
|
+
|
|
42
|
+
A candidate of a step with result declarations must return the dictionary of exactly the declared names to
|
|
43
|
+
non-blank observed strings. After a successful execution the loop validates it deterministically
|
|
44
|
+
(`validate_step_result` from the renderer): a violation is a failed check — the record lands in the history with
|
|
45
|
+
the violation text, the existing classification and healing budgets apply. A valid result publishes to the test
|
|
46
|
+
memory together with the gate pass: an accepted candidate publishes its captures exactly once, after validation
|
|
47
|
+
and the compliance gate; failed and gate-blocked attempts publish nothing.
|
|
48
|
+
|
|
49
|
+
## The execution boundary
|
|
50
|
+
|
|
51
|
+
The whole step executes inside the driver worker thread as one unit: compile and resolve stay on the calling thread,
|
|
52
|
+
the step call itself runs in the worker and receives the genuine sync Page — the calling thread never touches
|
|
53
|
+
Playwright, so interactive hosts keep working. An AssertionError of a step — a failed expect chain or a plain assert
|
|
54
|
+
on an immediate read — reaches failure classification untouched.
|
|
55
|
+
|
|
56
|
+
## The decision table
|
|
57
|
+
|
|
58
|
+
Every classification verdict drives the same table — the category decides, the path only delivers:
|
|
59
|
+
|
|
60
|
+
| Verdict | Action |
|
|
61
|
+
|---|---|
|
|
62
|
+
| product_defect | ProductDefectError carrying the verdict — loud, never healed |
|
|
63
|
+
| rot, fixable | regeneration carrying the classification recommendation |
|
|
64
|
+
| incurable | IncurableStepError carrying the verdict |
|
|
65
|
+
|
|
66
|
+
## Failed candidate check (bounded healing)
|
|
67
|
+
|
|
68
|
+
A failed check — an assertion that executed and did not hold, survived the settle window — appends its attempt
|
|
69
|
+
record, then is classified:
|
|
70
|
+
|
|
71
|
+
- product_defect → ProductDefectError with the verdict; one failed check is spent, never the whole budget
|
|
72
|
+
- rot or fixable → exactly one regeneration funded from the healing budget, the request carrying the recommendation as a RECOMMENDATION block and the grown history; success stores the healed step; a repeat failure gets one final classification deciding only the terminal kind — product_defect → ProductDefectError, anything else → IncurableStepError; no further regeneration
|
|
73
|
+
- incurable → IncurableStepError with the verdict
|
|
74
|
+
- LLM unavailable at the classification → the verdict is skipped quietly (WARNING in the log) and IncurableStepError raises without it
|
|
75
|
+
|
|
76
|
+
## Group steps (framing, no per-step classification)
|
|
77
|
+
|
|
78
|
+
Inside a group the generator behaves differently in exactly two ways:
|
|
79
|
+
|
|
80
|
+
- every request of a group step carries the additive group framing — the GROUP PROMPT block and
|
|
81
|
+
the group-marked earlier steps of the group on top of the full test scenario context; nothing
|
|
82
|
+
existing is removed
|
|
83
|
+
- the two internal classification points — a failed check of a candidate and the
|
|
84
|
+
generation-budget exhaustion — are suppressed: no classification request, no healing-funded
|
|
85
|
+
regeneration; the loop appends the attempt record and raises IncurableStepError carrying the
|
|
86
|
+
failed code and the full failure text; the executor routes it to the group recovery
|
|
87
|
+
|
|
88
|
+
With no group framing input (`group_prompt=None`) the ordinary paths are byte-identical. The
|
|
89
|
+
previous-steps context is typed: each entry carries the raw sentence plus its prepared instruction —
|
|
90
|
+
requests render the instruction field with the permanent group membership marked.
|
|
91
|
+
|
|
92
|
+
## The instruction compliance gate
|
|
93
|
+
|
|
94
|
+
Every successfully executed candidate is verified on two dimensions before it is cached —
|
|
95
|
+
instruction compliance and step adequacy — the default behavior; switch it off with
|
|
96
|
+
generation_approve = false:
|
|
97
|
+
|
|
98
|
+
- one verdict request per candidate through the provider (the effective classification
|
|
99
|
+
model); zero requests when the switch is off or the instructions are empty
|
|
100
|
+
- the request carries INSTRUCTIONS, STEP with its STEP TYPE line, optional INPUTS and RESULTS,
|
|
101
|
+
the ATTEMPT HISTORY records and the CODE block; the reviewer is instructed to use the attempt history as
|
|
102
|
+
the ground truth of what already happened on the page
|
|
103
|
+
- a high finding of either dimension fails the attempt: the record lands in the history
|
|
104
|
+
with the violation text in its error field, the retry carries the grown history — the
|
|
105
|
+
model fixes the finding targeted; budget exhaustion with a standing high finding is the
|
|
106
|
+
terminal incurable failure naming the violated instruction or the unaccomplished step
|
|
107
|
+
- medium and low findings pass with a WARNING naming the instructions
|
|
108
|
+
- a malformed verdict (ComplianceVerdictError) and provider unavailability
|
|
109
|
+
(LLMUnavailableError) are hard failures — a candidate is never cached unchecked
|
|
110
|
+
- replayed cached code is never re-gated: changing the instructions does not invalidate
|
|
111
|
+
the cache — purge it manually when the instructions change
|
|
112
|
+
|
|
113
|
+
## Budget exhaustion
|
|
114
|
+
|
|
115
|
+
Each provider request may resend the identical SDK request up to `llm_request_attempts`
|
|
116
|
+
(default 3, initial send included). These transport sends consume no additional generation
|
|
117
|
+
or healing budget and emit no additional generation-started hooks. If the provider raises
|
|
118
|
+
`LLMUnavailableError` after its retry policy terminates, generation propagates it without
|
|
119
|
+
another engine attempt; an optional terminal classification still uses the quiet verdict skip.
|
|
120
|
+
|
|
121
|
+
Exhaustion of the generation attempts classifies the last candidate: rot or fixable grants exactly one extra
|
|
122
|
+
recommendation-carrying regeneration funded from the healing budget — a repeat failure is terminal
|
|
123
|
+
IncurableStepError without reclassification; any other verdict is terminal as before. LLM unavailability at this
|
|
124
|
+
classification skips the verdict quietly.
|
|
125
|
+
|
|
126
|
+
## Classification call
|
|
127
|
+
|
|
128
|
+
Both engines classify through one routine:
|
|
129
|
+
|
|
130
|
+
```python
|
|
131
|
+
from prettyplay.engine import classify_step_failure
|
|
132
|
+
|
|
133
|
+
classification = classify_step_failure(
|
|
134
|
+
config=config,
|
|
135
|
+
provider=provider,
|
|
136
|
+
step_text="click the «Sign in» button",
|
|
137
|
+
code=step_code,
|
|
138
|
+
error="element not found: button «Sign in»",
|
|
139
|
+
page=page,
|
|
140
|
+
)
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
The routine collects the fresh page snapshot (plus the screenshot when enabled) and calls the provider with the engine classification prompt; a non-empty classification_prompt setting of the config reaches the request as a USER INSTRUCTIONS block. The step sentence is the prepared instruction passed by the caller — never the raw template sentence; the casefolded normalization is an addressing key only. The category set is four: rot, product_defect, fixable, incurable. Provider unavailability propagates: the calling path decides whether it is a terminal infrastructure failure or a quiet verdict skip.
|
|
144
|
+
|
|
145
|
+
## The fixed form
|
|
146
|
+
|
|
147
|
+
Generated code is one function receiving exactly one argument — the genuine sync Playwright Page — importing from
|
|
148
|
+
playwright.sync_api and the Python standard library only (third-party libraries forbidden; imports global only, at
|
|
149
|
+
the top level of the code block, before `def step`, never inside the function body) and working through the standard API: `page.get_by_role("button", name="Sign in").click()`,
|
|
150
|
+
`page.locator("form > button.primary")`, `items = page.get_by_role("listitem")` with
|
|
151
|
+
`assert items.count() > 1`,
|
|
152
|
+
`with page.expect_event("dialog") as info: ... info.value.accept()`,
|
|
153
|
+
`with page.expect_popup() as popup_info: ... popup_info.value`,
|
|
154
|
+
`page.frame_locator("#checkout").get_by_role("button", name="Pay").click()`,
|
|
155
|
+
`locator.scroll_into_view_if_needed()`, `page.mouse.wheel(0, 600)`. No provider constructs, no fixed delays, no
|
|
156
|
+
page.close()/context.close(), no stateful actions (route, clock, add_init_script, tracing, HAR, CDP) — the prompt
|
|
157
|
+
rules; the runtime never enforces them.
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# Step healing
|
|
2
|
+
|
|
3
|
+
Domain: healing a failed cached step. Audience: library internals and engineers reasoning about healed runs.
|
|
4
|
+
|
|
5
|
+
## Heal
|
|
6
|
+
|
|
7
|
+
```python
|
|
8
|
+
from prettyplay.engine.renderer import PreparedStep
|
|
9
|
+
from prettyplay.llm import ScenarioStep
|
|
10
|
+
|
|
11
|
+
healed = healer.heal(
|
|
12
|
+
step=failed_step,
|
|
13
|
+
error="element not found: button «Sign in»",
|
|
14
|
+
prepared=PreparedStep(instruction="click the Sign in button"), # the render product of the failed step
|
|
15
|
+
step_type="action",
|
|
16
|
+
previous_steps=[ScenarioStep(sentence="open the login page", instruction="open the login page", group_prompt="")],
|
|
17
|
+
page=page,
|
|
18
|
+
attempt_history=history,
|
|
19
|
+
window=window,
|
|
20
|
+
memory=memory,
|
|
21
|
+
)
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
The executor seeds record 0 of `attempt_history` before the delegation — the original cached code, the full replay
|
|
25
|
+
error and the URL pair of the replay — so regeneration never loses the original: the history carries every record,
|
|
26
|
+
the anchored cached code first. The prepared instruction and the step type ride every classification and regeneration
|
|
27
|
+
request — the casefolded normalization is an addressing key only.
|
|
28
|
+
|
|
29
|
+
The classification verdict decides the path — the uniform decision table:
|
|
30
|
+
|
|
31
|
+
| Category | Path |
|
|
32
|
+
|---|---|
|
|
33
|
+
| rot, fixable | regenerate from the current page within the healing budget (default 2), the request carrying the classification recommendation as a RECOMMENDATION block and the grown attempt history; execute under the settle window, save back to the cache on success, report loudly |
|
|
34
|
+
| product_defect | raise ProductDefectError carrying the verdict — explanation and recommendation reach the exception message and the log, all three fields reach the on_step_verdict hook (the category never renders) |
|
|
35
|
+
| incurable | raise IncurableStepError carrying the verdict; the reason names the incurability cause |
|
|
36
|
+
|
|
37
|
+
## Rules
|
|
38
|
+
|
|
39
|
+
- A healed candidate passes the two-dimension compliance gate before the write-back: a high
|
|
40
|
+
finding of either dimension fails the healing attempt — the record lands in the history
|
|
41
|
+
with the violation text in its error field; medium and low findings pass with a WARNING;
|
|
42
|
+
a malformed verdict or provider unavailability is a hard failure — nothing is cached
|
|
43
|
+
unchecked
|
|
44
|
+
- Anti-masking: healing never turns a product defect into a green test
|
|
45
|
+
- The healed code replaces the cached code only after a successful execution
|
|
46
|
+
- The accepted regeneration publishes its validated captures after the gate — nothing publishes on a failed or gate-blocked attempt
|
|
47
|
+
- Generation and healing attempts live in one per-test registry — owned by the runtime of the test — with separate per-step limits (default 3 and 2)
|
|
48
|
+
- Inside the regeneration loop no per-attempt classification happens (rejected: LLM cost): a failed attempt of any kind — a failed check included — appends its record and retries with the fresh error, the fresh snapshot and the grown history while budget remains; the entry classification guards the anti-masking
|
|
49
|
+
- A regeneration budget exhaustion raises IncurableStepError carrying the verdict of the original classification — no extra LLM request
|
|
50
|
+
- The scenario context is typed (the raw sentence + the prepared instruction + permanent group membership; requests render the instruction); group steps never reach the healer — their failures route to the group recovery
|
|
51
|
+
- Provider unavailability during the classification raises LLMUnavailableError after the provider's bounded transport policy terminates. Transport sends consume no healing budget; the healer adds no retry on this infrastructure failure.
|
|
52
|
+
- Healing never runs in strict mode: a failed cached step is at most classified, never regenerated
|
|
53
|
+
- Interactive steering attempts are separate from healing: they consume no budgets, join the same per-step history and report their own healings
|
|
54
|
+
|
|
55
|
+
## Verdicts
|
|
56
|
+
|
|
57
|
+
Every terminal failure carries its verdict in full and the full underlying error in the error field: the exception
|
|
58
|
+
message is the structured render — the first line carries the class name of the terminal failure and the authored
|
|
59
|
+
reason, then the `---` separated step/error section, the conditional received/cause/Call log details section and the
|
|
60
|
+
unpadded verdict block; the same text reaches on_step_verdict (structured fields) and the log record. IncurableStepError
|
|
61
|
+
also carries the failed step code in the code field — a programmatic field, never rendered.
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""Facade of the prettyplay.engine cell: generation, execution and healing of step code."""
|
|
2
|
+
|
|
3
|
+
from .attempts import StepAttempt
|
|
4
|
+
from .classification import classify_step_failure
|
|
5
|
+
from .compliance import check_step_compliance
|
|
6
|
+
from .execution import run_step_code
|
|
7
|
+
from .generator import StepGenerator
|
|
8
|
+
from .healer import StepHealer
|
|
9
|
+
from .text import format_step_error
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"StepAttempt",
|
|
13
|
+
"StepGenerator",
|
|
14
|
+
"StepHealer",
|
|
15
|
+
"check_step_compliance",
|
|
16
|
+
"classify_step_failure",
|
|
17
|
+
"format_step_error",
|
|
18
|
+
"run_step_code",
|
|
19
|
+
]
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"""The verbatim attempt record of the engine: one immutable entry of the per-step attempt history."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import TYPE_CHECKING
|
|
6
|
+
|
|
7
|
+
from pydantic import BaseModel, ConfigDict
|
|
8
|
+
|
|
9
|
+
if TYPE_CHECKING:
|
|
10
|
+
from ..driver import PageFacade
|
|
11
|
+
|
|
12
|
+
#: Record 0 of a healing cycle — the anchored cached code and the error of its failed replay.
|
|
13
|
+
OUTCOME_ORIGINAL = "original cached code"
|
|
14
|
+
|
|
15
|
+
#: The candidate executed but its own check — an AssertionError — did not hold.
|
|
16
|
+
OUTCOME_FAILED_CHECK = "failed check"
|
|
17
|
+
|
|
18
|
+
#: The candidate raised a non-assertion exception while executing.
|
|
19
|
+
OUTCOME_EXECUTION_FAILED = "execution failed"
|
|
20
|
+
|
|
21
|
+
#: The candidate executed green but the compliance gate blocked it from the cache.
|
|
22
|
+
OUTCOME_COMPLIANCE_BLOCKED = "compliance blocked"
|
|
23
|
+
|
|
24
|
+
#: The engineer rejected the candidate in the steering dialog; it never executed.
|
|
25
|
+
OUTCOME_REJECTED = "rejected by the engineer, not executed"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class StepAttempt(BaseModel):
|
|
29
|
+
"""One immutable verbatim record of the per-step attempt history — never rewritten or truncated.
|
|
30
|
+
|
|
31
|
+
Attributes:
|
|
32
|
+
code: the complete candidate code of the attempt, verbatim.
|
|
33
|
+
error: the complete failure text — the execution error, the failed
|
|
34
|
+
check text, or the gate violation text; empty on no error.
|
|
35
|
+
outcome: the outcome label of the closed five-label set (the
|
|
36
|
+
``OUTCOME_*`` constants of this module).
|
|
37
|
+
url_before: the page URL read immediately before the attempt's
|
|
38
|
+
execution.
|
|
39
|
+
url_after: the page URL read immediately after the attempt's
|
|
40
|
+
execution.
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
model_config = ConfigDict(kw_only=True, frozen=True)
|
|
44
|
+
|
|
45
|
+
code: str = ""
|
|
46
|
+
error: str = ""
|
|
47
|
+
outcome: str = ""
|
|
48
|
+
url_before: str = ""
|
|
49
|
+
url_after: str = ""
|
|
50
|
+
|
|
51
|
+
def render(self) -> str:
|
|
52
|
+
"""Render the complete verbatim record text.
|
|
53
|
+
|
|
54
|
+
Returns:
|
|
55
|
+
The multi-line record text — the pinned four-part format:
|
|
56
|
+
outcome line, URL pair, the ``code:`` part and, when non-empty,
|
|
57
|
+
the ``error:`` part; no collapsing, no truncation.
|
|
58
|
+
"""
|
|
59
|
+
record = f"{self.outcome}\nurl: {self.url_before} -> {self.url_after}\ncode:\n{self.code}"
|
|
60
|
+
if self.error:
|
|
61
|
+
if not record.endswith("\n"):
|
|
62
|
+
record += "\n"
|
|
63
|
+
record += f"error:\n{self.error}"
|
|
64
|
+
return record
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _read_url(page: PageFacade) -> str:
|
|
68
|
+
"""Read the page URL for an attempt bracket; never kills the attempt.
|
|
69
|
+
|
|
70
|
+
Args:
|
|
71
|
+
page: the page facade handle of the current test.
|
|
72
|
+
|
|
73
|
+
Returns:
|
|
74
|
+
The current URL of the page, or the empty string when the read
|
|
75
|
+
fails — the attempt itself proceeds normally.
|
|
76
|
+
"""
|
|
77
|
+
try:
|
|
78
|
+
return page.url
|
|
79
|
+
except Exception:
|
|
80
|
+
return ""
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""The shared classification call: collect the page state and ask the provider."""
|
|
2
|
+
|
|
3
|
+
from ..config import Config
|
|
4
|
+
from ..driver import PageFacade
|
|
5
|
+
from ..llm import FailureClassification, LLMProvider
|
|
6
|
+
|
|
7
|
+
#: System prompt of every classification request; used by both engines, applied verbatim.
|
|
8
|
+
CLASSIFICATION_PROMPT = """You classify a failure of a web UI test step.
|
|
9
|
+
|
|
10
|
+
Input you receive:
|
|
11
|
+
- STEP: the step sentence
|
|
12
|
+
- CODE: the step code that failed
|
|
13
|
+
- ERROR: the failure description
|
|
14
|
+
- PAGE SNAPSHOT: the accessibility snapshot of the current page
|
|
15
|
+
- SCREENSHOT: an image of the page, when attached
|
|
16
|
+
- USER INSTRUCTIONS: the project's binding classification guidance, when configured — follow it; it never overrides the fixed answer format above
|
|
17
|
+
|
|
18
|
+
Answer with exactly one line of the form:
|
|
19
|
+
category | explanation | recommendation
|
|
20
|
+
|
|
21
|
+
where category is one of:
|
|
22
|
+
- rot — the UI changed (selectors, texts, structure) and the step can be regenerated for the same intent
|
|
23
|
+
- product_defect — the step works as written but the expected behavior of the application is genuinely broken
|
|
24
|
+
- fixable — the step code is at fault (an ambiguous or wrong locator or strategy) while the intent stays satisfiable; regeneration for the same intent can help
|
|
25
|
+
- incurable — the step sentence no longer matches reality, the intent is ambiguous, or regeneration cannot help
|
|
26
|
+
|
|
27
|
+
explanation: one short sentence why. recommendation: one short sentence what the engineer should do.
|
|
28
|
+
Output only that single line — no code, no extra text."""
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def classify_step_failure( # noqa: PLR0913, PLR0917 — the parameter list is fixed by the engine contract
|
|
32
|
+
config: Config,
|
|
33
|
+
provider: LLMProvider,
|
|
34
|
+
step_text: str,
|
|
35
|
+
code: str,
|
|
36
|
+
error: str,
|
|
37
|
+
page: PageFacade,
|
|
38
|
+
) -> FailureClassification:
|
|
39
|
+
"""Classify a step failure: collect the page state and ask the provider.
|
|
40
|
+
|
|
41
|
+
Args:
|
|
42
|
+
config: project settings; ``send_screenshots`` attaches page images
|
|
43
|
+
and ``classification_prompt`` supplies the user instructions.
|
|
44
|
+
provider: the LLM port implementation classifying the failure.
|
|
45
|
+
step_text: the prepared instruction of the failed step as passed by
|
|
46
|
+
the calling engine — the code was generated for it; never the
|
|
47
|
+
raw template sentence, never the casefolded normalization.
|
|
48
|
+
code: the step code that failed.
|
|
49
|
+
error: the human-readable failure description.
|
|
50
|
+
page: the page facade of the current test.
|
|
51
|
+
|
|
52
|
+
Returns:
|
|
53
|
+
The classification verdict.
|
|
54
|
+
|
|
55
|
+
Raises:
|
|
56
|
+
LLMUnavailableError: the provider service failed — propagates
|
|
57
|
+
untouched, the calling path decides the handling.
|
|
58
|
+
"""
|
|
59
|
+
snapshot = page.aria_snapshot()
|
|
60
|
+
screenshot = page.screenshot() if config.send_screenshots else None
|
|
61
|
+
|
|
62
|
+
return provider.classify_step_failure(
|
|
63
|
+
prompt=CLASSIFICATION_PROMPT,
|
|
64
|
+
user_instructions=config.classification_prompt,
|
|
65
|
+
step_text=step_text,
|
|
66
|
+
code=code,
|
|
67
|
+
error=error,
|
|
68
|
+
snapshot=snapshot,
|
|
69
|
+
screenshot=screenshot,
|
|
70
|
+
)
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""The two-dimension compliance gate: instruction compliance and step adequacy of a green candidate."""
|
|
2
|
+
|
|
3
|
+
from ..config import Config
|
|
4
|
+
from ..llm import ComplianceFinding, LLMProvider
|
|
5
|
+
from .attempts import StepAttempt
|
|
6
|
+
from .renderer import PreparedStep
|
|
7
|
+
|
|
8
|
+
#: System prompt of every compliance verdict request; used by the engine gate, applied verbatim.
|
|
9
|
+
#: Frozen mirror of the ``compliance_prompt`` practice of the engine CODEMANIFEST —
|
|
10
|
+
#: the single source of the prompt; the constant changes only together with the manifest.
|
|
11
|
+
COMPLIANCE_PROMPT = """You verify generated step code on two dimensions: the project's user instructions and what the step says.
|
|
12
|
+
|
|
13
|
+
Input you receive:
|
|
14
|
+
- INSTRUCTIONS: the project's user instructions, verbatim
|
|
15
|
+
- STEP TYPE: action or assertion
|
|
16
|
+
- STEP: the prepared instruction of the step — the plain-text sentence with actual values embedded
|
|
17
|
+
- INPUTS: the call input bindings, one name = value line each — present when the step carries inputs
|
|
18
|
+
- RESULTS: the declared result names — present when the step declares captures; the code must return a dictionary of exactly these names to non-blank strings observed on the page
|
|
19
|
+
- ATTEMPT HISTORY: the verbatim record of every attempt of this step so far, when present — the original cached code first when it exists; each record carries the attempt outcome, the URL before -> after line, the complete candidate code and the complete error
|
|
20
|
+
- CODE: the successfully executed candidate code
|
|
21
|
+
|
|
22
|
+
Check the code on both dimensions and answer with exactly one JSON list of
|
|
23
|
+
findings:
|
|
24
|
+
[{"instruction": "<quote>", "priority": "high|medium|low",
|
|
25
|
+
"explanation": "<one short sentence>", "dimension": "instruction|adequacy"}]
|
|
26
|
+
|
|
27
|
+
Dimension calibration:
|
|
28
|
+
- instruction — the code violates a project instruction: the quote is the
|
|
29
|
+
violated instruction
|
|
30
|
+
- adequacy — the code does not match what the step says, given the step
|
|
31
|
+
type and the attempt history: the quote is the fragment of the step
|
|
32
|
+
sentence the code fails to accomplish, or — for behavior that exceeds
|
|
33
|
+
the step — the code line performing it
|
|
34
|
+
|
|
35
|
+
Priority calibration:
|
|
36
|
+
- high — a confident, material finding evident from the code, the step type
|
|
37
|
+
and the attempt history; for instruction: a material violation evident
|
|
38
|
+
from the code itself — the instruction was expressible through the page
|
|
39
|
+
API the code already uses, and the code plainly skipped or contradicted
|
|
40
|
+
it without any fallback attempt; for adequacy: an action step whose code
|
|
41
|
+
contains no action of the step — the page state the code relies on was
|
|
42
|
+
produced by a prior attempt or manual intervention, not by the code
|
|
43
|
+
itself, use the URL lines and the records of the history to see it — or
|
|
44
|
+
material behavior the step never asked for: an assertion step that
|
|
45
|
+
changes page state (a navigation, a click, a fill), a trailing check in
|
|
46
|
+
an action step that only confirms the action's own completion, a check
|
|
47
|
+
of a fact the step sentence never names; only high blocks the candidate
|
|
48
|
+
- medium and low — minor observations, partial compliance or doubt: visible,
|
|
49
|
+
never blocking; when in doubt, never high
|
|
50
|
+
- an empty list [] means the code complies and accomplishes the step
|
|
51
|
+
|
|
52
|
+
Rules:
|
|
53
|
+
- The attempt history is your ground truth for what already happened on the
|
|
54
|
+
page: the candidate ran on a page that may already contain effects of prior
|
|
55
|
+
attempts or manual intervention — code that only checks an already-achieved
|
|
56
|
+
state without producing it is an adequacy violation for an action step
|
|
57
|
+
- Adequacy covers both directions: falling short of the step and exceeding
|
|
58
|
+
it — extra actions inside an assertion step, a trailing confirmation of
|
|
59
|
+
the action's own completion, extra checks of facts the step sentence
|
|
60
|
+
never names are adequacy findings too
|
|
61
|
+
- Conditional prefer-type instructions are checked conditionally: when the code shows
|
|
62
|
+
a graceful fallback attempt, that is compliance; judge followability from the code
|
|
63
|
+
and the step sentence alone — never speculate about page state beyond the attempt
|
|
64
|
+
history and the inputs
|
|
65
|
+
- An instruction the code could not follow because the step sentence itself prevents
|
|
66
|
+
it is not a violation; when the inputs leave the followability in doubt, the finding
|
|
67
|
+
is never high
|
|
68
|
+
- Judge both dimensions: the code against the instructions, and the code against the
|
|
69
|
+
step sentence with its type
|
|
70
|
+
- Treat RESULTS as the exact-key return contract: the code must return a dictionary of exactly
|
|
71
|
+
the declared names to non-blank strings observed on the page — code that cannot produce it
|
|
72
|
+
is an adequacy finding
|
|
73
|
+
- When INPUTS values influence the step, code that hardcodes their current values instead of
|
|
74
|
+
reading step_inputs["vars"] cannot replay with new inputs; report a high adequacy finding
|
|
75
|
+
- Output only the JSON list, no other text"""
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def check_step_compliance( # noqa: PLR0913, PLR0917 — the parameter list is fixed by the engine contract
|
|
79
|
+
config: Config,
|
|
80
|
+
provider: LLMProvider,
|
|
81
|
+
prepared: PreparedStep,
|
|
82
|
+
step_type: str,
|
|
83
|
+
code: str,
|
|
84
|
+
attempt_history: list[StepAttempt],
|
|
85
|
+
) -> list[ComplianceFinding]:
|
|
86
|
+
"""Gate a successfully executed candidate on the two compliance dimensions.
|
|
87
|
+
|
|
88
|
+
Args:
|
|
89
|
+
config: project settings — ``generation_approve`` is the gate switch
|
|
90
|
+
and ``generation_prompt`` supplies the checked user instructions.
|
|
91
|
+
provider: the LLM port implementation returning the verdict.
|
|
92
|
+
prepared: the render product of the generated step — instruction,
|
|
93
|
+
input bindings and result declarations; the verdict request
|
|
94
|
+
renders them as the STEP, INPUTS and RESULTS blocks.
|
|
95
|
+
step_type: action or assertion — the adequacy dimension judges by it.
|
|
96
|
+
code: the successfully executed candidate code.
|
|
97
|
+
attempt_history: the step's attempt history — rendered through the
|
|
98
|
+
record render and passed to the verdict request as the ATTEMPT
|
|
99
|
+
HISTORY block.
|
|
100
|
+
|
|
101
|
+
Returns:
|
|
102
|
+
The verdict findings of both dimensions; an empty list — compliant
|
|
103
|
+
and adequate, or the gate is off. The single check of every caching
|
|
104
|
+
path — generation, healing and steering alike; never runs on
|
|
105
|
+
replayed cached code.
|
|
106
|
+
|
|
107
|
+
Raises:
|
|
108
|
+
LLMUnavailableError: the provider service failed after the bounded
|
|
109
|
+
transport retries; the calling path never caches the candidate.
|
|
110
|
+
ComplianceVerdictError: the provider answer did not parse into
|
|
111
|
+
findings; the calling path never caches the candidate.
|
|
112
|
+
"""
|
|
113
|
+
if not config.generation_approve or not config.generation_prompt:
|
|
114
|
+
return [] # the gate is off or nothing to check against — zero provider calls
|
|
115
|
+
|
|
116
|
+
return provider.check_instruction_compliance(
|
|
117
|
+
prompt=COMPLIANCE_PROMPT,
|
|
118
|
+
user_instructions=config.generation_prompt,
|
|
119
|
+
instruction=prepared.instruction,
|
|
120
|
+
step_type=step_type,
|
|
121
|
+
inputs=prepared.inputs,
|
|
122
|
+
declarations=prepared.declarations,
|
|
123
|
+
code=code,
|
|
124
|
+
attempt_history=[record.render() for record in attempt_history],
|
|
125
|
+
)
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Execution of step code of the fixed form through the run primitive of the page handle."""
|
|
2
|
+
|
|
3
|
+
from collections.abc import Iterator
|
|
4
|
+
from contextlib import contextmanager
|
|
5
|
+
from contextvars import ContextVar
|
|
6
|
+
|
|
7
|
+
from ..driver import PageFacade
|
|
8
|
+
|
|
9
|
+
_step_inputs: ContextVar[dict[str, object] | None] = ContextVar("prettyplay_step_inputs", default=None)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@contextmanager
|
|
13
|
+
def bind_step_inputs(memory: dict[str, str], inputs: dict[str, str]) -> Iterator[None]:
|
|
14
|
+
"""Expose this execution's current bindings to generated code without changing its page argument."""
|
|
15
|
+
token = _step_inputs.set({**memory, "vars": dict(inputs)})
|
|
16
|
+
try:
|
|
17
|
+
yield
|
|
18
|
+
finally:
|
|
19
|
+
_step_inputs.reset(token)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def run_step_code(code: str, page: PageFacade) -> dict[str, str] | None:
|
|
23
|
+
"""Execute step code of the fixed form against the page handle of the test.
|
|
24
|
+
|
|
25
|
+
Args:
|
|
26
|
+
code: the step code text produced by generation or loaded from the
|
|
27
|
+
cache — compiled and resolved on the calling thread, Playwright
|
|
28
|
+
untouched.
|
|
29
|
+
page: the page handle of the current test — the whole step call
|
|
30
|
+
crosses the worker boundary as one unit, the step receives the
|
|
31
|
+
genuine sync Page.
|
|
32
|
+
|
|
33
|
+
Returns:
|
|
34
|
+
The step function's return as-is — the dictionary of declared names
|
|
35
|
+
to observed strings for a step with declarations, None for a
|
|
36
|
+
declaration-free step; plain data only.
|
|
37
|
+
|
|
38
|
+
Raises:
|
|
39
|
+
Exception: whatever the step code raises propagates as-is — never
|
|
40
|
+
swallowed, translated or retried here; the engine classifies it.
|
|
41
|
+
"""
|
|
42
|
+
namespace: dict[str, object] = {"step_inputs": _step_inputs.get() or {}}
|
|
43
|
+
# the engine contract: compile and resolve on the calling thread, Playwright untouched
|
|
44
|
+
exec(compile(code, "<prettyplay-step>", "exec"), namespace)
|
|
45
|
+
|
|
46
|
+
step_fn = namespace["step"]
|
|
47
|
+
# the engine contract: one worker unit — the whole step against the genuine sync Page
|
|
48
|
+
return page.run(step_fn)
|