prettyplay 0.0.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. prettyplay/.usages/lifecycle.md +117 -0
  2. prettyplay/.usages/steps.md +169 -0
  3. prettyplay/__init__.py +18 -0
  4. prettyplay/cache/.usages/addressing.md +44 -0
  5. prettyplay/cache/.usages/budgets.md +51 -0
  6. prettyplay/cache/.usages/storage.md +32 -0
  7. prettyplay/cache/__init__.py +8 -0
  8. prettyplay/cache/budgets.py +93 -0
  9. prettyplay/cache/models.py +55 -0
  10. prettyplay/cache/store.py +276 -0
  11. prettyplay/cache/text.py +36 -0
  12. prettyplay/config/.usages/configuration.md +246 -0
  13. prettyplay/config/__init__.py +6 -0
  14. prettyplay/config/loader.py +336 -0
  15. prettyplay/config/models.py +288 -0
  16. prettyplay/driver/.usages/error_kinds.md +26 -0
  17. prettyplay/driver/.usages/plumbing.md +52 -0
  18. prettyplay/driver/__init__.py +11 -0
  19. prettyplay/driver/errors.py +76 -0
  20. prettyplay/driver/page.py +175 -0
  21. prettyplay/driver/session.py +330 -0
  22. prettyplay/engine/.usages/generation.md +157 -0
  23. prettyplay/engine/.usages/healing.md +61 -0
  24. prettyplay/engine/__init__.py +19 -0
  25. prettyplay/engine/attempts.py +80 -0
  26. prettyplay/engine/classification.py +70 -0
  27. prettyplay/engine/compliance.py +125 -0
  28. prettyplay/engine/execution.py +48 -0
  29. prettyplay/engine/generator.py +1110 -0
  30. prettyplay/engine/groups/.usages/recovery.md +71 -0
  31. prettyplay/engine/groups/__init__.py +7 -0
  32. prettyplay/engine/groups/diagnosis.py +128 -0
  33. prettyplay/engine/groups/outcome.py +55 -0
  34. prettyplay/engine/groups/recovery.py +491 -0
  35. prettyplay/engine/healer.py +149 -0
  36. prettyplay/engine/polling/.usages/settle.md +55 -0
  37. prettyplay/engine/polling/__init__.py +6 -0
  38. prettyplay/engine/polling/settle.py +98 -0
  39. prettyplay/engine/polling/window.py +72 -0
  40. prettyplay/engine/renderer/.usages/memory.md +24 -0
  41. prettyplay/engine/renderer/.usages/rendering.md +44 -0
  42. prettyplay/engine/renderer/.usages/validation.md +20 -0
  43. prettyplay/engine/renderer/__init__.py +7 -0
  44. prettyplay/engine/renderer/memory.py +37 -0
  45. prettyplay/engine/renderer/render.py +184 -0
  46. prettyplay/engine/renderer/validation.py +76 -0
  47. prettyplay/engine/steering/.usages/steering.md +95 -0
  48. prettyplay/engine/steering/__init__.py +5 -0
  49. prettyplay/engine/steering/steering.py +765 -0
  50. prettyplay/engine/text.py +41 -0
  51. prettyplay/executor.py +654 -0
  52. prettyplay/failures/.usages/taxonomy.md +131 -0
  53. prettyplay/failures/__init__.py +25 -0
  54. prettyplay/failures/errors.py +359 -0
  55. prettyplay/groups.py +276 -0
  56. prettyplay/llm/.usages/classification.md +31 -0
  57. prettyplay/llm/.usages/providers.md +127 -0
  58. prettyplay/llm/.usages/transport.md +81 -0
  59. prettyplay/llm/__init__.py +38 -0
  60. prettyplay/llm/_request.py +823 -0
  61. prettyplay/llm/anthropic_provider.py +468 -0
  62. prettyplay/llm/models.py +273 -0
  63. prettyplay/llm/openai_provider.py +444 -0
  64. prettyplay/llm/provider.py +279 -0
  65. prettyplay/reporting/.usages/hooks.md +61 -0
  66. prettyplay/reporting/__init__.py +6 -0
  67. prettyplay/reporting/hooks.py +47 -0
  68. prettyplay/reporting/reporter.py +70 -0
  69. prettyplay/runtime.py +81 -0
  70. prettyplay/scenario.py +469 -0
  71. prettyplay-0.0.2.dist-info/METADATA +515 -0
  72. prettyplay-0.0.2.dist-info/RECORD +75 -0
  73. prettyplay-0.0.2.dist-info/WHEEL +5 -0
  74. prettyplay-0.0.2.dist-info/licenses/LICENSE +28 -0
  75. prettyplay-0.0.2.dist-info/top_level.txt +1 -0
@@ -0,0 +1,157 @@
1
+ # Step generation
2
+
3
+ Domain: generating executable code for an unknown step. Audience: library internals and engineers debugging a first run.
4
+
5
+ ## Generate a step
6
+
7
+ ```python
8
+ from prettyplay.engine.renderer import PreparedStep
9
+ from prettyplay.llm import ScenarioStep
10
+
11
+ step = generator.generate(
12
+ identity=identity,
13
+ prepared=PreparedStep(
14
+ instruction="click the Sign in button"
15
+ ), # the render product of the step — instruction, inputs, declarations
16
+ step_type="action",
17
+ previous_steps=[
18
+ ScenarioStep(sentence="open the login page", instruction="open the login page", group_prompt=""),
19
+ ScenarioStep(
20
+ sentence="enter the login and password", instruction="enter the login and password", group_prompt=""
21
+ ),
22
+ ],
23
+ group_prompt=None, # None — an ordinary step; the group prompt of the step's group inside one
24
+ page=page,
25
+ attempt_history=history,
26
+ window=window,
27
+ memory=memory,
28
+ )
29
+ ```
30
+
31
+ - The loop: request code → execute against the live page → validate the returned result → on failure append the full attempt record and re-request with the fresh snapshot and the grown history
32
+ - The attempt history is one continuous verbatim list: every record carries the outcome, the `URL before -> after` line, the complete candidate code and the complete error; no collapsing, no size limits — the attempt budgets are the only bound. The error text follows the engine error-text policy: the bulky aria-snapshot section Playwright appends to a failed check message is excluded — the fresh page state reaches the loop through the separately captured snapshot
33
+ - Every request carries the honest inputs: the step type (action or assertion) and the prepared instruction of the step (with its INPUTS/RESULTS blocks when present) — the raw template sentence never reaches a request; the casefolded normalization is an addressing key only
34
+ - The page may carry side effects of failed candidates and manual intervention — the replayability requirement of the system prompt tells the model a regeneration never rides that leftover state: an action step repeats its action; a first attempt works on the page the previous steps produced. The structural separation rides the same prompt: an action step ends at its action, an assertion step observes without changing the page
35
+ - Every candidate execution runs under the settle window: transient failures re-execute the same code inside the window (settle_retry log records), no LLM budget consumed; deterministic failures go to the next request or classification; the URL pair brackets the whole attempt, settle re-executions included
36
+ - A non-empty generation_prompt setting adds a USER INSTRUCTIONS block to every generation and regeneration request; classification requests never carry it; changing the instructions never invalidates the cache — cached steps run as stored
37
+ - A non-empty classification_prompt setting adds a USER INSTRUCTIONS block to classification requests only; generation requests never carry it
38
+ - Every generation and regeneration request carries the CHEAT SHEET block right before the USER INSTRUCTIONS block — the compact standard Playwright sync API reference carried by every request; guidance, not an allowlist: everything standard stays allowed, the error-driven regeneration loop is the second line of defense
39
+
40
+ ## Result contract
41
+
42
+ A candidate of a step with result declarations must return the dictionary of exactly the declared names to
43
+ non-blank observed strings. After a successful execution the loop validates it deterministically
44
+ (`validate_step_result` from the renderer): a violation is a failed check — the record lands in the history with
45
+ the violation text, the existing classification and healing budgets apply. A valid result publishes to the test
46
+ memory together with the gate pass: an accepted candidate publishes its captures exactly once, after validation
47
+ and the compliance gate; failed and gate-blocked attempts publish nothing.
48
+
49
+ ## The execution boundary
50
+
51
+ The whole step executes inside the driver worker thread as one unit: compile and resolve stay on the calling thread,
52
+ the step call itself runs in the worker and receives the genuine sync Page — the calling thread never touches
53
+ Playwright, so interactive hosts keep working. An AssertionError of a step — a failed expect chain or a plain assert
54
+ on an immediate read — reaches failure classification untouched.
55
+
56
+ ## The decision table
57
+
58
+ Every classification verdict drives the same table — the category decides, the path only delivers:
59
+
60
+ | Verdict | Action |
61
+ |---|---|
62
+ | product_defect | ProductDefectError carrying the verdict — loud, never healed |
63
+ | rot, fixable | regeneration carrying the classification recommendation |
64
+ | incurable | IncurableStepError carrying the verdict |
65
+
66
+ ## Failed candidate check (bounded healing)
67
+
68
+ A failed check — an assertion that executed and did not hold, survived the settle window — appends its attempt
69
+ record, then is classified:
70
+
71
+ - product_defect → ProductDefectError with the verdict; one failed check is spent, never the whole budget
72
+ - rot or fixable → exactly one regeneration funded from the healing budget, the request carrying the recommendation as a RECOMMENDATION block and the grown history; success stores the healed step; a repeat failure gets one final classification deciding only the terminal kind — product_defect → ProductDefectError, anything else → IncurableStepError; no further regeneration
73
+ - incurable → IncurableStepError with the verdict
74
+ - LLM unavailable at the classification → the verdict is skipped quietly (WARNING in the log) and IncurableStepError raises without it
75
+
76
+ ## Group steps (framing, no per-step classification)
77
+
78
+ Inside a group the generator behaves differently in exactly two ways:
79
+
80
+ - every request of a group step carries the additive group framing — the GROUP PROMPT block and
81
+ the group-marked earlier steps of the group on top of the full test scenario context; nothing
82
+ existing is removed
83
+ - the two internal classification points — a failed check of a candidate and the
84
+ generation-budget exhaustion — are suppressed: no classification request, no healing-funded
85
+ regeneration; the loop appends the attempt record and raises IncurableStepError carrying the
86
+ failed code and the full failure text; the executor routes it to the group recovery
87
+
88
+ With no group framing input (`group_prompt=None`) the ordinary paths are byte-identical. The
89
+ previous-steps context is typed: each entry carries the raw sentence plus its prepared instruction —
90
+ requests render the instruction field with the permanent group membership marked.
91
+
92
+ ## The instruction compliance gate
93
+
94
+ Every successfully executed candidate is verified on two dimensions before it is cached —
95
+ instruction compliance and step adequacy — the default behavior; switch it off with
96
+ generation_approve = false:
97
+
98
+ - one verdict request per candidate through the provider (the effective classification
99
+ model); zero requests when the switch is off or the instructions are empty
100
+ - the request carries INSTRUCTIONS, STEP with its STEP TYPE line, optional INPUTS and RESULTS,
101
+ the ATTEMPT HISTORY records and the CODE block; the reviewer is instructed to use the attempt history as
102
+ the ground truth of what already happened on the page
103
+ - a high finding of either dimension fails the attempt: the record lands in the history
104
+ with the violation text in its error field, the retry carries the grown history — the
105
+ model fixes the finding targeted; budget exhaustion with a standing high finding is the
106
+ terminal incurable failure naming the violated instruction or the unaccomplished step
107
+ - medium and low findings pass with a WARNING naming the instructions
108
+ - a malformed verdict (ComplianceVerdictError) and provider unavailability
109
+ (LLMUnavailableError) are hard failures — a candidate is never cached unchecked
110
+ - replayed cached code is never re-gated: changing the instructions does not invalidate
111
+ the cache — purge it manually when the instructions change
112
+
113
+ ## Budget exhaustion
114
+
115
+ Each provider request may resend the identical SDK request up to `llm_request_attempts`
116
+ (default 3, initial send included). These transport sends consume no additional generation
117
+ or healing budget and emit no additional generation-started hooks. If the provider raises
118
+ `LLMUnavailableError` after its retry policy terminates, generation propagates it without
119
+ another engine attempt; an optional terminal classification still uses the quiet verdict skip.
120
+
121
+ Exhaustion of the generation attempts classifies the last candidate: rot or fixable grants exactly one extra
122
+ recommendation-carrying regeneration funded from the healing budget — a repeat failure is terminal
123
+ IncurableStepError without reclassification; any other verdict is terminal as before. LLM unavailability at this
124
+ classification skips the verdict quietly.
125
+
126
+ ## Classification call
127
+
128
+ Both engines classify through one routine:
129
+
130
+ ```python
131
+ from prettyplay.engine import classify_step_failure
132
+
133
+ classification = classify_step_failure(
134
+ config=config,
135
+ provider=provider,
136
+ step_text="click the «Sign in» button",
137
+ code=step_code,
138
+ error="element not found: button «Sign in»",
139
+ page=page,
140
+ )
141
+ ```
142
+
143
+ The routine collects the fresh page snapshot (plus the screenshot when enabled) and calls the provider with the engine classification prompt; a non-empty classification_prompt setting of the config reaches the request as a USER INSTRUCTIONS block. The step sentence is the prepared instruction passed by the caller — never the raw template sentence; the casefolded normalization is an addressing key only. The category set is four: rot, product_defect, fixable, incurable. Provider unavailability propagates: the calling path decides whether it is a terminal infrastructure failure or a quiet verdict skip.
144
+
145
+ ## The fixed form
146
+
147
+ Generated code is one function receiving exactly one argument — the genuine sync Playwright Page — importing from
148
+ playwright.sync_api and the Python standard library only (third-party libraries forbidden; imports global only, at
149
+ the top level of the code block, before `def step`, never inside the function body) and working through the standard API: `page.get_by_role("button", name="Sign in").click()`,
150
+ `page.locator("form > button.primary")`, `items = page.get_by_role("listitem")` with
151
+ `assert items.count() > 1`,
152
+ `with page.expect_event("dialog") as info: ... info.value.accept()`,
153
+ `with page.expect_popup() as popup_info: ... popup_info.value`,
154
+ `page.frame_locator("#checkout").get_by_role("button", name="Pay").click()`,
155
+ `locator.scroll_into_view_if_needed()`, `page.mouse.wheel(0, 600)`. No provider constructs, no fixed delays, no
156
+ page.close()/context.close(), no stateful actions (route, clock, add_init_script, tracing, HAR, CDP) — the prompt
157
+ rules; the runtime never enforces them.
@@ -0,0 +1,61 @@
1
+ # Step healing
2
+
3
+ Domain: healing a failed cached step. Audience: library internals and engineers reasoning about healed runs.
4
+
5
+ ## Heal
6
+
7
+ ```python
8
+ from prettyplay.engine.renderer import PreparedStep
9
+ from prettyplay.llm import ScenarioStep
10
+
11
+ healed = healer.heal(
12
+ step=failed_step,
13
+ error="element not found: button «Sign in»",
14
+ prepared=PreparedStep(instruction="click the Sign in button"), # the render product of the failed step
15
+ step_type="action",
16
+ previous_steps=[ScenarioStep(sentence="open the login page", instruction="open the login page", group_prompt="")],
17
+ page=page,
18
+ attempt_history=history,
19
+ window=window,
20
+ memory=memory,
21
+ )
22
+ ```
23
+
24
+ The executor seeds record 0 of `attempt_history` before the delegation — the original cached code, the full replay
25
+ error and the URL pair of the replay — so regeneration never loses the original: the history carries every record,
26
+ the anchored cached code first. The prepared instruction and the step type ride every classification and regeneration
27
+ request — the casefolded normalization is an addressing key only.
28
+
29
+ The classification verdict decides the path — the uniform decision table:
30
+
31
+ | Category | Path |
32
+ |---|---|
33
+ | rot, fixable | regenerate from the current page within the healing budget (default 2), the request carrying the classification recommendation as a RECOMMENDATION block and the grown attempt history; execute under the settle window, save back to the cache on success, report loudly |
34
+ | product_defect | raise ProductDefectError carrying the verdict — explanation and recommendation reach the exception message and the log, all three fields reach the on_step_verdict hook (the category never renders) |
35
+ | incurable | raise IncurableStepError carrying the verdict; the reason names the incurability cause |
36
+
37
+ ## Rules
38
+
39
+ - A healed candidate passes the two-dimension compliance gate before the write-back: a high
40
+ finding of either dimension fails the healing attempt — the record lands in the history
41
+ with the violation text in its error field; medium and low findings pass with a WARNING;
42
+ a malformed verdict or provider unavailability is a hard failure — nothing is cached
43
+ unchecked
44
+ - Anti-masking: healing never turns a product defect into a green test
45
+ - The healed code replaces the cached code only after a successful execution
46
+ - The accepted regeneration publishes its validated captures after the gate — nothing publishes on a failed or gate-blocked attempt
47
+ - Generation and healing attempts live in one per-test registry — owned by the runtime of the test — with separate per-step limits (default 3 and 2)
48
+ - Inside the regeneration loop no per-attempt classification happens (rejected: LLM cost): a failed attempt of any kind — a failed check included — appends its record and retries with the fresh error, the fresh snapshot and the grown history while budget remains; the entry classification guards the anti-masking
49
+ - A regeneration budget exhaustion raises IncurableStepError carrying the verdict of the original classification — no extra LLM request
50
+ - The scenario context is typed (the raw sentence + the prepared instruction + permanent group membership; requests render the instruction); group steps never reach the healer — their failures route to the group recovery
51
+ - Provider unavailability during the classification raises LLMUnavailableError after the provider's bounded transport policy terminates. Transport sends consume no healing budget; the healer adds no retry on this infrastructure failure.
52
+ - Healing never runs in strict mode: a failed cached step is at most classified, never regenerated
53
+ - Interactive steering attempts are separate from healing: they consume no budgets, join the same per-step history and report their own healings
54
+
55
+ ## Verdicts
56
+
57
+ Every terminal failure carries its verdict in full and the full underlying error in the error field: the exception
58
+ message is the structured render — the first line carries the class name of the terminal failure and the authored
59
+ reason, then the `---` separated step/error section, the conditional received/cause/Call log details section and the
60
+ unpadded verdict block; the same text reaches on_step_verdict (structured fields) and the log record. IncurableStepError
61
+ also carries the failed step code in the code field — a programmatic field, never rendered.
@@ -0,0 +1,19 @@
1
+ """Facade of the prettyplay.engine cell: generation, execution and healing of step code."""
2
+
3
+ from .attempts import StepAttempt
4
+ from .classification import classify_step_failure
5
+ from .compliance import check_step_compliance
6
+ from .execution import run_step_code
7
+ from .generator import StepGenerator
8
+ from .healer import StepHealer
9
+ from .text import format_step_error
10
+
11
+ __all__ = [
12
+ "StepAttempt",
13
+ "StepGenerator",
14
+ "StepHealer",
15
+ "check_step_compliance",
16
+ "classify_step_failure",
17
+ "format_step_error",
18
+ "run_step_code",
19
+ ]
@@ -0,0 +1,80 @@
1
+ """The verbatim attempt record of the engine: one immutable entry of the per-step attempt history."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import TYPE_CHECKING
6
+
7
+ from pydantic import BaseModel, ConfigDict
8
+
9
+ if TYPE_CHECKING:
10
+ from ..driver import PageFacade
11
+
12
+ #: Record 0 of a healing cycle — the anchored cached code and the error of its failed replay.
13
+ OUTCOME_ORIGINAL = "original cached code"
14
+
15
+ #: The candidate executed but its own check — an AssertionError — did not hold.
16
+ OUTCOME_FAILED_CHECK = "failed check"
17
+
18
+ #: The candidate raised a non-assertion exception while executing.
19
+ OUTCOME_EXECUTION_FAILED = "execution failed"
20
+
21
+ #: The candidate executed green but the compliance gate blocked it from the cache.
22
+ OUTCOME_COMPLIANCE_BLOCKED = "compliance blocked"
23
+
24
+ #: The engineer rejected the candidate in the steering dialog; it never executed.
25
+ OUTCOME_REJECTED = "rejected by the engineer, not executed"
26
+
27
+
28
+ class StepAttempt(BaseModel):
29
+ """One immutable verbatim record of the per-step attempt history — never rewritten or truncated.
30
+
31
+ Attributes:
32
+ code: the complete candidate code of the attempt, verbatim.
33
+ error: the complete failure text — the execution error, the failed
34
+ check text, or the gate violation text; empty on no error.
35
+ outcome: the outcome label of the closed five-label set (the
36
+ ``OUTCOME_*`` constants of this module).
37
+ url_before: the page URL read immediately before the attempt's
38
+ execution.
39
+ url_after: the page URL read immediately after the attempt's
40
+ execution.
41
+ """
42
+
43
+ model_config = ConfigDict(kw_only=True, frozen=True)
44
+
45
+ code: str = ""
46
+ error: str = ""
47
+ outcome: str = ""
48
+ url_before: str = ""
49
+ url_after: str = ""
50
+
51
+ def render(self) -> str:
52
+ """Render the complete verbatim record text.
53
+
54
+ Returns:
55
+ The multi-line record text — the pinned four-part format:
56
+ outcome line, URL pair, the ``code:`` part and, when non-empty,
57
+ the ``error:`` part; no collapsing, no truncation.
58
+ """
59
+ record = f"{self.outcome}\nurl: {self.url_before} -> {self.url_after}\ncode:\n{self.code}"
60
+ if self.error:
61
+ if not record.endswith("\n"):
62
+ record += "\n"
63
+ record += f"error:\n{self.error}"
64
+ return record
65
+
66
+
67
+ def _read_url(page: PageFacade) -> str:
68
+ """Read the page URL for an attempt bracket; never kills the attempt.
69
+
70
+ Args:
71
+ page: the page facade handle of the current test.
72
+
73
+ Returns:
74
+ The current URL of the page, or the empty string when the read
75
+ fails — the attempt itself proceeds normally.
76
+ """
77
+ try:
78
+ return page.url
79
+ except Exception:
80
+ return ""
@@ -0,0 +1,70 @@
1
+ """The shared classification call: collect the page state and ask the provider."""
2
+
3
+ from ..config import Config
4
+ from ..driver import PageFacade
5
+ from ..llm import FailureClassification, LLMProvider
6
+
7
+ #: System prompt of every classification request; used by both engines, applied verbatim.
8
+ CLASSIFICATION_PROMPT = """You classify a failure of a web UI test step.
9
+
10
+ Input you receive:
11
+ - STEP: the step sentence
12
+ - CODE: the step code that failed
13
+ - ERROR: the failure description
14
+ - PAGE SNAPSHOT: the accessibility snapshot of the current page
15
+ - SCREENSHOT: an image of the page, when attached
16
+ - USER INSTRUCTIONS: the project's binding classification guidance, when configured — follow it; it never overrides the fixed answer format above
17
+
18
+ Answer with exactly one line of the form:
19
+ category | explanation | recommendation
20
+
21
+ where category is one of:
22
+ - rot — the UI changed (selectors, texts, structure) and the step can be regenerated for the same intent
23
+ - product_defect — the step works as written but the expected behavior of the application is genuinely broken
24
+ - fixable — the step code is at fault (an ambiguous or wrong locator or strategy) while the intent stays satisfiable; regeneration for the same intent can help
25
+ - incurable — the step sentence no longer matches reality, the intent is ambiguous, or regeneration cannot help
26
+
27
+ explanation: one short sentence why. recommendation: one short sentence what the engineer should do.
28
+ Output only that single line — no code, no extra text."""
29
+
30
+
31
+ def classify_step_failure( # noqa: PLR0913, PLR0917 — the parameter list is fixed by the engine contract
32
+ config: Config,
33
+ provider: LLMProvider,
34
+ step_text: str,
35
+ code: str,
36
+ error: str,
37
+ page: PageFacade,
38
+ ) -> FailureClassification:
39
+ """Classify a step failure: collect the page state and ask the provider.
40
+
41
+ Args:
42
+ config: project settings; ``send_screenshots`` attaches page images
43
+ and ``classification_prompt`` supplies the user instructions.
44
+ provider: the LLM port implementation classifying the failure.
45
+ step_text: the prepared instruction of the failed step as passed by
46
+ the calling engine — the code was generated for it; never the
47
+ raw template sentence, never the casefolded normalization.
48
+ code: the step code that failed.
49
+ error: the human-readable failure description.
50
+ page: the page facade of the current test.
51
+
52
+ Returns:
53
+ The classification verdict.
54
+
55
+ Raises:
56
+ LLMUnavailableError: the provider service failed — propagates
57
+ untouched, the calling path decides the handling.
58
+ """
59
+ snapshot = page.aria_snapshot()
60
+ screenshot = page.screenshot() if config.send_screenshots else None
61
+
62
+ return provider.classify_step_failure(
63
+ prompt=CLASSIFICATION_PROMPT,
64
+ user_instructions=config.classification_prompt,
65
+ step_text=step_text,
66
+ code=code,
67
+ error=error,
68
+ snapshot=snapshot,
69
+ screenshot=screenshot,
70
+ )
@@ -0,0 +1,125 @@
1
+ """The two-dimension compliance gate: instruction compliance and step adequacy of a green candidate."""
2
+
3
+ from ..config import Config
4
+ from ..llm import ComplianceFinding, LLMProvider
5
+ from .attempts import StepAttempt
6
+ from .renderer import PreparedStep
7
+
8
+ #: System prompt of every compliance verdict request; used by the engine gate, applied verbatim.
9
+ #: Frozen mirror of the ``compliance_prompt`` practice of the engine CODEMANIFEST —
10
+ #: the single source of the prompt; the constant changes only together with the manifest.
11
+ COMPLIANCE_PROMPT = """You verify generated step code on two dimensions: the project's user instructions and what the step says.
12
+
13
+ Input you receive:
14
+ - INSTRUCTIONS: the project's user instructions, verbatim
15
+ - STEP TYPE: action or assertion
16
+ - STEP: the prepared instruction of the step — the plain-text sentence with actual values embedded
17
+ - INPUTS: the call input bindings, one name = value line each — present when the step carries inputs
18
+ - RESULTS: the declared result names — present when the step declares captures; the code must return a dictionary of exactly these names to non-blank strings observed on the page
19
+ - ATTEMPT HISTORY: the verbatim record of every attempt of this step so far, when present — the original cached code first when it exists; each record carries the attempt outcome, the URL before -> after line, the complete candidate code and the complete error
20
+ - CODE: the successfully executed candidate code
21
+
22
+ Check the code on both dimensions and answer with exactly one JSON list of
23
+ findings:
24
+ [{"instruction": "<quote>", "priority": "high|medium|low",
25
+ "explanation": "<one short sentence>", "dimension": "instruction|adequacy"}]
26
+
27
+ Dimension calibration:
28
+ - instruction — the code violates a project instruction: the quote is the
29
+ violated instruction
30
+ - adequacy — the code does not match what the step says, given the step
31
+ type and the attempt history: the quote is the fragment of the step
32
+ sentence the code fails to accomplish, or — for behavior that exceeds
33
+ the step — the code line performing it
34
+
35
+ Priority calibration:
36
+ - high — a confident, material finding evident from the code, the step type
37
+ and the attempt history; for instruction: a material violation evident
38
+ from the code itself — the instruction was expressible through the page
39
+ API the code already uses, and the code plainly skipped or contradicted
40
+ it without any fallback attempt; for adequacy: an action step whose code
41
+ contains no action of the step — the page state the code relies on was
42
+ produced by a prior attempt or manual intervention, not by the code
43
+ itself, use the URL lines and the records of the history to see it — or
44
+ material behavior the step never asked for: an assertion step that
45
+ changes page state (a navigation, a click, a fill), a trailing check in
46
+ an action step that only confirms the action's own completion, a check
47
+ of a fact the step sentence never names; only high blocks the candidate
48
+ - medium and low — minor observations, partial compliance or doubt: visible,
49
+ never blocking; when in doubt, never high
50
+ - an empty list [] means the code complies and accomplishes the step
51
+
52
+ Rules:
53
+ - The attempt history is your ground truth for what already happened on the
54
+ page: the candidate ran on a page that may already contain effects of prior
55
+ attempts or manual intervention — code that only checks an already-achieved
56
+ state without producing it is an adequacy violation for an action step
57
+ - Adequacy covers both directions: falling short of the step and exceeding
58
+ it — extra actions inside an assertion step, a trailing confirmation of
59
+ the action's own completion, extra checks of facts the step sentence
60
+ never names are adequacy findings too
61
+ - Conditional prefer-type instructions are checked conditionally: when the code shows
62
+ a graceful fallback attempt, that is compliance; judge followability from the code
63
+ and the step sentence alone — never speculate about page state beyond the attempt
64
+ history and the inputs
65
+ - An instruction the code could not follow because the step sentence itself prevents
66
+ it is not a violation; when the inputs leave the followability in doubt, the finding
67
+ is never high
68
+ - Judge both dimensions: the code against the instructions, and the code against the
69
+ step sentence with its type
70
+ - Treat RESULTS as the exact-key return contract: the code must return a dictionary of exactly
71
+ the declared names to non-blank strings observed on the page — code that cannot produce it
72
+ is an adequacy finding
73
+ - When INPUTS values influence the step, code that hardcodes their current values instead of
74
+ reading step_inputs["vars"] cannot replay with new inputs; report a high adequacy finding
75
+ - Output only the JSON list, no other text"""
76
+
77
+
78
+ def check_step_compliance( # noqa: PLR0913, PLR0917 — the parameter list is fixed by the engine contract
79
+ config: Config,
80
+ provider: LLMProvider,
81
+ prepared: PreparedStep,
82
+ step_type: str,
83
+ code: str,
84
+ attempt_history: list[StepAttempt],
85
+ ) -> list[ComplianceFinding]:
86
+ """Gate a successfully executed candidate on the two compliance dimensions.
87
+
88
+ Args:
89
+ config: project settings — ``generation_approve`` is the gate switch
90
+ and ``generation_prompt`` supplies the checked user instructions.
91
+ provider: the LLM port implementation returning the verdict.
92
+ prepared: the render product of the generated step — instruction,
93
+ input bindings and result declarations; the verdict request
94
+ renders them as the STEP, INPUTS and RESULTS blocks.
95
+ step_type: action or assertion — the adequacy dimension judges by it.
96
+ code: the successfully executed candidate code.
97
+ attempt_history: the step's attempt history — rendered through the
98
+ record render and passed to the verdict request as the ATTEMPT
99
+ HISTORY block.
100
+
101
+ Returns:
102
+ The verdict findings of both dimensions; an empty list — compliant
103
+ and adequate, or the gate is off. The single check of every caching
104
+ path — generation, healing and steering alike; never runs on
105
+ replayed cached code.
106
+
107
+ Raises:
108
+ LLMUnavailableError: the provider service failed after the bounded
109
+ transport retries; the calling path never caches the candidate.
110
+ ComplianceVerdictError: the provider answer did not parse into
111
+ findings; the calling path never caches the candidate.
112
+ """
113
+ if not config.generation_approve or not config.generation_prompt:
114
+ return [] # the gate is off or nothing to check against — zero provider calls
115
+
116
+ return provider.check_instruction_compliance(
117
+ prompt=COMPLIANCE_PROMPT,
118
+ user_instructions=config.generation_prompt,
119
+ instruction=prepared.instruction,
120
+ step_type=step_type,
121
+ inputs=prepared.inputs,
122
+ declarations=prepared.declarations,
123
+ code=code,
124
+ attempt_history=[record.render() for record in attempt_history],
125
+ )
@@ -0,0 +1,48 @@
1
+ """Execution of step code of the fixed form through the run primitive of the page handle."""
2
+
3
+ from collections.abc import Iterator
4
+ from contextlib import contextmanager
5
+ from contextvars import ContextVar
6
+
7
+ from ..driver import PageFacade
8
+
9
+ _step_inputs: ContextVar[dict[str, object] | None] = ContextVar("prettyplay_step_inputs", default=None)
10
+
11
+
12
+ @contextmanager
13
+ def bind_step_inputs(memory: dict[str, str], inputs: dict[str, str]) -> Iterator[None]:
14
+ """Expose this execution's current bindings to generated code without changing its page argument."""
15
+ token = _step_inputs.set({**memory, "vars": dict(inputs)})
16
+ try:
17
+ yield
18
+ finally:
19
+ _step_inputs.reset(token)
20
+
21
+
22
+ def run_step_code(code: str, page: PageFacade) -> dict[str, str] | None:
23
+ """Execute step code of the fixed form against the page handle of the test.
24
+
25
+ Args:
26
+ code: the step code text produced by generation or loaded from the
27
+ cache — compiled and resolved on the calling thread, Playwright
28
+ untouched.
29
+ page: the page handle of the current test — the whole step call
30
+ crosses the worker boundary as one unit, the step receives the
31
+ genuine sync Page.
32
+
33
+ Returns:
34
+ The step function's return as-is — the dictionary of declared names
35
+ to observed strings for a step with declarations, None for a
36
+ declaration-free step; plain data only.
37
+
38
+ Raises:
39
+ Exception: whatever the step code raises propagates as-is — never
40
+ swallowed, translated or retried here; the engine classifies it.
41
+ """
42
+ namespace: dict[str, object] = {"step_inputs": _step_inputs.get() or {}}
43
+ # the engine contract: compile and resolve on the calling thread, Playwright untouched
44
+ exec(compile(code, "<prettyplay-step>", "exec"), namespace)
45
+
46
+ step_fn = namespace["step"]
47
+ # the engine contract: one worker unit — the whole step against the genuine sync Page
48
+ return page.run(step_fn)