prettyplay 0.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prettyplay/.usages/lifecycle.md +117 -0
- prettyplay/.usages/steps.md +169 -0
- prettyplay/__init__.py +18 -0
- prettyplay/cache/.usages/addressing.md +44 -0
- prettyplay/cache/.usages/budgets.md +51 -0
- prettyplay/cache/.usages/storage.md +32 -0
- prettyplay/cache/__init__.py +8 -0
- prettyplay/cache/budgets.py +93 -0
- prettyplay/cache/models.py +55 -0
- prettyplay/cache/store.py +276 -0
- prettyplay/cache/text.py +36 -0
- prettyplay/config/.usages/configuration.md +246 -0
- prettyplay/config/__init__.py +6 -0
- prettyplay/config/loader.py +336 -0
- prettyplay/config/models.py +288 -0
- prettyplay/driver/.usages/error_kinds.md +26 -0
- prettyplay/driver/.usages/plumbing.md +52 -0
- prettyplay/driver/__init__.py +11 -0
- prettyplay/driver/errors.py +76 -0
- prettyplay/driver/page.py +175 -0
- prettyplay/driver/session.py +330 -0
- prettyplay/engine/.usages/generation.md +157 -0
- prettyplay/engine/.usages/healing.md +61 -0
- prettyplay/engine/__init__.py +19 -0
- prettyplay/engine/attempts.py +80 -0
- prettyplay/engine/classification.py +70 -0
- prettyplay/engine/compliance.py +125 -0
- prettyplay/engine/execution.py +48 -0
- prettyplay/engine/generator.py +1110 -0
- prettyplay/engine/groups/.usages/recovery.md +71 -0
- prettyplay/engine/groups/__init__.py +7 -0
- prettyplay/engine/groups/diagnosis.py +128 -0
- prettyplay/engine/groups/outcome.py +55 -0
- prettyplay/engine/groups/recovery.py +491 -0
- prettyplay/engine/healer.py +149 -0
- prettyplay/engine/polling/.usages/settle.md +55 -0
- prettyplay/engine/polling/__init__.py +6 -0
- prettyplay/engine/polling/settle.py +98 -0
- prettyplay/engine/polling/window.py +72 -0
- prettyplay/engine/renderer/.usages/memory.md +24 -0
- prettyplay/engine/renderer/.usages/rendering.md +44 -0
- prettyplay/engine/renderer/.usages/validation.md +20 -0
- prettyplay/engine/renderer/__init__.py +7 -0
- prettyplay/engine/renderer/memory.py +37 -0
- prettyplay/engine/renderer/render.py +184 -0
- prettyplay/engine/renderer/validation.py +76 -0
- prettyplay/engine/steering/.usages/steering.md +95 -0
- prettyplay/engine/steering/__init__.py +5 -0
- prettyplay/engine/steering/steering.py +765 -0
- prettyplay/engine/text.py +41 -0
- prettyplay/executor.py +654 -0
- prettyplay/failures/.usages/taxonomy.md +131 -0
- prettyplay/failures/__init__.py +25 -0
- prettyplay/failures/errors.py +359 -0
- prettyplay/groups.py +276 -0
- prettyplay/llm/.usages/classification.md +31 -0
- prettyplay/llm/.usages/providers.md +127 -0
- prettyplay/llm/.usages/transport.md +81 -0
- prettyplay/llm/__init__.py +38 -0
- prettyplay/llm/_request.py +823 -0
- prettyplay/llm/anthropic_provider.py +468 -0
- prettyplay/llm/models.py +273 -0
- prettyplay/llm/openai_provider.py +444 -0
- prettyplay/llm/provider.py +279 -0
- prettyplay/reporting/.usages/hooks.md +61 -0
- prettyplay/reporting/__init__.py +6 -0
- prettyplay/reporting/hooks.py +47 -0
- prettyplay/reporting/reporter.py +70 -0
- prettyplay/runtime.py +81 -0
- prettyplay/scenario.py +469 -0
- prettyplay-0.0.2.dist-info/METADATA +515 -0
- prettyplay-0.0.2.dist-info/RECORD +75 -0
- prettyplay-0.0.2.dist-info/WHEEL +5 -0
- prettyplay-0.0.2.dist-info/licenses/LICENSE +28 -0
- prettyplay-0.0.2.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
# Run lifecycle
|
|
2
|
+
|
|
3
|
+
Domain: how a run is composed — runtimes, contexts, hooks, failures, strict mode, polling, steering. Audience: integrators wiring the library into a runner and CI.
|
|
4
|
+
|
|
5
|
+
## Composition
|
|
6
|
+
|
|
7
|
+
One runtime per test: each PrettyPlay builds its own runtime — its own configuration, browser process, LLM provider and attempt budgets. Tests never share browser state or budgets through the library; outcomes do not depend on the execution order. Constructing a test is cheap and requires no LLM credentials: the browser starts lazily on the first step. A config passed to the test overrides only the explicitly set values — everything else resolves from pyproject+env; the override reaches inside the nested browser group. When the process exits, every runtime stops its browser and driver synchronously before returning control to the terminal — scripts never leave browser processes behind.
|
|
8
|
+
|
|
9
|
+
```python
|
|
10
|
+
from prettyplay import BrowserConfig, PrettyConfig, PrettyPlay
|
|
11
|
+
|
|
12
|
+
# a strict run with a fullscreen browser
|
|
13
|
+
test = PrettyPlay(
|
|
14
|
+
cache_key="login",
|
|
15
|
+
config=PrettyConfig(
|
|
16
|
+
strict=True,
|
|
17
|
+
browser=BrowserConfig(screen="fullscreen", headless=False),
|
|
18
|
+
),
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
# a local generation session with polling and steering
|
|
22
|
+
test = PrettyPlay(
|
|
23
|
+
cache_key="login",
|
|
24
|
+
config=PrettyConfig(polling_timeout=8.0, interactive=True),
|
|
25
|
+
)
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## Wiring into a framework
|
|
29
|
+
|
|
30
|
+
The library is framework-agnostic: no plugins, no base classes. Construct the object in your test, call the step methods, let failures propagate — the runner counts them as ordinary test failures. A few lines of glue are enough; the suite runs by the standard runner command.
|
|
31
|
+
|
|
32
|
+
## Strict mode — replay-only CI runs
|
|
33
|
+
|
|
34
|
+
`strict = true` (env PRETTYPLAY_STRICT, per-test override) turns the run into an honest replay: cached code executes exactly as stored and nothing is ever (re)generated. Settle polling still applies to cached code — re-executing it is execution, not generation.
|
|
35
|
+
|
|
36
|
+
- A cache miss fails the step as IncurableStepError stating that strict mode forbids generation — an expected CI signal: generating a step is a deliberate non-strict act
|
|
37
|
+
- A failed cached step is classified when LLM access is configured: product_defect raises ProductDefectError, rot/fixable/incurable raises IncurableStepError — never regenerated
|
|
38
|
+
- Without LLM access the failure raises immediately by step type — an assertion step raises ProductDefectError, an action step raises IncurableStepError — each carrying the full underlying error; a WARNING is logged
|
|
39
|
+
- Classifications are the only LLM calls; generation and healing budgets are never consumed
|
|
40
|
+
|
|
41
|
+
Team workflow: generate locally where the LLM is reachable, commit the cache directory, run CI fully from the cache with no LLM keys — optionally with strict=true for guaranteed replay-only behavior.
|
|
42
|
+
|
|
43
|
+
Template steps replay like any cached step: captures re-read from the current page, results validate and publish
|
|
44
|
+
on the strict path exactly as on the ordinary replay; strict adds no generation, healing or new checks.
|
|
45
|
+
|
|
46
|
+
## Settle polling
|
|
47
|
+
|
|
48
|
+
`polling_timeout` (default None — off; 0 — explicit disable) opens one settle window per step execution, measured
|
|
49
|
+
from the first execution of the step's code: a transient failure of a pollable kind re-executes the same code after
|
|
50
|
+
`polling_delay` (default 0.5 s) until success or window end. Re-executions appear as settle_retry log records; no LLM
|
|
51
|
+
budget is consumed. Locator ambiguity and Python-level errors of the step code never poll.
|
|
52
|
+
|
|
53
|
+
## Interactive steering
|
|
54
|
+
|
|
55
|
+
`interactive = true` (env PRETTYPLAY_INTERACTIVE, per-test override) arms the steering REPL for local generation
|
|
56
|
+
sessions: when a step terminally fails with IncurableStepError, a terminal dialog opens — step, failed code, the
|
|
57
|
+
terminal error render, the current page URL, a screenshot path, the commands — and every engineer message drives one
|
|
58
|
+
regeneration whose complete code is shown for approval with run? [y/N] before it executes against the live page. A
|
|
59
|
+
green turn heals the step and writes it back to the cache — only after the instruction compliance gate
|
|
60
|
+
passes (a high finding never reaches the cache: the violation joins the history and the prompt reopens);
|
|
61
|
+
quit/EOF/SIGINT/unreadable stdin raises the
|
|
62
|
+
original terminal failure. The dialog opens only on IncurableStepError of a non-strict run — an LLMUnavailableError
|
|
63
|
+
leaving the engine never reaches it (with a dead provider the dialog still opens and its local commands work; the
|
|
64
|
+
first guidance message then declines and the original failure propagates) — and consumes no budgets.
|
|
65
|
+
Keep it off in CI — an accidentally opened dialog would hang the run.
|
|
66
|
+
|
|
67
|
+
## Group recovery
|
|
68
|
+
|
|
69
|
+
Steps inside a group fail differently: on a non-strict run every failure that would be classified
|
|
70
|
+
and healed per-step triggers one group-level diagnosis instead — the classification model sees the
|
|
71
|
+
group prompt, every step's outcome and URL transition, and the current page. recoverable → the
|
|
72
|
+
affected row regenerates and re-executes automatically (per-step cache write-back, loud
|
|
73
|
+
reporting); product_defect → the test fails loudly. The recovery budget: every new cycle grants
|
|
74
|
+
each row step a fresh healing counter, the number of cycles per group is capped by
|
|
75
|
+
healing_attempts. Strict mode never recovers groups; a terminally failed group step still reaches
|
|
76
|
+
the steering dialog with the group context available.
|
|
77
|
+
|
|
78
|
+
## Hooks
|
|
79
|
+
|
|
80
|
+
Implement the StepHooks callback contract and register the implementation — either pass the list to the keyword-only constructor parameter `hooks` (events are captured from the very construction) or call add_hooks before the first step. Step, generation, healing, cache and verdict events reach the handler synchronously. on_step_failed carries the full rendered failure message; on_step_verdict fires after it whenever the terminal failure carries an LLM verdict; on_step_finished closes every step exactly once, passed or failed. A group block frames itself with the four group lifecycle events — on_group_started on entry, then on_group_passed (a recovered group reports passed) or on_group_failed, and the closing on_group_finished exactly once. In strict mode generation and healing events never fire.
|
|
81
|
+
|
|
82
|
+
## Failures
|
|
83
|
+
|
|
84
|
+
Five kinds reach the runner:
|
|
85
|
+
|
|
86
|
+
| Kind | Meaning | Reaction |
|
|
87
|
+
|---|---|---|
|
|
88
|
+
| ProductDefectError | a real regression — also an AssertionError: runners show a failure, not an error; the traceback is folded to the library boundary; aria-snapshot tails are removed from single-string exception arguments in the chain, preserving exception identities and links; other argument shapes and custom message rendering are outside this guarantee | treat as a bug — this failure is the value of the suite |
|
|
89
|
+
| IncurableStepError | the step cannot be generated or healed — in strict mode also: the cache miss | follow the carried recommendation |
|
|
90
|
+
| LLMUnavailableError | the LLM is down | only generation and healing are blocked; cached steps keep running |
|
|
91
|
+
| ComplianceVerdictError | the instruction compliance gate could not obtain a usable verdict — the executed candidate is never cached unchecked | rerun the step to retry generation; a repeatedly malformed verdict points at the verdict model |
|
|
92
|
+
| ConfigurationError | the settings are invalid | fix the named setting — the message lists the allowed values |
|
|
93
|
+
|
|
94
|
+
ProductDefectError and IncurableStepError render one structured message — the first line
|
|
95
|
+
with the class name of the terminal failure and the authored reason, the `---` separated step/error section, the
|
|
96
|
+
conditional received/cause/Call log details section and the unpadded verdict block — and the same text reaches the
|
|
97
|
+
exception message, the on_step_failed hook event and the log. When the LLM is unavailable the verdict is skipped quietly; the failure itself never waits for it.
|
|
98
|
+
|
|
99
|
+
## Interactive sessions (IPython, Jupyter)
|
|
100
|
+
|
|
101
|
+
The Playwright session lives in a background driver thread owned by the library: the thread that executes the steps never holds a running asyncio loop, so interactive hosts that drive their own prompt through asyncio (IPython, Jupyter) keep working after every step — passed or failed.
|
|
102
|
+
|
|
103
|
+
The author escape hatch run_on_page crosses the same worker boundary the same way: the author callable executes wholly inside the worker thread with the genuine Page — the host thread keeps driving its own prompt loop.
|
|
104
|
+
|
|
105
|
+
Each test owns its browser process: it starts on the first step of the test and stops when the test closes. In scripts every runtime stops automatically at process exit through its atexit hook. In an interactive session the process keeps living between cells, so close the test object explicitly when the interactive exploration is over:
|
|
106
|
+
|
|
107
|
+
```python
|
|
108
|
+
from prettyplay import PrettyPlay
|
|
109
|
+
|
|
110
|
+
test = PrettyPlay("login-flow")
|
|
111
|
+
test.step("open the login page")
|
|
112
|
+
test.step("enter the login and password")
|
|
113
|
+
test.expect("the «Welcome back» message appears")
|
|
114
|
+
test.close() # stops this test's browser and driver thread
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
Generation of the step cache remains a batch workflow: prefer a plain script or a pytest run over a REPL when generating many steps. The `interactive` setting of the previous section is a different thing entirely: it is the steering dialog of a stuck step, not a host mode.
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
# Writing steps
|
|
2
|
+
|
|
3
|
+
Domain: authoring UI tests as plain sentences. Audience: engineers writing tests and integrators wiring the library into a test framework.
|
|
4
|
+
|
|
5
|
+
## A test as a scenario
|
|
6
|
+
|
|
7
|
+
```python
|
|
8
|
+
from prettyplay import PrettyPlay
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def test_login():
|
|
12
|
+
t = PrettyPlay("login-flow")
|
|
13
|
+
t.step("open the login page")
|
|
14
|
+
t.step("enter the login and password")
|
|
15
|
+
t.step("click the «Sign in» button")
|
|
16
|
+
t.expect("the «Welcome back» message appears")
|
|
17
|
+
t.close()
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
Or with the context manager:
|
|
21
|
+
|
|
22
|
+
```python
|
|
23
|
+
with PrettyPlay("login-flow") as t:
|
|
24
|
+
t.step("open the login page")
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
## Step kinds
|
|
28
|
+
|
|
29
|
+
- step(text) — performs what the sentence says
|
|
30
|
+
- expect(text) — verifies what the sentence says; a legitimately failed expectation fails the test as a product defect
|
|
31
|
+
|
|
32
|
+
## Step parameters
|
|
33
|
+
|
|
34
|
+
- step(text, tries=3) — the total number of executions of the step's code (the first included);
|
|
35
|
+
replaces the time-bounded settle window for that step; the global polling settings stay
|
|
36
|
+
untouched; absorbed retries appear as settle_retry records only, the step stays green
|
|
37
|
+
- step(text, delay=1.5) — a quiet pause in seconds before the step's code runs: the started event
|
|
38
|
+
fires, the declared seconds pass, then the code; a step never reached never pauses
|
|
39
|
+
- step(text, vars={"expected": "Details"}) — the call-local string inputs; separate namespace from memory
|
|
40
|
+
- The optional tries, delay and vars parameters are keyword-only and accepted by both step kinds; invalid values (zero/negative
|
|
41
|
+
tries, negative delay) and non-string vars values fail loudly at the call
|
|
42
|
+
|
|
43
|
+
## Templates and step memory
|
|
44
|
+
|
|
45
|
+
Capture an observation, navigate, verify — ordinary step and expect calls plus explicit template syntax:
|
|
46
|
+
|
|
47
|
+
```python
|
|
48
|
+
with PrettyPlay("item-name") as t:
|
|
49
|
+
t.step("Open https://shop.example/products")
|
|
50
|
+
t.step("Read the first item name into {% var name %}")
|
|
51
|
+
t.step("Open the item named {{ name }}")
|
|
52
|
+
t.expect("The page contains {{ name }}")
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
- `{% var name %}` declares where the step saves text read from the page; once the step is accepted, `name` is
|
|
56
|
+
available in later templates
|
|
57
|
+
- Several captures in one step publish together — `t.step("Read the first item name into {% var first_name %} and
|
|
58
|
+
the second item name into {% var second_name %}")`; a successful recapture replaces the previous value; a failed
|
|
59
|
+
step publishes nothing
|
|
60
|
+
- Memory survives navigation and belongs to one test execution — never shared across tests, concurrent runs included
|
|
61
|
+
- A same-step read observes the previous value or fails with the unavailable name; use two steps to capture then use
|
|
62
|
+
- Templates are ordinary Python strings — f-strings evaluate before PrettyPlay receives them
|
|
63
|
+
|
|
64
|
+
Call-local inputs through the reserved `vars` namespace:
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
t.expect("The page contains {{ name }} and {{ vars.expected }}", vars={"expected": "Details"})
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
- `{{ name }}` reads memory, `{{ vars.name }}` reads the call input — separate namespaces, neither overwrites the
|
|
71
|
+
other; call inputs never establish memory; values are strings only
|
|
72
|
+
- Standard Jinja stays available (filters, conditions, `set` — template-local); missing values fail loudly naming
|
|
73
|
+
the name; author-written `| default(...)` and `is defined` behave as authored
|
|
74
|
+
|
|
75
|
+
## Literal Jinja in existing sentences
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
t.expect("The page contains {% raw %}{{ name }}{% endraw %}")
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
## Cache and replay limitation
|
|
82
|
+
|
|
83
|
+
- Template sentences address the cache by their source: `{{ name }}` and `{{ Name }}` are different steps; runtime
|
|
84
|
+
values never enter the address
|
|
85
|
+
- Cached code re-reads captures from the current page on every execution — a changed item name for the same
|
|
86
|
+
operation replays without regeneration
|
|
87
|
+
- Accepted limitation: a template that changes the operation itself may let old cached code succeed at the previous
|
|
88
|
+
operation without triggering healing
|
|
89
|
+
|
|
90
|
+
## Groups
|
|
91
|
+
|
|
92
|
+
```python
|
|
93
|
+
with t.group("accept cookies, fill and submit the order form") as g:
|
|
94
|
+
g.step("accept the cookie banner")
|
|
95
|
+
g.step("fill the email field", delay=0.5)
|
|
96
|
+
g.step("submit the form")
|
|
97
|
+
g.expect("the status shows order confirmed", tries=2)
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
- The group block is one coherent mini-scenario with a shared goal; inside, steps use the ordinary
|
|
101
|
+
authoring surface — retry count and start pause included
|
|
102
|
+
- The block frames itself with the four group lifecycle events — on_group_started on entry, then
|
|
103
|
+
on_group_passed/on_group_failed and the closing on_group_finished on exit; a recovered group
|
|
104
|
+
reports passed
|
|
105
|
+
- An empty group prompt fails loudly at entry; a group with zero steps emits only the group lifecycle events
|
|
106
|
+
- Group membership changes no step's cache address: cached group steps replay as ordinary steps —
|
|
107
|
+
no LLM calls, strict replay-only included
|
|
108
|
+
- On a non-strict run a failing group step is diagnosed with the whole group in view and the
|
|
109
|
+
affected row is regenerated and re-executed automatically — a product defect fails loudly,
|
|
110
|
+
recovery never loops forever
|
|
111
|
+
- Group pace and pause: `with t.group("…", speed=30, delay=2) as g:` slows the pauses between the
|
|
112
|
+
group's steps (the same percent→pause mapping as the browser speed setting) and waits the
|
|
113
|
+
declared seconds before the group's first step — quiet, library-level pauses
|
|
114
|
+
|
|
115
|
+
## The honest step context
|
|
116
|
+
|
|
117
|
+
The step cycle carries an honest context window end to end: every generation, healing and
|
|
118
|
+
steering request receives the step type (action or assertion), the prepared instruction with its input bindings and
|
|
119
|
+
result declarations, and the verbatim per-step attempt history — every prior candidate
|
|
120
|
+
with its outcome, its URL before -> after line, its complete code and complete error, the
|
|
121
|
+
original cached code anchored first. The cache write is guarded by a two-dimension gate —
|
|
122
|
+
instruction compliance and step adequacy — so a cached step contains the action the
|
|
123
|
+
sentence asks for, not a check of an already-achieved state. All of this is internal: the
|
|
124
|
+
authoring surface uses step() and expect() with optional vars; cache addressing uses the original
|
|
125
|
+
template source.
|
|
126
|
+
|
|
127
|
+
## Author page access
|
|
128
|
+
|
|
129
|
+
The excluded-from-generation stateful actions are performed explicitly by the author — the callable runs wholly
|
|
130
|
+
inside the driver worker thread and receives the genuine sync Page:
|
|
131
|
+
|
|
132
|
+
with PrettyPlay("videos-flow") as t:
|
|
133
|
+
t.step("open the videos page")
|
|
134
|
+
t.run_on_page(lambda page: page.route("**/api/videos", lambda route: route.fulfill(json={"items": []})))
|
|
135
|
+
t.expect("the page shows a list of videos")
|
|
136
|
+
|
|
137
|
+
- Requires an opened page: call it after the first step — a loud error otherwise
|
|
138
|
+
- The callable returns plain data; Playwright objects (locators, handles, pages, contexts) never cross back to the calling thread
|
|
139
|
+
- Prompt rules do not bind the author: page.route, page.clock, tracing, HAR, CDP are the author's explicit tools
|
|
140
|
+
- The callable must use the page API only — calling back into the test object (a step, a screenshot, a nested
|
|
141
|
+
run_on_page) re-enters the worker thread the action itself runs on and is rejected with a loud error instead of a
|
|
142
|
+
deadlock
|
|
143
|
+
- The callable runs sequentially with the steps — the shared worker takes one unit at a time
|
|
144
|
+
|
|
145
|
+
## Screenshots
|
|
146
|
+
|
|
147
|
+
Two author-facing abilities on the test object:
|
|
148
|
+
|
|
149
|
+
```python
|
|
150
|
+
with PrettyPlay("login-flow") as t:
|
|
151
|
+
t.step("open the login page")
|
|
152
|
+
png = t.get_screenshot() # full-page PNG bytes of the current state
|
|
153
|
+
t.save_screenshot("artifacts/home.png") # write full-page PNG to an explicit path
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
- Both require an opened page: call them after the first step of the test
|
|
157
|
+
- Nothing is captured automatically on failures — attaching screenshots to reports is the author's decision
|
|
158
|
+
|
|
159
|
+
## Addressing
|
|
160
|
+
|
|
161
|
+
The constructor arguments form the cache address: cache_key (mandatory) and cache_path (optional subdirectory). Equal cache keys in the shared root reuse one cached step across tests; a different language, step type or key is a different step. User instructions (generation_prompt, classification_prompt) take no part in the address — a cached step never regenerates because the instructions changed. Replayed cached code is never re-checked against the current instructions: purge the cache manually after changing them. The attempt history and the honest request inputs take no part in the address either.
|
|
162
|
+
|
|
163
|
+
## What you see
|
|
164
|
+
|
|
165
|
+
Step sentences go to the logger prettyplay at info level — the suite output reads as a plain-language scenario. A failed step renders one structured message — the first line with the class name of the terminal failure and the authored reason, the `---` separated step/error section, the conditional received/cause/Call log details section and the unpadded verdict block — identical in the runner output, the log and the on_step_failed hook. Every step ends with one closing on_step_finished event (passed or failed). Healing, cache writes and skipped writes are reported loudly through the same logger. Transient failures absorbed by the settle window appear as settle_retry records — the step itself stays green.
|
|
166
|
+
|
|
167
|
+
## Limitations
|
|
168
|
+
|
|
169
|
+
Step sentences land in the repository cache, the logs and the LLM requests: never put secrets or personal data into a step.
|
prettyplay/__init__.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""Facade of the prettyplay root: the composition root of the library."""
|
|
2
|
+
|
|
3
|
+
from .config import BrowserConfig, PrettyConfig
|
|
4
|
+
from .executor import StepExecutor
|
|
5
|
+
from .groups import StepGroup
|
|
6
|
+
from .reporting import StepHooks
|
|
7
|
+
from .runtime import PrettyplayRuntime
|
|
8
|
+
from .scenario import PrettyPlay
|
|
9
|
+
|
|
10
|
+
__all__ = [ # noqa: RUF022 — the facade listing order is fixed by the root cell contract
|
|
11
|
+
"PrettyPlay",
|
|
12
|
+
"StepHooks",
|
|
13
|
+
"PrettyConfig",
|
|
14
|
+
"BrowserConfig",
|
|
15
|
+
"PrettyplayRuntime",
|
|
16
|
+
"StepExecutor",
|
|
17
|
+
"StepGroup",
|
|
18
|
+
]
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# Step addressing
|
|
2
|
+
|
|
3
|
+
Domain: step identity and addressing of the cache. Audience: engineers reasoning about step reuse and inspecting the repository cache.
|
|
4
|
+
|
|
5
|
+
## Identity triple
|
|
6
|
+
|
|
7
|
+
| Component | Source | Effect on identity |
|
|
8
|
+
|---|---|---|
|
|
9
|
+
| cache_key | the main object constructor argument | a different key — a different step |
|
|
10
|
+
| step type | action vs assertion | the same sentence as action and as assertion — two steps |
|
|
11
|
+
| normalized sentence | templates: NFC + trim (verbatim); ordinary: NFC, trim, whitespace collapse, casefold — a template addresses by its source; an ordinary sentence casefolds as before | «Click Sign in» equals «click sign in »; a Russian sentence and its English translation are different steps |
|
|
12
|
+
|
|
13
|
+
A missing cache entry for the computed address is a cache miss — the step is generated, not an error.
|
|
14
|
+
|
|
15
|
+
## Template sentences
|
|
16
|
+
|
|
17
|
+
A sentence containing Jinja markers (`{{`, `{%` or `{#`) is a template step:
|
|
18
|
+
|
|
19
|
+
```python
|
|
20
|
+
normalize_step_text("Read the first item name into {% var name %}")
|
|
21
|
+
# NFC + trim only — case-sensitive names and expression whitespace stay significant
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
- `{{ name }}` and `{{ Name }}` are different steps — Jinja names are case-sensitive
|
|
25
|
+
- Two templates differing only by prose case or spacing do not share an entry — a fresh generation, never a wrong hit
|
|
26
|
+
- Runtime values never enter the address: the same template with different observed values reuses the same cached code, which re-reads captures on every execution
|
|
27
|
+
|
|
28
|
+
## Normalize and address
|
|
29
|
+
|
|
30
|
+
```python
|
|
31
|
+
from prettyplay.cache import StepIdentity, normalize_step_text
|
|
32
|
+
|
|
33
|
+
normalized = normalize_step_text(" Click Sign In ")
|
|
34
|
+
identity = StepIdentity(
|
|
35
|
+
cache_key="login-flow",
|
|
36
|
+
step_type="action",
|
|
37
|
+
normalized_text=normalized,
|
|
38
|
+
)
|
|
39
|
+
# identity.filename — the deterministic digest of the triple
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## Layout
|
|
43
|
+
|
|
44
|
+
The cache root defaults to <cwd>/.prettyplay/cache/ — the working directory of the run — and is set by cache_root. The optional subdirectory argument is part of the address: steps never leak across subdirectories; without a subdirectory, equal cache keys are reused across tests. One .py file per step.
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# Attempt budgets
|
|
2
|
+
|
|
3
|
+
Domain: generation and healing attempt budgets. Audience: engineers tuning budgets and reasoning about budget exhaustion.
|
|
4
|
+
|
|
5
|
+
## Consume attempts
|
|
6
|
+
|
|
7
|
+
```python
|
|
8
|
+
from prettyplay.cache import RunBudgets
|
|
9
|
+
|
|
10
|
+
budgets = RunBudgets(generation_limit=3, healing_limit=2)
|
|
11
|
+
|
|
12
|
+
if budgets.try_generation(identity):
|
|
13
|
+
... # attempt allowed; False — budget exhausted, the caller reports incurability
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
## Semantics
|
|
17
|
+
|
|
18
|
+
- Budgets are per step per test: every test owns its registry and starts with full limits — a step reused across tests gets a fresh budget in each test (N tests running one step in a process spend N × attempts in total)
|
|
19
|
+
- Defaults: 3 generation attempts, 2 healing attempts — configurable in the project settings
|
|
20
|
+
- An exhausted budget is the incurable failure, never an infinite loop
|
|
21
|
+
- Budgets exist only in the memory of the running process — nothing is persisted
|
|
22
|
+
|
|
23
|
+
## Healing-funded regenerations
|
|
24
|
+
|
|
25
|
+
Two verdict-driven regenerations are paid from the healing budget via `try_healing`, exactly one each:
|
|
26
|
+
|
|
27
|
+
- generation-budget exhaustion with a rot or fixable verdict grants one extra recommendation-carrying regeneration; a repeat failure is terminal — no reclassification
|
|
28
|
+
- a failed candidate check (generation loop) classified rot or fixable gets one recommendation-carrying regeneration before the final terminal-kind classification
|
|
29
|
+
|
|
30
|
+
A refused `try_healing` — the healing budget already exhausted — leaves the failure terminal: no unfunded regeneration ever runs.
|
|
31
|
+
|
|
32
|
+
Settle re-executions of the same code never consume budgets — re-execution is execution, not generation. Interactive steering attempts never consume budgets either: the human in the loop is the bound.
|
|
33
|
+
|
|
34
|
+
## Recovery budgets (groups)
|
|
35
|
+
|
|
36
|
+
The group recovery engine draws on the same registry with two group-scoped operations:
|
|
37
|
+
|
|
38
|
+
```python
|
|
39
|
+
if budgets.open_group_cycle(group_key=group_prompt):
|
|
40
|
+
for step_identity in row:
|
|
41
|
+
budgets.refresh_healing(step_identity) # a fresh full healing counter per cycle
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
- `open_group_cycle(group_key)` — consume one recovery cycle of the group; False — the per-group
|
|
45
|
+
cycle cap (the `healing_attempts` value) is exhausted → the terminal incurable failure
|
|
46
|
+
- `refresh_healing(identity)` — at every new cycle each row step gets a fresh full healing
|
|
47
|
+
counter, so a second attempt is never starved by the first
|
|
48
|
+
|
|
49
|
+
Ordinary per-step healing keeps its per-test pools unchanged. Row regenerations pay from the
|
|
50
|
+
healing budget via the ordinary `try_healing`. Group cycle counters live only in the memory of the
|
|
51
|
+
running process, like every budget.
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# Cache storage
|
|
2
|
+
|
|
3
|
+
Domain: reading and writing the step cache. Audience: engineers inspecting cache files and building tooling around the store.
|
|
4
|
+
|
|
5
|
+
## Read and write
|
|
6
|
+
|
|
7
|
+
```python
|
|
8
|
+
from prettyplay.cache import CachedStep, StepCache
|
|
9
|
+
from prettyplay.config import Config
|
|
10
|
+
from prettyplay.reporting import StepReporter
|
|
11
|
+
|
|
12
|
+
cache = StepCache(config=Config(), path="checkout", reporter=StepReporter(hooks=[]))
|
|
13
|
+
|
|
14
|
+
step = cache.load(identity) # None on a cache miss
|
|
15
|
+
if step is None:
|
|
16
|
+
# a miss means: generate the step, then store it
|
|
17
|
+
cache.save(CachedStep(identity=identity, code=step_code, created_at="2026-09-07"))
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
## File format
|
|
21
|
+
|
|
22
|
+
Each file carries the metadata fields (step sentence, cache key, step type, creation date), then the `# --- step code ---` sentinel line, then the generated step code — top-level imports when present, then the fixed-form `def step` function, restored verbatim on load. Files carry no library version and are never invalidated by a library upgrade.
|
|
23
|
+
|
|
24
|
+
## Write behavior
|
|
25
|
+
|
|
26
|
+
- save never fails the run: a read-only cache or a busy Windows target skips the write loudly — save returns whether the step was stored (False on a skip; the on_cache_skipped hook carries the reason)
|
|
27
|
+
- writes are atomic: a unique temporary file in the target directory, then an atomic replace; the last writer wins, a partial file never becomes visible
|
|
28
|
+
- load always works, in every environment
|
|
29
|
+
|
|
30
|
+
## Team workflow
|
|
31
|
+
|
|
32
|
+
Generate locally where the LLM is reachable → commit the cache directory → CI runs the whole suite from the cache with no LLM keys at all.
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""Facade of the prettyplay.cache cell: repository step cache and addressing."""
|
|
2
|
+
|
|
3
|
+
from .budgets import RunBudgets
|
|
4
|
+
from .models import CachedStep, StepIdentity
|
|
5
|
+
from .store import StepCache
|
|
6
|
+
from .text import normalize_step_text
|
|
7
|
+
|
|
8
|
+
__all__ = ["CachedStep", "RunBudgets", "StepCache", "StepIdentity", "normalize_step_text"]
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
"""Per-test attempt registry: how many tries a step still has in this test."""
|
|
2
|
+
|
|
3
|
+
from .models import StepIdentity
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class RunBudgets:
|
|
7
|
+
"""The per-test registry of generation and healing attempts per step — nothing persisted, process memory only.
|
|
8
|
+
|
|
9
|
+
Attributes:
|
|
10
|
+
_generation_limit: how many generation attempts a step may take per test.
|
|
11
|
+
_healing_limit: how many healing attempts a step may take per test.
|
|
12
|
+
_generation_used: attempts already spent per step in the generation pool.
|
|
13
|
+
_healing_used: attempts already spent per step in the healing pool.
|
|
14
|
+
_group_cycles_used: recovery cycles already opened per group in this test.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
def __init__(self, generation_limit: int, healing_limit: int) -> None:
|
|
18
|
+
"""Init the registry with separate generation and healing limits.
|
|
19
|
+
|
|
20
|
+
Args:
|
|
21
|
+
generation_limit: the per-step generation attempt limit of the test.
|
|
22
|
+
healing_limit: the per-step healing attempt limit of the test.
|
|
23
|
+
"""
|
|
24
|
+
self._generation_limit = generation_limit
|
|
25
|
+
self._healing_limit = healing_limit
|
|
26
|
+
self._generation_used: dict[str, int] = {}
|
|
27
|
+
self._healing_used: dict[str, int] = {}
|
|
28
|
+
self._group_cycles_used: dict[str, int] = {}
|
|
29
|
+
|
|
30
|
+
def try_generation(self, identity: StepIdentity) -> bool:
|
|
31
|
+
"""Spend one generation attempt of the step or refuse on exhaustion.
|
|
32
|
+
|
|
33
|
+
Args:
|
|
34
|
+
identity: the address of the step asking for an attempt.
|
|
35
|
+
|
|
36
|
+
Returns:
|
|
37
|
+
True when the attempt is granted and counted, False when the
|
|
38
|
+
per-test generation budget of the step is exhausted.
|
|
39
|
+
"""
|
|
40
|
+
used = self._generation_used.get(identity.filename, 0)
|
|
41
|
+
if used >= self._generation_limit:
|
|
42
|
+
return False
|
|
43
|
+
|
|
44
|
+
self._generation_used[identity.filename] = used + 1
|
|
45
|
+
|
|
46
|
+
return True
|
|
47
|
+
|
|
48
|
+
def try_healing(self, identity: StepIdentity) -> bool:
|
|
49
|
+
"""Spend one healing attempt of the step or refuse on exhaustion.
|
|
50
|
+
|
|
51
|
+
Args:
|
|
52
|
+
identity: the address of the step asking for an attempt.
|
|
53
|
+
|
|
54
|
+
Returns:
|
|
55
|
+
True when the attempt is granted and counted, False when the
|
|
56
|
+
per-test healing budget of the step is exhausted.
|
|
57
|
+
"""
|
|
58
|
+
used = self._healing_used.get(identity.filename, 0)
|
|
59
|
+
if used >= self._healing_limit:
|
|
60
|
+
return False
|
|
61
|
+
|
|
62
|
+
self._healing_used[identity.filename] = used + 1
|
|
63
|
+
|
|
64
|
+
return True
|
|
65
|
+
|
|
66
|
+
def refresh_healing(self, identity: StepIdentity) -> None:
|
|
67
|
+
"""Grant the step a fresh full healing counter.
|
|
68
|
+
|
|
69
|
+
Args:
|
|
70
|
+
identity: the address of the row step — the recovery refreshes
|
|
71
|
+
every row step at each new cycle; other steps' pools stay
|
|
72
|
+
untouched.
|
|
73
|
+
"""
|
|
74
|
+
self._healing_used[identity.filename] = 0
|
|
75
|
+
|
|
76
|
+
def open_group_cycle(self, group_key: str) -> bool:
|
|
77
|
+
"""Spend one recovery cycle of the group or refuse on exhaustion — the loop is never infinite.
|
|
78
|
+
|
|
79
|
+
Args:
|
|
80
|
+
group_key: the key of the group — its group prompt.
|
|
81
|
+
|
|
82
|
+
Returns:
|
|
83
|
+
True when a new cycle is open and counted, False when the
|
|
84
|
+
per-test cycle cap of the group (the healing limit) is
|
|
85
|
+
exhausted.
|
|
86
|
+
"""
|
|
87
|
+
used = self._group_cycles_used.get(group_key, 0)
|
|
88
|
+
if used >= self._healing_limit:
|
|
89
|
+
return False
|
|
90
|
+
|
|
91
|
+
self._group_cycles_used[group_key] = used + 1
|
|
92
|
+
|
|
93
|
+
return True
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"""In-memory models of the step cache: step addressing and a cached unit."""
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
|
|
5
|
+
from pydantic import BaseModel, ConfigDict
|
|
6
|
+
|
|
7
|
+
#: Unit Separator: makes the identity concatenation unambiguous.
|
|
8
|
+
_IDENTITY_SEPARATOR = "\x1f"
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class StepIdentity(BaseModel):
|
|
12
|
+
"""The address of a step in the repository cache.
|
|
13
|
+
|
|
14
|
+
Attributes:
|
|
15
|
+
cache_key: the key of the test the step belongs to.
|
|
16
|
+
step_type: the kind of the step sentence ({action, assertion}).
|
|
17
|
+
normalized_text: the normalized step sentence.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
model_config = ConfigDict(kw_only=True)
|
|
21
|
+
|
|
22
|
+
cache_key: str
|
|
23
|
+
step_type: str
|
|
24
|
+
normalized_text: str
|
|
25
|
+
|
|
26
|
+
@property
|
|
27
|
+
def filename(self) -> str:
|
|
28
|
+
"""Return the cache filename deterministically derived from the triple.
|
|
29
|
+
|
|
30
|
+
Returns:
|
|
31
|
+
The filename — the parts joined with the Unit Separator,
|
|
32
|
+
sha256-hashed, ``.py``-extended; the cache loader parses the
|
|
33
|
+
text and never imports the module by name.
|
|
34
|
+
"""
|
|
35
|
+
identity_string = _IDENTITY_SEPARATOR.join((self.cache_key, self.step_type, self.normalized_text))
|
|
36
|
+
digest = hashlib.sha256(identity_string.encode("utf-8")).hexdigest()
|
|
37
|
+
return f"{digest}.py"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class CachedStep(BaseModel):
|
|
41
|
+
"""One unit of the step cache: a working step bound to its identity.
|
|
42
|
+
|
|
43
|
+
Attributes:
|
|
44
|
+
identity: the address of the step.
|
|
45
|
+
code: the step code — top-level imports when present, then the fixed
|
|
46
|
+
form ``def step(page) -> None:``; the cache stores and restores
|
|
47
|
+
the whole text verbatim.
|
|
48
|
+
created_at: the date the code was generated (ISO format).
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
model_config = ConfigDict(kw_only=True)
|
|
52
|
+
|
|
53
|
+
identity: StepIdentity
|
|
54
|
+
code: str
|
|
55
|
+
created_at: str
|