assay-ui 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
assay_ui-0.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 assay contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,325 @@
1
+ Metadata-Version: 2.4
2
+ Name: assay-ui
3
+ Version: 0.1.0
4
+ Summary: Find out whether a generated web page actually works.
5
+ Author: awss1i
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/awss1i/assay
8
+ Project-URL: Source, https://github.com/awss1i/assay
9
+ Project-URL: Issues, https://github.com/awss1i/assay/issues
10
+ Project-URL: Changelog, https://github.com/awss1i/assay/releases
11
+ Keywords: testing,llm,codegen,playwright,evaluation,qa,ai-agents,browser-testing
12
+ Classifier: Development Status :: 4 - Beta
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.10
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Programming Language :: Python :: 3.13
20
+ Classifier: Environment :: Console
21
+ Classifier: Topic :: Software Development :: Testing
22
+ Requires-Python: >=3.10
23
+ Description-Content-Type: text/markdown
24
+ License-File: LICENSE
25
+ Requires-Dist: playwright>=1.40
26
+ Provides-Extra: dev
27
+ Requires-Dist: pytest>=7; extra == "dev"
28
+ Dynamic: license-file
29
+
30
+ # assay
31
+
32
+ [![python](https://img.shields.io/badge/python-3.10%2B-blue)](pyproject.toml)
33
+ [![licence](https://img.shields.io/badge/licence-MIT-blue)](LICENSE)
34
+
35
+ **Find out whether a generated web page actually works.**
36
+
37
+ <img src="https://raw.githubusercontent.com/awss1i/assay/main/docs/run.svg" alt="A terminal running assay against a
38
+ generated paint program. Twenty-five cases planned from the page itself,
39
+ twenty-three passed, and two failed: one because the canvas took the first
40
+ stroke and ignored the second, one because Undo answered the second press and
41
+ not the first.">
42
+
43
+ <p align="center"><i>Checking a generated paint program. Nobody wrote a test.</i></p>
44
+
45
+ Point it at a page. assay opens it in a real browser, measures every control
46
+ it renders, works out a test plan from what it finds, drives all of it, and
47
+ tells you what broke.
48
+
49
+ ## Highlights
50
+
51
+ - **No tests to write.** The plan comes from the page, so a program written
52
+ ten seconds ago can be checked ten seconds later.
53
+ - **No LLM. Purely mechanical.** No API key, no tokens, no rate limit, nothing
54
+ to bill. It gives the same answer twice.
55
+ - **No baseline images and no recordings.** Nothing to capture first, nothing
56
+ to keep in step with the design.
57
+ - **It says why.** Not *case 14 failed*, but *the first press did nothing and
58
+ the second did something, so this control is one behind*.
59
+
60
+ ## Contents
61
+
62
+ - [Install](#install)
63
+ - [CLI](#cli)
64
+ - [Skills and Plugins](#skills-and-plugins)
65
+ - [Use](#use)
66
+ - [CLI](#cli-1)
67
+ - [Skills](#skills)
68
+ - [Plugins](#plugins)
69
+ - [Benchmarks](#benchmarks)
70
+ - [Generated Programs](#generated-programs)
71
+ - [Planted Bugs](#planted-bugs)
72
+ - [How It Works](#how-it-works)
73
+ - [Why](#why)
74
+ - [Limits Worth Knowing](#limits-worth-knowing)
75
+ - [Contributing](#contributing)
76
+ - [Licence](#licence)
77
+
78
+ ## Install
79
+
80
+ ### CLI
81
+
82
+ ```bash
83
+ pip install assay-ui
84
+ ```
85
+
86
+ The command is `assay`. The first run fetches a browser if there is not one
87
+ already, so there is no second command to forget.
88
+
89
+ To hack on it, clone and install it in place:
90
+
91
+ ```bash
92
+ git clone https://github.com/awss1i/assay.git && cd assay
93
+ pip install -e ".[dev]"
94
+ ```
95
+
96
+ ### Skills and Plugins
97
+
98
+ Fifteen harnesses, one command each. All of them want the CLI above first.
99
+
100
+ **[Install it in your harness →](docs/harnesses.md)**
101
+
102
+ ## Use
103
+
104
+ ### CLI
105
+
106
+ ```console
107
+ assay ./my-app # check the page in this folder
108
+ assay ./my-app/todo.html # check one page by name
109
+ assay ./my-app -e app.html # or name the page inside a folder
110
+
111
+ assay ./my-app --report out.html # write an HTML report with screenshots
112
+ assay ./my-app --json # print the whole run as JSON
113
+ assay ./my-app --one-line # print one sentence, for a script to relay
114
+ assay ./my-app --surface # list the controls it found, then stop
115
+ ```
116
+
117
+ There is a broken drawing program in this repository. Run it yourself:
118
+
119
+ ```console
120
+ $ assay bench/programs/dsh/gpt-oss-120b/37_draw2
121
+ 23 case(s) planned, 23 carried out, 22 passed, 1 failed
122
+
123
+ C013 [ok] use canvas: click it, drag on it, and press the keys a program like this is driven with
124
+ C014 [ok] draw on canvas, then draw somewhere else on it
125
+ C015 [FAILED] draw on canvas twice, then press Undo twice
126
+ → the first press did nothing and the second did something, from the same state, so this control is one behind
127
+ C016 [ok] draw on canvas twice, then press Redo twice
128
+ C017 [ok] draw on canvas twice, then press Clear twice
129
+ ```
130
+
131
+ Drawing works. Redo works. Clear works. Undo is one press behind, and nothing
132
+ about the source says so.
133
+
134
+ **Where the results go.** Everything goes to stdout and **nothing is written
135
+ to disk unless you ask**, because a CI check that only cares about the exit
136
+ code should not litter. `--report FILE` writes one self-contained HTML page
137
+ plus a `shots/` folder of screenshots beside it. `--json` prints the whole run
138
+ for piping. The exit code is non-zero if anything failed.
139
+
140
+ `--report` gives you every case with the page as the browser drew it, before
141
+ and after:
142
+
143
+ <img src="https://raw.githubusercontent.com/awss1i/assay/main/docs/report.png" alt="One case from an assay report: the
144
+ act that was performed, the reason it failed, and screenshots of the page
145
+ before and after">
146
+
147
+ **From Python.** The same run, as an object.
148
+
149
+ ```python
150
+ from assay import check
151
+
152
+ report = check("./my-app")
153
+ print(report.summary())
154
+ for result in report.failing:
155
+ print(result.case.what, "->", result.detail)
156
+ ```
157
+
158
+ ### Skills
159
+
160
+ *Claude Code, DeepSeek Harness, opencode, Antigravity, Codex App, Codex CLI,
161
+ Cursor, Devin CLI, Factory Droid, Gemini CLI, GitHub Copilot CLI, Grok Build
162
+ CLI, Kimi Code, Pi, Hermes Agent.*
163
+
164
+ One markdown file. Your agent runs assay when it finishes a page and prints
165
+ what came back:
166
+
167
+ ```
168
+ assay: checked todo/todo.html, 8 checks, nothing flagged.
169
+ ```
170
+
171
+ One line, every time, whether or not it found anything. It reports and never
172
+ fixes: the agent hands over what it pressed and what happened, and does not
173
+ edit code on the strength of it.
174
+
175
+ **[How the skill behaves →](plugins/assay/README.md#the-skill)**
176
+
177
+ ### Plugins
178
+
179
+ *Claude Code, DeepSeek Harness.*
180
+
181
+ The same skill plus a hook, so the check happens at the end of every turn
182
+ that touched a page, whether or not the agent thought to run it.
183
+
184
+ ```
185
+ /plugin marketplace add awss1i/assay
186
+ /plugin install assay@assay
187
+ ```
188
+
189
+ **[How the plugin behaves →](plugins/assay/README.md#the-plugin)**
190
+
191
+ ## Benchmarks
192
+
193
+ A checker nobody has checked is an opinion with a progress bar. Two sets,
194
+ built differently, both checked in.
195
+
196
+ ### Generated Programs
197
+
198
+ 225 programs written to 75 objectives by three harnesses. A person opened
199
+ every one and drove it before assay saw it. 20 are broken.
200
+
201
+ <!-- score2 -->
202
+
203
+ **Across 225 pages checked by hand, assay found 15 of the 20 real defects and raised 0 false alarms.** When it reports a problem it is a real one 15 times out of 15.
204
+
205
+ <!-- /score2 -->
206
+
207
+ **[The benchmark →](bench/README.md)**
208
+
209
+ ### Planted Bugs
210
+
211
+ Ten working programs, and a copy of each with five bugs put in by a
212
+ different harness and model. Fifty defects known by construction, and the
213
+ harder set: pages that work and are wrong, not pages that stopped.
214
+
215
+ <!-- planted -->
216
+
217
+ **assay found 10 of the 50 planted bugs and flagged 0 of the 10 working originals.**
218
+
219
+ <!-- /planted -->
220
+
221
+ **[The planted set →](bench/planted/README.md)**
222
+
223
+ Both reproduce with `python bench/score.py` and `python bench/planted/score.py`.
224
+ No key, no network.
225
+
226
+ ## How It Works
227
+
228
+ **It waits until the page stops arriving**, watching whether the set of
229
+ controls is still changing rather than pausing for a fixed time. **It measures
230
+ the surface** from the rendered page and never the markup, down to a grid of
231
+ plain divs, because a pointer cursor is the page saying *click this* and a
232
+ handler attached in JavaScript cannot be read any other way. **It derives a
233
+ plan from that** and carries all of it out in a fresh tab, measuring painted
234
+ pixels per canvas, their centroid and mean colour, an exact pixel hash, the
235
+ visible text, and a hash over every element's style.
236
+
237
+ **It is deliberately narrow about what counts as a failure.** A plan derived
238
+ from the page cannot know what a control is *for*, so a button only has to
239
+ survive being pressed. Demanding that every press change something would fail
240
+ a working program for having a Clear button on an empty canvas.
241
+
242
+ What it can judge without knowing the design is whether the program
243
+ contradicts itself. A surface that took the first stroke has to take the
244
+ second. A control that does nothing on its first press and something on its
245
+ second, from the same state, is one press behind. And if nothing on the page
246
+ responds to anything, the script probably never ran.
247
+
248
+ **It never says a page is broken.** It says what it pressed and what
249
+ happened:
250
+
251
+ ```
252
+ C006 [FAILED] type into Quantity then press +
253
+ → nothing on the page changed at all
254
+ ```
255
+
256
+ A verdict makes you change code. A measurement makes you look first, and
257
+ sometimes you find the page was right and the check was aimed at the wrong
258
+ thing. That is the cheaper mistake to make, and it reads the same either way
259
+ round: a person knows where to start, and an agent gets a precise place to
260
+ look instead of a whole file to re-read.
261
+
262
+ **It serves the folder and never builds it.** The page is served over
263
+ loopback rather than opened off the disk, because a `file://` origin blocks
264
+ every module the page loads. What it will not do is run your build: `npm
265
+ install` executes whatever the dependency tree asks for, as whoever typed the
266
+ command, and this is a tool for checking code nobody has read. Point it at a
267
+ source tree and it says so:
268
+
269
+ ```console
270
+ $ assay ./my-vite-app
271
+ assay: index.html loads /src/main.jsx, which a browser cannot run. This is a source tree, not a built one.
272
+ Build it first, in your own shell, then check the output:
273
+ npm install && npm run build && assay ./my-vite-app/dist
274
+ ```
275
+
276
+ **[Every rule →](docs/how-it-works.md)**
277
+
278
+ ## Why
279
+
280
+ A lot of code is written by models now, and *"does this actually run?"* is
281
+ mostly still answered by a person opening it and clicking around.
282
+
283
+ Every existing tool needs something you do not have for a program that was
284
+ generated ten seconds ago. Playwright and Cypress need tests somebody wrote.
285
+ Visual regression needs a golden image to compare against. Benchmarks like
286
+ SWE-bench use the repository's own suite.
287
+
288
+ So the thing most people reach for instead is another model: paste the code in
289
+ and ask whether it looks right. **That is a reader guessing about code.** assay
290
+ opens the page and drives it, which is the only way to find out that a button
291
+ does nothing.
292
+
293
+ | | needs tests written | needs a baseline | runs the program |
294
+ |---|---|---|---|
295
+ | Playwright / Cypress | yes | no | yes, the parts you wrote |
296
+ | Percy / Chromatic | no | **yes** | it screenshots it |
297
+ | ask a model to review it | no | no | **no. It reads the source** |
298
+ | **assay** | **no** | **no** | **yes, all of it** |
299
+
300
+ ## Limits Worth Knowing
301
+
302
+ - **Browser programs.** It opens a page. A program with no page is not
303
+ something it can measure.
304
+ - **Coverage cannot judge intent.** A control that works mechanically and does
305
+ the wrong thing passes. Criteria are the answer, and you have to write those.
306
+ - **A game that ends looks like a page that died.** When a program finishes
307
+ and offers no way to start again, it stops responding to anything, and a
308
+ plan derived from the page cannot tell that apart from a page that broke.
309
+ - **Built output, not source trees.** It will not run your build. Point it at
310
+ a source tree and it says so and names the command.
311
+ - **Single-page programs.** It checks the page you point it at and does not
312
+ crawl. A multi-page site means running it per page, and client-side routing
313
+ is untested.
314
+ - **Generated pages, not the live web.** It is built for programs somebody
315
+ just generated, not for sites with a login, a cookie banner or live network
316
+ calls.
317
+
318
+ ## Contributing
319
+
320
+ Issues and pull requests are welcome. Setup, tests, benchmarks and what a PR
321
+ needs are in **[CONTRIBUTING.md](CONTRIBUTING.md)**.
322
+
323
+ ## Licence
324
+
325
+ MIT.
@@ -0,0 +1,296 @@
1
+ # assay
2
+
3
+ [![python](https://img.shields.io/badge/python-3.10%2B-blue)](pyproject.toml)
4
+ [![licence](https://img.shields.io/badge/licence-MIT-blue)](LICENSE)
5
+
6
+ **Find out whether a generated web page actually works.**
7
+
8
+ <img src="https://raw.githubusercontent.com/awss1i/assay/main/docs/run.svg" alt="A terminal running assay against a
9
+ generated paint program. Twenty-five cases planned from the page itself,
10
+ twenty-three passed, and two failed: one because the canvas took the first
11
+ stroke and ignored the second, one because Undo answered the second press and
12
+ not the first.">
13
+
14
+ <p align="center"><i>Checking a generated paint program. Nobody wrote a test.</i></p>
15
+
16
+ Point it at a page. assay opens it in a real browser, measures every control
17
+ it renders, works out a test plan from what it finds, drives all of it, and
18
+ tells you what broke.
19
+
20
+ ## Highlights
21
+
22
+ - **No tests to write.** The plan comes from the page, so a program written
23
+ ten seconds ago can be checked ten seconds later.
24
+ - **No LLM. Purely mechanical.** No API key, no tokens, no rate limit, nothing
25
+ to bill. It gives the same answer twice.
26
+ - **No baseline images and no recordings.** Nothing to capture first, nothing
27
+ to keep in step with the design.
28
+ - **It says why.** Not *case 14 failed*, but *the first press did nothing and
29
+ the second did something, so this control is one behind*.
30
+
31
+ ## Contents
32
+
33
+ - [Install](#install)
34
+ - [CLI](#cli)
35
+ - [Skills and Plugins](#skills-and-plugins)
36
+ - [Use](#use)
37
+ - [CLI](#cli-1)
38
+ - [Skills](#skills)
39
+ - [Plugins](#plugins)
40
+ - [Benchmarks](#benchmarks)
41
+ - [Generated Programs](#generated-programs)
42
+ - [Planted Bugs](#planted-bugs)
43
+ - [How It Works](#how-it-works)
44
+ - [Why](#why)
45
+ - [Limits Worth Knowing](#limits-worth-knowing)
46
+ - [Contributing](#contributing)
47
+ - [Licence](#licence)
48
+
49
+ ## Install
50
+
51
+ ### CLI
52
+
53
+ ```bash
54
+ pip install assay-ui
55
+ ```
56
+
57
+ The command is `assay`. The first run fetches a browser if there is not one
58
+ already, so there is no second command to forget.
59
+
60
+ To hack on it, clone and install it in place:
61
+
62
+ ```bash
63
+ git clone https://github.com/awss1i/assay.git && cd assay
64
+ pip install -e ".[dev]"
65
+ ```
66
+
67
+ ### Skills and Plugins
68
+
69
+ Fifteen harnesses, one command each. All of them want the CLI above first.
70
+
71
+ **[Install it in your harness →](docs/harnesses.md)**
72
+
73
+ ## Use
74
+
75
+ ### CLI
76
+
77
+ ```console
78
+ assay ./my-app # check the page in this folder
79
+ assay ./my-app/todo.html # check one page by name
80
+ assay ./my-app -e app.html # or name the page inside a folder
81
+
82
+ assay ./my-app --report out.html # write an HTML report with screenshots
83
+ assay ./my-app --json # print the whole run as JSON
84
+ assay ./my-app --one-line # print one sentence, for a script to relay
85
+ assay ./my-app --surface # list the controls it found, then stop
86
+ ```
87
+
88
+ There is a broken drawing program in this repository. Run it yourself:
89
+
90
+ ```console
91
+ $ assay bench/programs/dsh/gpt-oss-120b/37_draw2
92
+ 23 case(s) planned, 23 carried out, 22 passed, 1 failed
93
+
94
+ C013 [ok] use canvas: click it, drag on it, and press the keys a program like this is driven with
95
+ C014 [ok] draw on canvas, then draw somewhere else on it
96
+ C015 [FAILED] draw on canvas twice, then press Undo twice
97
+ → the first press did nothing and the second did something, from the same state, so this control is one behind
98
+ C016 [ok] draw on canvas twice, then press Redo twice
99
+ C017 [ok] draw on canvas twice, then press Clear twice
100
+ ```
101
+
102
+ Drawing works. Redo works. Clear works. Undo is one press behind, and nothing
103
+ about the source says so.
104
+
105
+ **Where the results go.** Everything goes to stdout and **nothing is written
106
+ to disk unless you ask**, because a CI check that only cares about the exit
107
+ code should not litter. `--report FILE` writes one self-contained HTML page
108
+ plus a `shots/` folder of screenshots beside it. `--json` prints the whole run
109
+ for piping. The exit code is non-zero if anything failed.
110
+
111
+ `--report` gives you every case with the page as the browser drew it, before
112
+ and after:
113
+
114
+ <img src="https://raw.githubusercontent.com/awss1i/assay/main/docs/report.png" alt="One case from an assay report: the
115
+ act that was performed, the reason it failed, and screenshots of the page
116
+ before and after">
117
+
118
+ **From Python.** The same run, as an object.
119
+
120
+ ```python
121
+ from assay import check
122
+
123
+ report = check("./my-app")
124
+ print(report.summary())
125
+ for result in report.failing:
126
+ print(result.case.what, "->", result.detail)
127
+ ```
128
+
129
+ ### Skills
130
+
131
+ *Claude Code, DeepSeek Harness, opencode, Antigravity, Codex App, Codex CLI,
132
+ Cursor, Devin CLI, Factory Droid, Gemini CLI, GitHub Copilot CLI, Grok Build
133
+ CLI, Kimi Code, Pi, Hermes Agent.*
134
+
135
+ One markdown file. Your agent runs assay when it finishes a page and prints
136
+ what came back:
137
+
138
+ ```
139
+ assay: checked todo/todo.html, 8 checks, nothing flagged.
140
+ ```
141
+
142
+ One line, every time, whether or not it found anything. It reports and never
143
+ fixes: the agent hands over what it pressed and what happened, and does not
144
+ edit code on the strength of it.
145
+
146
+ **[How the skill behaves →](plugins/assay/README.md#the-skill)**
147
+
148
+ ### Plugins
149
+
150
+ *Claude Code, DeepSeek Harness.*
151
+
152
+ The same skill plus a hook, so the check happens at the end of every turn
153
+ that touched a page, whether or not the agent thought to run it.
154
+
155
+ ```
156
+ /plugin marketplace add awss1i/assay
157
+ /plugin install assay@assay
158
+ ```
159
+
160
+ **[How the plugin behaves →](plugins/assay/README.md#the-plugin)**
161
+
162
+ ## Benchmarks
163
+
164
+ A checker nobody has checked is an opinion with a progress bar. Two sets,
165
+ built differently, both checked in.
166
+
167
+ ### Generated Programs
168
+
169
+ 225 programs written to 75 objectives by three harnesses. A person opened
170
+ every one and drove it before assay saw it. 20 are broken.
171
+
172
+ <!-- score2 -->
173
+
174
+ **Across 225 pages checked by hand, assay found 15 of the 20 real defects and raised 0 false alarms.** When it reports a problem it is a real one 15 times out of 15.
175
+
176
+ <!-- /score2 -->
177
+
178
+ **[The benchmark →](bench/README.md)**
179
+
180
+ ### Planted Bugs
181
+
182
+ Ten working programs, and a copy of each with five bugs put in by a
183
+ different harness and model. Fifty defects known by construction, and the
184
+ harder set: pages that work and are wrong, not pages that stopped.
185
+
186
+ <!-- planted -->
187
+
188
+ **assay found 10 of the 50 planted bugs and flagged 0 of the 10 working originals.**
189
+
190
+ <!-- /planted -->
191
+
192
+ **[The planted set →](bench/planted/README.md)**
193
+
194
+ Both reproduce with `python bench/score.py` and `python bench/planted/score.py`.
195
+ No key, no network.
196
+
197
+ ## How It Works
198
+
199
+ **It waits until the page stops arriving**, watching whether the set of
200
+ controls is still changing rather than pausing for a fixed time. **It measures
201
+ the surface** from the rendered page and never the markup, down to a grid of
202
+ plain divs, because a pointer cursor is the page saying *click this* and a
203
+ handler attached in JavaScript cannot be read any other way. **It derives a
204
+ plan from that** and carries all of it out in a fresh tab, measuring painted
205
+ pixels per canvas, their centroid and mean colour, an exact pixel hash, the
206
+ visible text, and a hash over every element's style.
207
+
208
+ **It is deliberately narrow about what counts as a failure.** A plan derived
209
+ from the page cannot know what a control is *for*, so a button only has to
210
+ survive being pressed. Demanding that every press change something would fail
211
+ a working program for having a Clear button on an empty canvas.
212
+
213
+ What it can judge without knowing the design is whether the program
214
+ contradicts itself. A surface that took the first stroke has to take the
215
+ second. A control that does nothing on its first press and something on its
216
+ second, from the same state, is one press behind. And if nothing on the page
217
+ responds to anything, the script probably never ran.
218
+
219
+ **It never says a page is broken.** It says what it pressed and what
220
+ happened:
221
+
222
+ ```
223
+ C006 [FAILED] type into Quantity then press +
224
+ → nothing on the page changed at all
225
+ ```
226
+
227
+ A verdict makes you change code. A measurement makes you look first, and
228
+ sometimes you find the page was right and the check was aimed at the wrong
229
+ thing. That is the cheaper mistake to make, and it reads the same either way
230
+ round: a person knows where to start, and an agent gets a precise place to
231
+ look instead of a whole file to re-read.
232
+
233
+ **It serves the folder and never builds it.** The page is served over
234
+ loopback rather than opened off the disk, because a `file://` origin blocks
235
+ every module the page loads. What it will not do is run your build: `npm
236
+ install` executes whatever the dependency tree asks for, as whoever typed the
237
+ command, and this is a tool for checking code nobody has read. Point it at a
238
+ source tree and it says so:
239
+
240
+ ```console
241
+ $ assay ./my-vite-app
242
+ assay: index.html loads /src/main.jsx, which a browser cannot run. This is a source tree, not a built one.
243
+ Build it first, in your own shell, then check the output:
244
+ npm install && npm run build && assay ./my-vite-app/dist
245
+ ```
246
+
247
+ **[Every rule →](docs/how-it-works.md)**
248
+
249
+ ## Why
250
+
251
+ A lot of code is written by models now, and *"does this actually run?"* is
252
+ mostly still answered by a person opening it and clicking around.
253
+
254
+ Every existing tool needs something you do not have for a program that was
255
+ generated ten seconds ago. Playwright and Cypress need tests somebody wrote.
256
+ Visual regression needs a golden image to compare against. Benchmarks like
257
+ SWE-bench use the repository's own suite.
258
+
259
+ So the thing most people reach for instead is another model: paste the code in
260
+ and ask whether it looks right. **That is a reader guessing about code.** assay
261
+ opens the page and drives it, which is the only way to find out that a button
262
+ does nothing.
263
+
264
+ | | needs tests written | needs a baseline | runs the program |
265
+ |---|---|---|---|
266
+ | Playwright / Cypress | yes | no | yes, the parts you wrote |
267
+ | Percy / Chromatic | no | **yes** | it screenshots it |
268
+ | ask a model to review it | no | no | **no. It reads the source** |
269
+ | **assay** | **no** | **no** | **yes, all of it** |
270
+
271
+ ## Limits Worth Knowing
272
+
273
+ - **Browser programs.** It opens a page. A program with no page is not
274
+ something it can measure.
275
+ - **Coverage cannot judge intent.** A control that works mechanically and does
276
+ the wrong thing passes. Criteria are the answer, and you have to write those.
277
+ - **A game that ends looks like a page that died.** When a program finishes
278
+ and offers no way to start again, it stops responding to anything, and a
279
+ plan derived from the page cannot tell that apart from a page that broke.
280
+ - **Built output, not source trees.** It will not run your build. Point it at
281
+ a source tree and it says so and names the command.
282
+ - **Single-page programs.** It checks the page you point it at and does not
283
+ crawl. A multi-page site means running it per page, and client-side routing
284
+ is untested.
285
+ - **Generated pages, not the live web.** It is built for programs somebody
286
+ just generated, not for sites with a login, a cookie banner or live network
287
+ calls.
288
+
289
+ ## Contributing
290
+
291
+ Issues and pull requests are welcome. Setup, tests, benchmarks and what a PR
292
+ needs are in **[CONTRIBUTING.md](CONTRIBUTING.md)**.
293
+
294
+ ## Licence
295
+
296
+ MIT.
@@ -0,0 +1,42 @@
1
+ [project]
2
+ name = "assay-ui"
3
+ version = "0.1.0"
4
+ description = "Find out whether a generated web page actually works."
5
+ keywords = ["testing", "llm", "codegen", "playwright", "evaluation", "qa",
6
+ "ai-agents", "browser-testing"]
7
+ authors = [{ name = "awss1i" }]
8
+ classifiers = [
9
+ "Development Status :: 4 - Beta",
10
+ "Intended Audience :: Developers",
11
+ "License :: OSI Approved :: MIT License",
12
+ "Programming Language :: Python :: 3",
13
+ "Programming Language :: Python :: 3.10",
14
+ "Programming Language :: Python :: 3.11",
15
+ "Programming Language :: Python :: 3.12",
16
+ "Programming Language :: Python :: 3.13",
17
+ "Environment :: Console",
18
+ "Topic :: Software Development :: Testing",
19
+ ]
20
+ readme = "README.md"
21
+ requires-python = ">=3.10"
22
+ license = { text = "MIT" }
23
+ dependencies = ["playwright>=1.40"]
24
+
25
+ [project.urls]
26
+ Homepage = "https://github.com/awss1i/assay"
27
+ Source = "https://github.com/awss1i/assay"
28
+ Issues = "https://github.com/awss1i/assay/issues"
29
+ Changelog = "https://github.com/awss1i/assay/releases"
30
+
31
+ [project.scripts]
32
+ assay = "assay.cli:main"
33
+
34
+ [project.optional-dependencies]
35
+ dev = ["pytest>=7"]
36
+
37
+ [build-system]
38
+ requires = ["setuptools>=68"]
39
+ build-backend = "setuptools.build_meta"
40
+
41
+ [tool.setuptools.packages.find]
42
+ where = ["src"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+