assay-ui 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- assay_ui-0.1.0/LICENSE +21 -0
- assay_ui-0.1.0/PKG-INFO +325 -0
- assay_ui-0.1.0/README.md +296 -0
- assay_ui-0.1.0/pyproject.toml +42 -0
- assay_ui-0.1.0/setup.cfg +4 -0
- assay_ui-0.1.0/src/assay/__init__.py +24 -0
- assay_ui-0.1.0/src/assay/browser.py +1516 -0
- assay_ui-0.1.0/src/assay/cli.py +278 -0
- assay_ui-0.1.0/src/assay/qa.py +385 -0
- assay_ui-0.1.0/src/assay/report.py +175 -0
- assay_ui-0.1.0/src/assay/run.py +1526 -0
- assay_ui-0.1.0/src/assay/surface.py +658 -0
- assay_ui-0.1.0/src/assay/tint.py +43 -0
- assay_ui-0.1.0/src/assay_ui.egg-info/PKG-INFO +325 -0
- assay_ui-0.1.0/src/assay_ui.egg-info/SOURCES.txt +23 -0
- assay_ui-0.1.0/src/assay_ui.egg-info/dependency_links.txt +1 -0
- assay_ui-0.1.0/src/assay_ui.egg-info/entry_points.txt +2 -0
- assay_ui-0.1.0/src/assay_ui.egg-info/requires.txt +4 -0
- assay_ui-0.1.0/src/assay_ui.egg-info/top_level.txt +1 -0
- assay_ui-0.1.0/tests/test_cli.py +167 -0
- assay_ui-0.1.0/tests/test_end_to_end.py +1728 -0
- assay_ui-0.1.0/tests/test_hook.py +149 -0
- assay_ui-0.1.0/tests/test_judge.py +229 -0
- assay_ui-0.1.0/tests/test_one_line.py +95 -0
- assay_ui-0.1.0/tests/test_plan.py +315 -0
assay_ui-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 assay contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
assay_ui-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,325 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: assay-ui
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Find out whether a generated web page actually works.
|
|
5
|
+
Author: awss1i
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/awss1i/assay
|
|
8
|
+
Project-URL: Source, https://github.com/awss1i/assay
|
|
9
|
+
Project-URL: Issues, https://github.com/awss1i/assay/issues
|
|
10
|
+
Project-URL: Changelog, https://github.com/awss1i/assay/releases
|
|
11
|
+
Keywords: testing,llm,codegen,playwright,evaluation,qa,ai-agents,browser-testing
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Environment :: Console
|
|
21
|
+
Classifier: Topic :: Software Development :: Testing
|
|
22
|
+
Requires-Python: >=3.10
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
License-File: LICENSE
|
|
25
|
+
Requires-Dist: playwright>=1.40
|
|
26
|
+
Provides-Extra: dev
|
|
27
|
+
Requires-Dist: pytest>=7; extra == "dev"
|
|
28
|
+
Dynamic: license-file
|
|
29
|
+
|
|
30
|
+
# assay
|
|
31
|
+
|
|
32
|
+
[](pyproject.toml)
|
|
33
|
+
[](LICENSE)
|
|
34
|
+
|
|
35
|
+
**Find out whether a generated web page actually works.**
|
|
36
|
+
|
|
37
|
+
<img src="https://raw.githubusercontent.com/awss1i/assay/main/docs/run.svg" alt="A terminal running assay against a
|
|
38
|
+
generated paint program. Twenty-five cases planned from the page itself,
|
|
39
|
+
twenty-three passed, and two failed: one because the canvas took the first
|
|
40
|
+
stroke and ignored the second, one because Undo answered the second press and
|
|
41
|
+
not the first.">
|
|
42
|
+
|
|
43
|
+
<p align="center"><i>Checking a generated paint program. Nobody wrote a test.</i></p>
|
|
44
|
+
|
|
45
|
+
Point it at a page. assay opens it in a real browser, measures every control
|
|
46
|
+
it renders, works out a test plan from what it finds, drives all of it, and
|
|
47
|
+
tells you what broke.
|
|
48
|
+
|
|
49
|
+
## Highlights
|
|
50
|
+
|
|
51
|
+
- **No tests to write.** The plan comes from the page, so a program written
|
|
52
|
+
ten seconds ago can be checked ten seconds later.
|
|
53
|
+
- **No LLM. Purely mechanical.** No API key, no tokens, no rate limit, nothing
|
|
54
|
+
to bill. It gives the same answer twice.
|
|
55
|
+
- **No baseline images and no recordings.** Nothing to capture first, nothing
|
|
56
|
+
to keep in step with the design.
|
|
57
|
+
- **It says why.** Not *case 14 failed*, but *the first press did nothing and
|
|
58
|
+
the second did something, so this control is one behind*.
|
|
59
|
+
|
|
60
|
+
## Contents
|
|
61
|
+
|
|
62
|
+
- [Install](#install)
|
|
63
|
+
- [CLI](#cli)
|
|
64
|
+
- [Skills and Plugins](#skills-and-plugins)
|
|
65
|
+
- [Use](#use)
|
|
66
|
+
- [CLI](#cli-1)
|
|
67
|
+
- [Skills](#skills)
|
|
68
|
+
- [Plugins](#plugins)
|
|
69
|
+
- [Benchmarks](#benchmarks)
|
|
70
|
+
- [Generated Programs](#generated-programs)
|
|
71
|
+
- [Planted Bugs](#planted-bugs)
|
|
72
|
+
- [How It Works](#how-it-works)
|
|
73
|
+
- [Why](#why)
|
|
74
|
+
- [Limits Worth Knowing](#limits-worth-knowing)
|
|
75
|
+
- [Contributing](#contributing)
|
|
76
|
+
- [Licence](#licence)
|
|
77
|
+
|
|
78
|
+
## Install
|
|
79
|
+
|
|
80
|
+
### CLI
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
pip install assay-ui
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
The command is `assay`. The first run fetches a browser if there is not one
|
|
87
|
+
already, so there is no second command to forget.
|
|
88
|
+
|
|
89
|
+
To hack on it, clone and install it in place:
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
git clone https://github.com/awss1i/assay.git && cd assay
|
|
93
|
+
pip install -e ".[dev]"
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
### Skills and Plugins
|
|
97
|
+
|
|
98
|
+
Fifteen harnesses, one command each. All of them want the CLI above first.
|
|
99
|
+
|
|
100
|
+
**[Install it in your harness →](docs/harnesses.md)**
|
|
101
|
+
|
|
102
|
+
## Use
|
|
103
|
+
|
|
104
|
+
### CLI
|
|
105
|
+
|
|
106
|
+
```console
|
|
107
|
+
assay ./my-app # check the page in this folder
|
|
108
|
+
assay ./my-app/todo.html # check one page by name
|
|
109
|
+
assay ./my-app -e app.html # or name the page inside a folder
|
|
110
|
+
|
|
111
|
+
assay ./my-app --report out.html # write an HTML report with screenshots
|
|
112
|
+
assay ./my-app --json # print the whole run as JSON
|
|
113
|
+
assay ./my-app --one-line # print one sentence, for a script to relay
|
|
114
|
+
assay ./my-app --surface # list the controls it found, then stop
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
There is a broken drawing program in this repository. Run it yourself:
|
|
118
|
+
|
|
119
|
+
```console
|
|
120
|
+
$ assay bench/programs/dsh/gpt-oss-120b/37_draw2
|
|
121
|
+
23 case(s) planned, 23 carried out, 22 passed, 1 failed
|
|
122
|
+
|
|
123
|
+
C013 [ok] use canvas: click it, drag on it, and press the keys a program like this is driven with
|
|
124
|
+
C014 [ok] draw on canvas, then draw somewhere else on it
|
|
125
|
+
C015 [FAILED] draw on canvas twice, then press Undo twice
|
|
126
|
+
→ the first press did nothing and the second did something, from the same state, so this control is one behind
|
|
127
|
+
C016 [ok] draw on canvas twice, then press Redo twice
|
|
128
|
+
C017 [ok] draw on canvas twice, then press Clear twice
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
Drawing works. Redo works. Clear works. Undo is one press behind, and nothing
|
|
132
|
+
about the source says so.
|
|
133
|
+
|
|
134
|
+
**Where the results go.** Everything goes to stdout and **nothing is written
|
|
135
|
+
to disk unless you ask**, because a CI check that only cares about the exit
|
|
136
|
+
code should not litter. `--report FILE` writes one self-contained HTML page
|
|
137
|
+
plus a `shots/` folder of screenshots beside it. `--json` prints the whole run
|
|
138
|
+
for piping. The exit code is non-zero if anything failed.
|
|
139
|
+
|
|
140
|
+
`--report` gives you every case with the page as the browser drew it, before
|
|
141
|
+
and after:
|
|
142
|
+
|
|
143
|
+
<img src="https://raw.githubusercontent.com/awss1i/assay/main/docs/report.png" alt="One case from an assay report: the
|
|
144
|
+
act that was performed, the reason it failed, and screenshots of the page
|
|
145
|
+
before and after">
|
|
146
|
+
|
|
147
|
+
**From Python.** The same run, as an object.
|
|
148
|
+
|
|
149
|
+
```python
|
|
150
|
+
from assay import check
|
|
151
|
+
|
|
152
|
+
report = check("./my-app")
|
|
153
|
+
print(report.summary())
|
|
154
|
+
for result in report.failing:
|
|
155
|
+
print(result.case.what, "->", result.detail)
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
### Skills
|
|
159
|
+
|
|
160
|
+
*Claude Code, DeepSeek Harness, opencode, Antigravity, Codex App, Codex CLI,
|
|
161
|
+
Cursor, Devin CLI, Factory Droid, Gemini CLI, GitHub Copilot CLI, Grok Build
|
|
162
|
+
CLI, Kimi Code, Pi, Hermes Agent.*
|
|
163
|
+
|
|
164
|
+
One markdown file. Your agent runs assay when it finishes a page and prints
|
|
165
|
+
what came back:
|
|
166
|
+
|
|
167
|
+
```
|
|
168
|
+
assay: checked todo/todo.html, 8 checks, nothing flagged.
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
One line, every time, whether or not it found anything. It reports and never
|
|
172
|
+
fixes: the agent hands over what it pressed and what happened, and does not
|
|
173
|
+
edit code on the strength of it.
|
|
174
|
+
|
|
175
|
+
**[How the skill behaves →](plugins/assay/README.md#the-skill)**
|
|
176
|
+
|
|
177
|
+
### Plugins
|
|
178
|
+
|
|
179
|
+
*Claude Code, DeepSeek Harness.*
|
|
180
|
+
|
|
181
|
+
The same skill plus a hook, so the check happens at the end of every turn
|
|
182
|
+
that touched a page, whether or not the agent thought to run it.
|
|
183
|
+
|
|
184
|
+
```
|
|
185
|
+
/plugin marketplace add awss1i/assay
|
|
186
|
+
/plugin install assay@assay
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
**[How the plugin behaves →](plugins/assay/README.md#the-plugin)**
|
|
190
|
+
|
|
191
|
+
## Benchmarks
|
|
192
|
+
|
|
193
|
+
A checker nobody has checked is an opinion with a progress bar. Two sets,
|
|
194
|
+
built differently, both checked in.
|
|
195
|
+
|
|
196
|
+
### Generated Programs
|
|
197
|
+
|
|
198
|
+
225 programs written to 75 objectives by three harnesses. A person opened
|
|
199
|
+
every one and drove it before assay saw it. 20 are broken.
|
|
200
|
+
|
|
201
|
+
<!-- score2 -->
|
|
202
|
+
|
|
203
|
+
**Across 225 pages checked by hand, assay found 15 of the 20 real defects and raised 0 false alarms.** When it reports a problem it is a real one 15 times out of 15.
|
|
204
|
+
|
|
205
|
+
<!-- /score2 -->
|
|
206
|
+
|
|
207
|
+
**[The benchmark →](bench/README.md)**
|
|
208
|
+
|
|
209
|
+
### Planted Bugs
|
|
210
|
+
|
|
211
|
+
Ten working programs, and a copy of each with five bugs put in by a
|
|
212
|
+
different harness and model. Fifty defects known by construction, and the
|
|
213
|
+
harder set: pages that work and are wrong, not pages that stopped.
|
|
214
|
+
|
|
215
|
+
<!-- planted -->
|
|
216
|
+
|
|
217
|
+
**assay found 10 of the 50 planted bugs and flagged 0 of the 10 working originals.**
|
|
218
|
+
|
|
219
|
+
<!-- /planted -->
|
|
220
|
+
|
|
221
|
+
**[The planted set →](bench/planted/README.md)**
|
|
222
|
+
|
|
223
|
+
Both reproduce with `python bench/score.py` and `python bench/planted/score.py`.
|
|
224
|
+
No key, no network.
|
|
225
|
+
|
|
226
|
+
## How It Works
|
|
227
|
+
|
|
228
|
+
**It waits until the page stops arriving**, watching whether the set of
|
|
229
|
+
controls is still changing rather than pausing for a fixed time. **It measures
|
|
230
|
+
the surface** from the rendered page and never the markup, down to a grid of
|
|
231
|
+
plain divs, because a pointer cursor is the page saying *click this* and a
|
|
232
|
+
handler attached in JavaScript cannot be read any other way. **It derives a
|
|
233
|
+
plan from that** and carries all of it out in a fresh tab, measuring painted
|
|
234
|
+
pixels per canvas, their centroid and mean colour, an exact pixel hash, the
|
|
235
|
+
visible text, and a hash over every element's style.
|
|
236
|
+
|
|
237
|
+
**It is deliberately narrow about what counts as a failure.** A plan derived
|
|
238
|
+
from the page cannot know what a control is *for*, so a button only has to
|
|
239
|
+
survive being pressed. Demanding that every press change something would fail
|
|
240
|
+
a working program for having a Clear button on an empty canvas.
|
|
241
|
+
|
|
242
|
+
What it can judge without knowing the design is whether the program
|
|
243
|
+
contradicts itself. A surface that took the first stroke has to take the
|
|
244
|
+
second. A control that does nothing on its first press and something on its
|
|
245
|
+
second, from the same state, is one press behind. And if nothing on the page
|
|
246
|
+
responds to anything, the script probably never ran.
|
|
247
|
+
|
|
248
|
+
**It never says a page is broken.** It says what it pressed and what
|
|
249
|
+
happened:
|
|
250
|
+
|
|
251
|
+
```
|
|
252
|
+
C006 [FAILED] type into Quantity then press +
|
|
253
|
+
→ nothing on the page changed at all
|
|
254
|
+
```
|
|
255
|
+
|
|
256
|
+
A verdict makes you change code. A measurement makes you look first, and
|
|
257
|
+
sometimes you find the page was right and the check was aimed at the wrong
|
|
258
|
+
thing. That is the cheaper mistake to make, and it reads the same either way
|
|
259
|
+
round: a person knows where to start, and an agent gets a precise place to
|
|
260
|
+
look instead of a whole file to re-read.
|
|
261
|
+
|
|
262
|
+
**It serves the folder and never builds it.** The page is served over
|
|
263
|
+
loopback rather than opened off the disk, because a `file://` origin blocks
|
|
264
|
+
every module the page loads. What it will not do is run your build: `npm
|
|
265
|
+
install` executes whatever the dependency tree asks for, as whoever typed the
|
|
266
|
+
command, and this is a tool for checking code nobody has read. Point it at a
|
|
267
|
+
source tree and it says so:
|
|
268
|
+
|
|
269
|
+
```console
|
|
270
|
+
$ assay ./my-vite-app
|
|
271
|
+
assay: index.html loads /src/main.jsx, which a browser cannot run. This is a source tree, not a built one.
|
|
272
|
+
Build it first, in your own shell, then check the output:
|
|
273
|
+
npm install && npm run build && assay ./my-vite-app/dist
|
|
274
|
+
```
|
|
275
|
+
|
|
276
|
+
**[Every rule →](docs/how-it-works.md)**
|
|
277
|
+
|
|
278
|
+
## Why
|
|
279
|
+
|
|
280
|
+
A lot of code is written by models now, and *"does this actually run?"* is
|
|
281
|
+
mostly still answered by a person opening it and clicking around.
|
|
282
|
+
|
|
283
|
+
Every existing tool needs something you do not have for a program that was
|
|
284
|
+
generated ten seconds ago. Playwright and Cypress need tests somebody wrote.
|
|
285
|
+
Visual regression needs a golden image to compare against. Benchmarks like
|
|
286
|
+
SWE-bench use the repository's own suite.
|
|
287
|
+
|
|
288
|
+
So the thing most people reach for instead is another model: paste the code in
|
|
289
|
+
and ask whether it looks right. **That is a reader guessing about code.** assay
|
|
290
|
+
opens the page and drives it, which is the only way to find out that a button
|
|
291
|
+
does nothing.
|
|
292
|
+
|
|
293
|
+
| | needs tests written | needs a baseline | runs the program |
|
|
294
|
+
|---|---|---|---|
|
|
295
|
+
| Playwright / Cypress | yes | no | yes, the parts you wrote |
|
|
296
|
+
| Percy / Chromatic | no | **yes** | it screenshots it |
|
|
297
|
+
| ask a model to review it | no | no | **no. It reads the source** |
|
|
298
|
+
| **assay** | **no** | **no** | **yes, all of it** |
|
|
299
|
+
|
|
300
|
+
## Limits Worth Knowing
|
|
301
|
+
|
|
302
|
+
- **Browser programs.** It opens a page. A program with no page is not
|
|
303
|
+
something it can measure.
|
|
304
|
+
- **Coverage cannot judge intent.** A control that works mechanically and does
|
|
305
|
+
the wrong thing passes. Criteria are the answer, and you have to write those.
|
|
306
|
+
- **A game that ends looks like a page that died.** When a program finishes
|
|
307
|
+
and offers no way to start again, it stops responding to anything, and a
|
|
308
|
+
plan derived from the page cannot tell that apart from a page that broke.
|
|
309
|
+
- **Built output, not source trees.** It will not run your build. Point it at
|
|
310
|
+
a source tree and it says so and names the command.
|
|
311
|
+
- **Single-page programs.** It checks the page you point it at and does not
|
|
312
|
+
crawl. A multi-page site means running it per page, and client-side routing
|
|
313
|
+
is untested.
|
|
314
|
+
- **Generated pages, not the live web.** It is built for programs somebody
|
|
315
|
+
just generated, not for sites with a login, a cookie banner or live network
|
|
316
|
+
calls.
|
|
317
|
+
|
|
318
|
+
## Contributing
|
|
319
|
+
|
|
320
|
+
Issues and pull requests are welcome. Setup, tests, benchmarks and what a PR
|
|
321
|
+
needs are in **[CONTRIBUTING.md](CONTRIBUTING.md)**.
|
|
322
|
+
|
|
323
|
+
## Licence
|
|
324
|
+
|
|
325
|
+
MIT.
|
assay_ui-0.1.0/README.md
ADDED
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
# assay
|
|
2
|
+
|
|
3
|
+
[](pyproject.toml)
|
|
4
|
+
[](LICENSE)
|
|
5
|
+
|
|
6
|
+
**Find out whether a generated web page actually works.**
|
|
7
|
+
|
|
8
|
+
<img src="https://raw.githubusercontent.com/awss1i/assay/main/docs/run.svg" alt="A terminal running assay against a
|
|
9
|
+
generated paint program. Twenty-five cases planned from the page itself,
|
|
10
|
+
twenty-three passed, and two failed: one because the canvas took the first
|
|
11
|
+
stroke and ignored the second, one because Undo answered the second press and
|
|
12
|
+
not the first.">
|
|
13
|
+
|
|
14
|
+
<p align="center"><i>Checking a generated paint program. Nobody wrote a test.</i></p>
|
|
15
|
+
|
|
16
|
+
Point it at a page. assay opens it in a real browser, measures every control
|
|
17
|
+
it renders, works out a test plan from what it finds, drives all of it, and
|
|
18
|
+
tells you what broke.
|
|
19
|
+
|
|
20
|
+
## Highlights
|
|
21
|
+
|
|
22
|
+
- **No tests to write.** The plan comes from the page, so a program written
|
|
23
|
+
ten seconds ago can be checked ten seconds later.
|
|
24
|
+
- **No LLM. Purely mechanical.** No API key, no tokens, no rate limit, nothing
|
|
25
|
+
to bill. It gives the same answer twice.
|
|
26
|
+
- **No baseline images and no recordings.** Nothing to capture first, nothing
|
|
27
|
+
to keep in step with the design.
|
|
28
|
+
- **It says why.** Not *case 14 failed*, but *the first press did nothing and
|
|
29
|
+
the second did something, so this control is one behind*.
|
|
30
|
+
|
|
31
|
+
## Contents
|
|
32
|
+
|
|
33
|
+
- [Install](#install)
|
|
34
|
+
- [CLI](#cli)
|
|
35
|
+
- [Skills and Plugins](#skills-and-plugins)
|
|
36
|
+
- [Use](#use)
|
|
37
|
+
- [CLI](#cli-1)
|
|
38
|
+
- [Skills](#skills)
|
|
39
|
+
- [Plugins](#plugins)
|
|
40
|
+
- [Benchmarks](#benchmarks)
|
|
41
|
+
- [Generated Programs](#generated-programs)
|
|
42
|
+
- [Planted Bugs](#planted-bugs)
|
|
43
|
+
- [How It Works](#how-it-works)
|
|
44
|
+
- [Why](#why)
|
|
45
|
+
- [Limits Worth Knowing](#limits-worth-knowing)
|
|
46
|
+
- [Contributing](#contributing)
|
|
47
|
+
- [Licence](#licence)
|
|
48
|
+
|
|
49
|
+
## Install
|
|
50
|
+
|
|
51
|
+
### CLI
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
pip install assay-ui
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
The command is `assay`. The first run fetches a browser if there is not one
|
|
58
|
+
already, so there is no second command to forget.
|
|
59
|
+
|
|
60
|
+
To hack on it, clone and install it in place:
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
git clone https://github.com/awss1i/assay.git && cd assay
|
|
64
|
+
pip install -e ".[dev]"
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
### Skills and Plugins
|
|
68
|
+
|
|
69
|
+
Fifteen harnesses, one command each. All of them want the CLI above first.
|
|
70
|
+
|
|
71
|
+
**[Install it in your harness →](docs/harnesses.md)**
|
|
72
|
+
|
|
73
|
+
## Use
|
|
74
|
+
|
|
75
|
+
### CLI
|
|
76
|
+
|
|
77
|
+
```console
|
|
78
|
+
assay ./my-app # check the page in this folder
|
|
79
|
+
assay ./my-app/todo.html # check one page by name
|
|
80
|
+
assay ./my-app -e app.html # or name the page inside a folder
|
|
81
|
+
|
|
82
|
+
assay ./my-app --report out.html # write an HTML report with screenshots
|
|
83
|
+
assay ./my-app --json # print the whole run as JSON
|
|
84
|
+
assay ./my-app --one-line # print one sentence, for a script to relay
|
|
85
|
+
assay ./my-app --surface # list the controls it found, then stop
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
There is a broken drawing program in this repository. Run it yourself:
|
|
89
|
+
|
|
90
|
+
```console
|
|
91
|
+
$ assay bench/programs/dsh/gpt-oss-120b/37_draw2
|
|
92
|
+
23 case(s) planned, 23 carried out, 22 passed, 1 failed
|
|
93
|
+
|
|
94
|
+
C013 [ok] use canvas: click it, drag on it, and press the keys a program like this is driven with
|
|
95
|
+
C014 [ok] draw on canvas, then draw somewhere else on it
|
|
96
|
+
C015 [FAILED] draw on canvas twice, then press Undo twice
|
|
97
|
+
→ the first press did nothing and the second did something, from the same state, so this control is one behind
|
|
98
|
+
C016 [ok] draw on canvas twice, then press Redo twice
|
|
99
|
+
C017 [ok] draw on canvas twice, then press Clear twice
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
Drawing works. Redo works. Clear works. Undo is one press behind, and nothing
|
|
103
|
+
about the source says so.
|
|
104
|
+
|
|
105
|
+
**Where the results go.** Everything goes to stdout and **nothing is written
|
|
106
|
+
to disk unless you ask**, because a CI check that only cares about the exit
|
|
107
|
+
code should not litter. `--report FILE` writes one self-contained HTML page
|
|
108
|
+
plus a `shots/` folder of screenshots beside it. `--json` prints the whole run
|
|
109
|
+
for piping. The exit code is non-zero if anything failed.
|
|
110
|
+
|
|
111
|
+
`--report` gives you every case with the page as the browser drew it, before
|
|
112
|
+
and after:
|
|
113
|
+
|
|
114
|
+
<img src="https://raw.githubusercontent.com/awss1i/assay/main/docs/report.png" alt="One case from an assay report: the
|
|
115
|
+
act that was performed, the reason it failed, and screenshots of the page
|
|
116
|
+
before and after">
|
|
117
|
+
|
|
118
|
+
**From Python.** The same run, as an object.
|
|
119
|
+
|
|
120
|
+
```python
|
|
121
|
+
from assay import check
|
|
122
|
+
|
|
123
|
+
report = check("./my-app")
|
|
124
|
+
print(report.summary())
|
|
125
|
+
for result in report.failing:
|
|
126
|
+
print(result.case.what, "->", result.detail)
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
### Skills
|
|
130
|
+
|
|
131
|
+
*Claude Code, DeepSeek Harness, opencode, Antigravity, Codex App, Codex CLI,
|
|
132
|
+
Cursor, Devin CLI, Factory Droid, Gemini CLI, GitHub Copilot CLI, Grok Build
|
|
133
|
+
CLI, Kimi Code, Pi, Hermes Agent.*
|
|
134
|
+
|
|
135
|
+
One markdown file. Your agent runs assay when it finishes a page and prints
|
|
136
|
+
what came back:
|
|
137
|
+
|
|
138
|
+
```
|
|
139
|
+
assay: checked todo/todo.html, 8 checks, nothing flagged.
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
One line, every time, whether or not it found anything. It reports and never
|
|
143
|
+
fixes: the agent hands over what it pressed and what happened, and does not
|
|
144
|
+
edit code on the strength of it.
|
|
145
|
+
|
|
146
|
+
**[How the skill behaves →](plugins/assay/README.md#the-skill)**
|
|
147
|
+
|
|
148
|
+
### Plugins
|
|
149
|
+
|
|
150
|
+
*Claude Code, DeepSeek Harness.*
|
|
151
|
+
|
|
152
|
+
The same skill plus a hook, so the check happens at the end of every turn
|
|
153
|
+
that touched a page, whether or not the agent thought to run it.
|
|
154
|
+
|
|
155
|
+
```
|
|
156
|
+
/plugin marketplace add awss1i/assay
|
|
157
|
+
/plugin install assay@assay
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
**[How the plugin behaves →](plugins/assay/README.md#the-plugin)**
|
|
161
|
+
|
|
162
|
+
## Benchmarks
|
|
163
|
+
|
|
164
|
+
A checker nobody has checked is an opinion with a progress bar. Two sets,
|
|
165
|
+
built differently, both checked in.
|
|
166
|
+
|
|
167
|
+
### Generated Programs
|
|
168
|
+
|
|
169
|
+
225 programs written to 75 objectives by three harnesses. A person opened
|
|
170
|
+
every one and drove it before assay saw it. 20 are broken.
|
|
171
|
+
|
|
172
|
+
<!-- score2 -->
|
|
173
|
+
|
|
174
|
+
**Across 225 pages checked by hand, assay found 15 of the 20 real defects and raised 0 false alarms.** When it reports a problem it is a real one 15 times out of 15.
|
|
175
|
+
|
|
176
|
+
<!-- /score2 -->
|
|
177
|
+
|
|
178
|
+
**[The benchmark →](bench/README.md)**
|
|
179
|
+
|
|
180
|
+
### Planted Bugs
|
|
181
|
+
|
|
182
|
+
Ten working programs, and a copy of each with five bugs put in by a
|
|
183
|
+
different harness and model. Fifty defects known by construction, and the
|
|
184
|
+
harder set: pages that work and are wrong, not pages that stopped.
|
|
185
|
+
|
|
186
|
+
<!-- planted -->
|
|
187
|
+
|
|
188
|
+
**assay found 10 of the 50 planted bugs and flagged 0 of the 10 working originals.**
|
|
189
|
+
|
|
190
|
+
<!-- /planted -->
|
|
191
|
+
|
|
192
|
+
**[The planted set →](bench/planted/README.md)**
|
|
193
|
+
|
|
194
|
+
Both reproduce with `python bench/score.py` and `python bench/planted/score.py`.
|
|
195
|
+
No key, no network.
|
|
196
|
+
|
|
197
|
+
## How It Works
|
|
198
|
+
|
|
199
|
+
**It waits until the page stops arriving**, watching whether the set of
|
|
200
|
+
controls is still changing rather than pausing for a fixed time. **It measures
|
|
201
|
+
the surface** from the rendered page and never the markup, down to a grid of
|
|
202
|
+
plain divs, because a pointer cursor is the page saying *click this* and a
|
|
203
|
+
handler attached in JavaScript cannot be read any other way. **It derives a
|
|
204
|
+
plan from that** and carries all of it out in a fresh tab, measuring painted
|
|
205
|
+
pixels per canvas, their centroid and mean colour, an exact pixel hash, the
|
|
206
|
+
visible text, and a hash over every element's style.
|
|
207
|
+
|
|
208
|
+
**It is deliberately narrow about what counts as a failure.** A plan derived
|
|
209
|
+
from the page cannot know what a control is *for*, so a button only has to
|
|
210
|
+
survive being pressed. Demanding that every press change something would fail
|
|
211
|
+
a working program for having a Clear button on an empty canvas.
|
|
212
|
+
|
|
213
|
+
What it can judge without knowing the design is whether the program
|
|
214
|
+
contradicts itself. A surface that took the first stroke has to take the
|
|
215
|
+
second. A control that does nothing on its first press and something on its
|
|
216
|
+
second, from the same state, is one press behind. And if nothing on the page
|
|
217
|
+
responds to anything, the script probably never ran.
|
|
218
|
+
|
|
219
|
+
**It never says a page is broken.** It says what it pressed and what
|
|
220
|
+
happened:
|
|
221
|
+
|
|
222
|
+
```
|
|
223
|
+
C006 [FAILED] type into Quantity then press +
|
|
224
|
+
→ nothing on the page changed at all
|
|
225
|
+
```
|
|
226
|
+
|
|
227
|
+
A verdict makes you change code. A measurement makes you look first, and
|
|
228
|
+
sometimes you find the page was right and the check was aimed at the wrong
|
|
229
|
+
thing. That is the cheaper mistake to make, and it reads the same either way
|
|
230
|
+
round: a person knows where to start, and an agent gets a precise place to
|
|
231
|
+
look instead of a whole file to re-read.
|
|
232
|
+
|
|
233
|
+
**It serves the folder and never builds it.** The page is served over
|
|
234
|
+
loopback rather than opened off the disk, because a `file://` origin blocks
|
|
235
|
+
every module the page loads. What it will not do is run your build: `npm
|
|
236
|
+
install` executes whatever the dependency tree asks for, as whoever typed the
|
|
237
|
+
command, and this is a tool for checking code nobody has read. Point it at a
|
|
238
|
+
source tree and it says so:
|
|
239
|
+
|
|
240
|
+
```console
|
|
241
|
+
$ assay ./my-vite-app
|
|
242
|
+
assay: index.html loads /src/main.jsx, which a browser cannot run. This is a source tree, not a built one.
|
|
243
|
+
Build it first, in your own shell, then check the output:
|
|
244
|
+
npm install && npm run build && assay ./my-vite-app/dist
|
|
245
|
+
```
|
|
246
|
+
|
|
247
|
+
**[Every rule →](docs/how-it-works.md)**
|
|
248
|
+
|
|
249
|
+
## Why
|
|
250
|
+
|
|
251
|
+
A lot of code is written by models now, and *"does this actually run?"* is
|
|
252
|
+
mostly still answered by a person opening it and clicking around.
|
|
253
|
+
|
|
254
|
+
Every existing tool needs something you do not have for a program that was
|
|
255
|
+
generated ten seconds ago. Playwright and Cypress need tests somebody wrote.
|
|
256
|
+
Visual regression needs a golden image to compare against. Benchmarks like
|
|
257
|
+
SWE-bench use the repository's own suite.
|
|
258
|
+
|
|
259
|
+
So the thing most people reach for instead is another model: paste the code in
|
|
260
|
+
and ask whether it looks right. **That is a reader guessing about code.** assay
|
|
261
|
+
opens the page and drives it, which is the only way to find out that a button
|
|
262
|
+
does nothing.
|
|
263
|
+
|
|
264
|
+
| | needs tests written | needs a baseline | runs the program |
|
|
265
|
+
|---|---|---|---|
|
|
266
|
+
| Playwright / Cypress | yes | no | yes, the parts you wrote |
|
|
267
|
+
| Percy / Chromatic | no | **yes** | it screenshots it |
|
|
268
|
+
| ask a model to review it | no | no | **no. It reads the source** |
|
|
269
|
+
| **assay** | **no** | **no** | **yes, all of it** |
|
|
270
|
+
|
|
271
|
+
## Limits Worth Knowing
|
|
272
|
+
|
|
273
|
+
- **Browser programs.** It opens a page. A program with no page is not
|
|
274
|
+
something it can measure.
|
|
275
|
+
- **Coverage cannot judge intent.** A control that works mechanically and does
|
|
276
|
+
the wrong thing passes. Criteria are the answer, and you have to write those.
|
|
277
|
+
- **A game that ends looks like a page that died.** When a program finishes
|
|
278
|
+
and offers no way to start again, it stops responding to anything, and a
|
|
279
|
+
plan derived from the page cannot tell that apart from a page that broke.
|
|
280
|
+
- **Built output, not source trees.** It will not run your build. Point it at
|
|
281
|
+
a source tree and it says so and names the command.
|
|
282
|
+
- **Single-page programs.** It checks the page you point it at and does not
|
|
283
|
+
crawl. A multi-page site means running it per page, and client-side routing
|
|
284
|
+
is untested.
|
|
285
|
+
- **Generated pages, not the live web.** It is built for programs somebody
|
|
286
|
+
just generated, not for sites with a login, a cookie banner or live network
|
|
287
|
+
calls.
|
|
288
|
+
|
|
289
|
+
## Contributing
|
|
290
|
+
|
|
291
|
+
Issues and pull requests are welcome. Setup, tests, benchmarks and what a PR
|
|
292
|
+
needs are in **[CONTRIBUTING.md](CONTRIBUTING.md)**.
|
|
293
|
+
|
|
294
|
+
## Licence
|
|
295
|
+
|
|
296
|
+
MIT.
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "assay-ui"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Find out whether a generated web page actually works."
|
|
5
|
+
keywords = ["testing", "llm", "codegen", "playwright", "evaluation", "qa",
|
|
6
|
+
"ai-agents", "browser-testing"]
|
|
7
|
+
authors = [{ name = "awss1i" }]
|
|
8
|
+
classifiers = [
|
|
9
|
+
"Development Status :: 4 - Beta",
|
|
10
|
+
"Intended Audience :: Developers",
|
|
11
|
+
"License :: OSI Approved :: MIT License",
|
|
12
|
+
"Programming Language :: Python :: 3",
|
|
13
|
+
"Programming Language :: Python :: 3.10",
|
|
14
|
+
"Programming Language :: Python :: 3.11",
|
|
15
|
+
"Programming Language :: Python :: 3.12",
|
|
16
|
+
"Programming Language :: Python :: 3.13",
|
|
17
|
+
"Environment :: Console",
|
|
18
|
+
"Topic :: Software Development :: Testing",
|
|
19
|
+
]
|
|
20
|
+
readme = "README.md"
|
|
21
|
+
requires-python = ">=3.10"
|
|
22
|
+
license = { text = "MIT" }
|
|
23
|
+
dependencies = ["playwright>=1.40"]
|
|
24
|
+
|
|
25
|
+
[project.urls]
|
|
26
|
+
Homepage = "https://github.com/awss1i/assay"
|
|
27
|
+
Source = "https://github.com/awss1i/assay"
|
|
28
|
+
Issues = "https://github.com/awss1i/assay/issues"
|
|
29
|
+
Changelog = "https://github.com/awss1i/assay/releases"
|
|
30
|
+
|
|
31
|
+
[project.scripts]
|
|
32
|
+
assay = "assay.cli:main"
|
|
33
|
+
|
|
34
|
+
[project.optional-dependencies]
|
|
35
|
+
dev = ["pytest>=7"]
|
|
36
|
+
|
|
37
|
+
[build-system]
|
|
38
|
+
requires = ["setuptools>=68"]
|
|
39
|
+
build-backend = "setuptools.build_meta"
|
|
40
|
+
|
|
41
|
+
[tool.setuptools.packages.find]
|
|
42
|
+
where = ["src"]
|
assay_ui-0.1.0/setup.cfg
ADDED