servo-embedder-tests 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/PKG-INFO +42 -18
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/README.md +40 -17
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/pyproject.toml +2 -1
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/pyproject.toml.orig +2 -1
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/cli.py +11 -4
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/diffing.py +151 -28
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/page_scenarios.py +18 -6
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/plugin.py +18 -2
- servo_embedder_tests-0.3.0/src/servo_embedder_tests/report.py +932 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/scenarios/page_scenarios/test_pages.py +33 -26
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/skills/compare-renderings/SKILL.md +17 -5
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/skills/compare-renderings/describe_regions.py +4 -2
- servo_embedder_tests-0.2.0/src/servo_embedder_tests/report.py +0 -458
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/LICENSE-APACHE +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/LICENSE-MIT +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/__init__.py +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/__main__.py +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/describe.py +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/engines.py +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/errors.py +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/expectations.py +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/hookspecs.py +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/oracles.py +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/basic.html +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/history-a.html +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/history-b.html +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/ime.html +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/report.js +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/10_coordinate_transforms.html +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/11_symbols_and_use.html +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/12_basic_text.html +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/13_text_paths.html +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/14_clipping_paths.html +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/15_masking.html +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/16_filter_effects_basic.html +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/1_basic_shapes.html +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/2_path_commands.html +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/3_path_curves.html +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/4_stroke_properties.html +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/5_linear_gradients.html +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/6_radial_gradients.html +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/7_patterns.html +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/8_markers.html +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/9_viewbox_scaling.html +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/LICENSE +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/README.md +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/security/layer1_xml_validation.html +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/security/layer2_css_injection.html +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/security/layer2_import_fetch.html +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/security/layer3_fetch_allowlist.html +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/security/layer4_expansion_limits.html +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/security/layer4_use_self_cycle.html +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/security/layer5_mask_recursion.html +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/security/layer5_render_safety.html +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/webview_apps/scroll.html +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages.py +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/platform/__init__.py +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/platform/desktop.py +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/platform/ohos.py +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/proxy.py +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/scenarios/__init__.py +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/scenarios/ime/__init__.py +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/scenarios/ime/test_text_input.py +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/scenarios/navigation/__init__.py +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/scenarios/navigation/test_local_pages.py +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/scenarios/page_scenarios/__init__.py +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/scenarios/sites/__init__.py +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/scenarios/sites/mossel.py +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/scenarios/sites/test_mossel.py +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/scenarios/sites/test_servo_org.py +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/scenarios/svg/__init__.py +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/scenarios/svg/test_security.py +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/scenarios/webview_apps/__init__.py +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/scenarios/webview_apps/apps/arkweb-test.toml +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/scenarios/webview_apps/flows.py +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/scenarios/webview_apps/test_apps.py +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/screenshots.py +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/servoshell.py +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/svg_cards.py +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/touch.py +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/ui.py +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/webdriver.py +0 -0
- {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/webview_app.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: servo-embedder-tests
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Embedder tests for Servo: what servoshell and its users do beyond WPT, on OpenHarmony, HarmonyOS and desktop
|
|
5
5
|
License-Expression: Apache-2.0 OR MIT
|
|
6
6
|
License-File: LICENSE-APACHE
|
|
@@ -16,6 +16,7 @@ Requires-Dist: tomlkit>=0.13
|
|
|
16
16
|
Requires-Dist: urllib3>=2
|
|
17
17
|
Requires-Python: >=3.11
|
|
18
18
|
Project-URL: Repository, https://github.com/openharmony-rs/servo-embedder-tests
|
|
19
|
+
Project-URL: Changelog, https://github.com/openharmony-rs/servo-embedder-tests/blob/main/CHANGELOG.md
|
|
19
20
|
Description-Content-Type: text/markdown
|
|
20
21
|
|
|
21
22
|
# servo-embedder-tests
|
|
@@ -160,8 +161,11 @@ Each scenario has three tests:
|
|
|
160
161
|
- `test_matches_baseline`: each capture matches the engine's baseline;
|
|
161
162
|
- `test_matches_reference`: each capture matches the reference engine's.
|
|
162
163
|
|
|
163
|
-
|
|
164
|
-
|
|
164
|
+
`test_matches_baseline` is a regression check: it finds what changed since the baselines were stored, e.g. with an
|
|
165
|
+
engine update. `test_matches_reference` is an engine comparison: it finds the gaps between two engines. Together they
|
|
166
|
+
make a three-way comparison: where a page changed since its baseline, the reference engine's stored rendering tells
|
|
167
|
+
whether each change brought it closer to the reference, likely an improvement, or took it further away, likely a
|
|
168
|
+
regression. `--engine-version NAME` and
|
|
165
169
|
`--reference-engine-version NAME` name the builds, e.g. a commit or a release. The name is kept with their captures
|
|
166
170
|
and baselines, so that `diff.json` and the report say which builds were compared. Stored references keep the name
|
|
167
171
|
they were rendered with: `--reference-engine-version` names those rendered in the run, see `--update-references`.
|
|
@@ -170,7 +174,8 @@ The page is rendered once per engine for all three. Known differences between en
|
|
|
170
174
|
`test_matches_reference`, so that they don't hide regressions. The captures of each engine are kept in
|
|
171
175
|
`--baselines` under `<device>/<engine>/<scenario>/`; the device is its model, or `--reference-key`. The reference
|
|
172
176
|
engine's captures are expectations like the baselines: with them stored, a run doesn't need the reference engine. It
|
|
173
|
-
renders the missing ones if it has the engine, and `--update-references` renders all anew.
|
|
177
|
+
renders the missing ones if it has the engine, and `--update-references` renders all anew. The engine comparisons
|
|
178
|
+
run before the regression checks, so that references rendered in the run judge the changes too.
|
|
174
179
|
|
|
175
180
|
A capture is a screenshot of the web view, taken by the OS the same way for every engine. It's taken after:
|
|
176
181
|
|
|
@@ -191,16 +196,22 @@ Each comparison test writes `diff.json` with, for each capture:
|
|
|
191
196
|
|
|
192
197
|
- the differing regions in the actual capture's viewport, in CSS px;
|
|
193
198
|
- the elements whose boxes best match each region, with their text, computed fonts and paint properties;
|
|
194
|
-
- for the
|
|
195
|
-
the same ink (spacing), less or more ink (stroke weight, or content missing), shapes that
|
|
196
|
-
aligned, or a colour drawn in another colour (such as a colour emoji drawn black). It starts with
|
|
197
|
-
region that did not load, if any;
|
|
199
|
+
- for the regions (of baselines all, else the largest), a `reading` of pixel measurements: moved by how much, only
|
|
200
|
+
anti-aliasing, wider with the same ink (spacing), less or more ink (stroke weight, or content missing), shapes that
|
|
201
|
+
differ even when aligned, or a colour drawn in another colour (such as a colour emoji drawn black). It starts with
|
|
202
|
+
the images in the region that did not load, if any;
|
|
198
203
|
- `dom_changes`, the elements whose text, box, style or image load state changed;
|
|
199
|
-
- for baselines, whether
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
+
- for baselines, whether a region's change is `minor`: anti-aliasing or sub-pixel positions only, as when text is
|
|
205
|
+
rasterized with another gamma. It vanishes when the captures are compared in CSS px with a tolerance above what
|
|
206
|
+
engines differ by, lies on edges both captures have, didn't move by a device pixel or more and changed no colour;
|
|
207
|
+
- for baselines, with the reference engine's captures stored, whether each region moved closer to the reference
|
|
208
|
+
engine's rendering or further from it (`toward_reference`: closer, further or same), a hint whether a change is an
|
|
209
|
+
improvement. Each rendering's part of the region is looked for in the other's within 24 CSS px, so that content the
|
|
210
|
+
engines lay out a few pixels apart counts as found, and a blank as far from content.
|
|
211
|
+
|
|
212
|
+
It also writes `<capture>-compare.png` (expected, actual and the differing pixels), for baselines the reference
|
|
213
|
+
engine's view of the compared part as `<capture>-reference.png`, and a crop of each region in `regions/`, expected
|
|
214
|
+
above actual above the differing pixels. Those crops are what to send to an image-capable model
|
|
204
215
|
for what the readings leave open: the `compare-renderings` skill describes how an agent triages a run, and its
|
|
205
216
|
script asks any OpenAI-compatible endpoint about the regions.
|
|
206
217
|
|
|
@@ -208,11 +219,24 @@ A comparison test passes or fails as soon as the tiles are compared. Describing
|
|
|
208
219
|
readings, DOM changes and images, takes seconds per capture, so it runs in up to four background processes while the
|
|
209
220
|
run goes on to the next test; `diff.json` and the images are complete when the run ends.
|
|
210
221
|
|
|
211
|
-
For people, a run writes `report.html` to the artifacts folder
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
222
|
+
For people, a run writes `report.html` to the artifacts folder. It starts with each page's verdict: unchanged since
|
|
223
|
+
its baseline, or, where it changed, a likely regression or improvement by whether the changes took it away from the
|
|
224
|
+
reference engine's rendering or toward it, mixed, or unclear. Minor changes are shown, with dashed boxes and folded
|
|
225
|
+
away, but left out of the verdict; a page that changed only in them has minor changes. Pages get a "Check" flag, with
|
|
226
|
+
links, for what their verdict leaves out: captures that weren't compared, such as those of a page that got shorter,
|
|
227
|
+
and regions that matched the reference engine's rendering before but no longer do, however small. For each changed capture it shows the old rendering,
|
|
228
|
+
the new one (hover it to see the old one in its place), the reference engine's and the changed pixels, then each
|
|
229
|
+
region numbered with its verdict, the renderings of it side by side and enlarged, its readings and the DOM changes.
|
|
230
|
+
The engine comparison, the gaps that remain to the reference engine, is folded away below; without baselines, it is
|
|
231
|
+
all the report shows. Where an agent triaged the run, the report also shows the image model's answers and the agent's
|
|
232
|
+
conclusion (`triage.json` in the test's folder, see the skill), so that you can check them against the pictures.
|
|
233
|
+
`servo-embedder-tests report <artifacts>` writes it again.
|
|
234
|
+
|
|
235
|
+
The report shows the images in the artifacts folder, so it needs the folder. To share one file instead, pass
|
|
236
|
+
`--standalone-report` to `run`, or `--standalone` to `report`: `report-standalone.html` has its images inlined, the
|
|
237
|
+
full captures at one pixel per CSS pixel and the region crops sharper, in WebP, each image once. For 8 GenUI pages it
|
|
238
|
+
is 13 MB, against 106 MB of PNGs, and takes about a second per page. It can be made later from the artifacts folder,
|
|
239
|
+
and `report` writes it again along with `report.html` if it's there.
|
|
216
240
|
|
|
217
241
|
`page-scenarios.toml` in the folder sets defaults for all scenarios, `scenario.toml` in a scenario's folder
|
|
218
242
|
overrides them. Changes to the page are opt-in, under `[page]`:
|
|
@@ -140,8 +140,11 @@ Each scenario has three tests:
|
|
|
140
140
|
- `test_matches_baseline`: each capture matches the engine's baseline;
|
|
141
141
|
- `test_matches_reference`: each capture matches the reference engine's.
|
|
142
142
|
|
|
143
|
-
|
|
144
|
-
|
|
143
|
+
`test_matches_baseline` is a regression check: it finds what changed since the baselines were stored, e.g. with an
|
|
144
|
+
engine update. `test_matches_reference` is an engine comparison: it finds the gaps between two engines. Together they
|
|
145
|
+
make a three-way comparison: where a page changed since its baseline, the reference engine's stored rendering tells
|
|
146
|
+
whether each change brought it closer to the reference, likely an improvement, or took it further away, likely a
|
|
147
|
+
regression. `--engine-version NAME` and
|
|
145
148
|
`--reference-engine-version NAME` name the builds, e.g. a commit or a release. The name is kept with their captures
|
|
146
149
|
and baselines, so that `diff.json` and the report say which builds were compared. Stored references keep the name
|
|
147
150
|
they were rendered with: `--reference-engine-version` names those rendered in the run, see `--update-references`.
|
|
@@ -150,7 +153,8 @@ The page is rendered once per engine for all three. Known differences between en
|
|
|
150
153
|
`test_matches_reference`, so that they don't hide regressions. The captures of each engine are kept in
|
|
151
154
|
`--baselines` under `<device>/<engine>/<scenario>/`; the device is its model, or `--reference-key`. The reference
|
|
152
155
|
engine's captures are expectations like the baselines: with them stored, a run doesn't need the reference engine. It
|
|
153
|
-
renders the missing ones if it has the engine, and `--update-references` renders all anew.
|
|
156
|
+
renders the missing ones if it has the engine, and `--update-references` renders all anew. The engine comparisons
|
|
157
|
+
run before the regression checks, so that references rendered in the run judge the changes too.
|
|
154
158
|
|
|
155
159
|
A capture is a screenshot of the web view, taken by the OS the same way for every engine. It's taken after:
|
|
156
160
|
|
|
@@ -171,16 +175,22 @@ Each comparison test writes `diff.json` with, for each capture:
|
|
|
171
175
|
|
|
172
176
|
- the differing regions in the actual capture's viewport, in CSS px;
|
|
173
177
|
- the elements whose boxes best match each region, with their text, computed fonts and paint properties;
|
|
174
|
-
- for the
|
|
175
|
-
the same ink (spacing), less or more ink (stroke weight, or content missing), shapes that
|
|
176
|
-
aligned, or a colour drawn in another colour (such as a colour emoji drawn black). It starts with
|
|
177
|
-
region that did not load, if any;
|
|
178
|
+
- for the regions (of baselines all, else the largest), a `reading` of pixel measurements: moved by how much, only
|
|
179
|
+
anti-aliasing, wider with the same ink (spacing), less or more ink (stroke weight, or content missing), shapes that
|
|
180
|
+
differ even when aligned, or a colour drawn in another colour (such as a colour emoji drawn black). It starts with
|
|
181
|
+
the images in the region that did not load, if any;
|
|
178
182
|
- `dom_changes`, the elements whose text, box, style or image load state changed;
|
|
179
|
-
- for baselines, whether
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
183
|
+
- for baselines, whether a region's change is `minor`: anti-aliasing or sub-pixel positions only, as when text is
|
|
184
|
+
rasterized with another gamma. It vanishes when the captures are compared in CSS px with a tolerance above what
|
|
185
|
+
engines differ by, lies on edges both captures have, didn't move by a device pixel or more and changed no colour;
|
|
186
|
+
- for baselines, with the reference engine's captures stored, whether each region moved closer to the reference
|
|
187
|
+
engine's rendering or further from it (`toward_reference`: closer, further or same), a hint whether a change is an
|
|
188
|
+
improvement. Each rendering's part of the region is looked for in the other's within 24 CSS px, so that content the
|
|
189
|
+
engines lay out a few pixels apart counts as found, and a blank as far from content.
|
|
190
|
+
|
|
191
|
+
It also writes `<capture>-compare.png` (expected, actual and the differing pixels), for baselines the reference
|
|
192
|
+
engine's view of the compared part as `<capture>-reference.png`, and a crop of each region in `regions/`, expected
|
|
193
|
+
above actual above the differing pixels. Those crops are what to send to an image-capable model
|
|
184
194
|
for what the readings leave open: the `compare-renderings` skill describes how an agent triages a run, and its
|
|
185
195
|
script asks any OpenAI-compatible endpoint about the regions.
|
|
186
196
|
|
|
@@ -188,11 +198,24 @@ A comparison test passes or fails as soon as the tiles are compared. Describing
|
|
|
188
198
|
readings, DOM changes and images, takes seconds per capture, so it runs in up to four background processes while the
|
|
189
199
|
run goes on to the next test; `diff.json` and the images are complete when the run ends.
|
|
190
200
|
|
|
191
|
-
For people, a run writes `report.html` to the artifacts folder
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
201
|
+
For people, a run writes `report.html` to the artifacts folder. It starts with each page's verdict: unchanged since
|
|
202
|
+
its baseline, or, where it changed, a likely regression or improvement by whether the changes took it away from the
|
|
203
|
+
reference engine's rendering or toward it, mixed, or unclear. Minor changes are shown, with dashed boxes and folded
|
|
204
|
+
away, but left out of the verdict; a page that changed only in them has minor changes. Pages get a "Check" flag, with
|
|
205
|
+
links, for what their verdict leaves out: captures that weren't compared, such as those of a page that got shorter,
|
|
206
|
+
and regions that matched the reference engine's rendering before but no longer do, however small. For each changed capture it shows the old rendering,
|
|
207
|
+
the new one (hover it to see the old one in its place), the reference engine's and the changed pixels, then each
|
|
208
|
+
region numbered with its verdict, the renderings of it side by side and enlarged, its readings and the DOM changes.
|
|
209
|
+
The engine comparison, the gaps that remain to the reference engine, is folded away below; without baselines, it is
|
|
210
|
+
all the report shows. Where an agent triaged the run, the report also shows the image model's answers and the agent's
|
|
211
|
+
conclusion (`triage.json` in the test's folder, see the skill), so that you can check them against the pictures.
|
|
212
|
+
`servo-embedder-tests report <artifacts>` writes it again.
|
|
213
|
+
|
|
214
|
+
The report shows the images in the artifacts folder, so it needs the folder. To share one file instead, pass
|
|
215
|
+
`--standalone-report` to `run`, or `--standalone` to `report`: `report-standalone.html` has its images inlined, the
|
|
216
|
+
full captures at one pixel per CSS pixel and the region crops sharper, in WebP, each image once. For 8 GenUI pages it
|
|
217
|
+
is 13 MB, against 106 MB of PNGs, and takes about a second per page. It can be made later from the artifacts folder,
|
|
218
|
+
and `report` writes it again along with `report.html` if it's there.
|
|
196
219
|
|
|
197
220
|
`page-scenarios.toml` in the folder sets defaults for all scenarios, `scenario.toml` in a scenario's folder
|
|
198
221
|
overrides them. Changes to the page are opt-in, under `[page]`:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "servo-embedder-tests"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.3.0"
|
|
4
4
|
description = "Embedder tests for Servo: what servoshell and its users do beyond WPT, on OpenHarmony, HarmonyOS and desktop"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.11"
|
|
@@ -23,6 +23,7 @@ dependencies = [
|
|
|
23
23
|
|
|
24
24
|
[project.urls]
|
|
25
25
|
Repository = "https://github.com/openharmony-rs/servo-embedder-tests"
|
|
26
|
+
Changelog = "https://github.com/openharmony-rs/servo-embedder-tests/blob/main/CHANGELOG.md"
|
|
26
27
|
|
|
27
28
|
[project.scripts]
|
|
28
29
|
servo-embedder-tests = "servo_embedder_tests.cli:main"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "servo-embedder-tests"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.3.0"
|
|
4
4
|
description = "Embedder tests for Servo: what servoshell and its users do beyond WPT, on OpenHarmony, HarmonyOS and desktop"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.11"
|
|
@@ -20,6 +20,7 @@ dependencies = [
|
|
|
20
20
|
|
|
21
21
|
[project.urls]
|
|
22
22
|
Repository = "https://github.com/openharmony-rs/servo-embedder-tests"
|
|
23
|
+
Changelog = "https://github.com/openharmony-rs/servo-embedder-tests/blob/main/CHANGELOG.md"
|
|
23
24
|
|
|
24
25
|
[project.scripts]
|
|
25
26
|
servo-embedder-tests = "servo_embedder_tests.cli:main"
|
|
@@ -17,7 +17,7 @@ from . import __version__, svg_cards
|
|
|
17
17
|
from .expectations import Expectations, RunReport, update
|
|
18
18
|
from .page_scenarios import Capture, CaptureStore, Rendering
|
|
19
19
|
from .plugin import BUNDLED_SCENARIOS, SKILL
|
|
20
|
-
from .report import write_report
|
|
20
|
+
from .report import STANDALONE, write_report
|
|
21
21
|
|
|
22
22
|
|
|
23
23
|
def _bundled(arg: str) -> str | None:
|
|
@@ -221,12 +221,19 @@ def _report(argv: Sequence[str]) -> int:
|
|
|
221
221
|
description="Write the page scenario report of a run again, e.g. after an agent added its conclusions.",
|
|
222
222
|
)
|
|
223
223
|
parser.add_argument("artifacts", type=Path, help="the run's --artifacts folder")
|
|
224
|
+
parser.add_argument(
|
|
225
|
+
"--standalone",
|
|
226
|
+
action="store_true",
|
|
227
|
+
help=f"also write {STANDALONE}, the report with its images inlined, one file to share (written anyway if "
|
|
228
|
+
"it's there)",
|
|
229
|
+
)
|
|
224
230
|
options = parser.parse_args(argv)
|
|
225
|
-
path = write_report(options.artifacts)
|
|
231
|
+
path = write_report(options.artifacts, standalone=options.standalone)
|
|
226
232
|
if path is None:
|
|
227
233
|
print(f"error: no page scenarios ran in {options.artifacts}")
|
|
228
234
|
return 1
|
|
229
|
-
|
|
235
|
+
standalone = path.with_name(STANDALONE)
|
|
236
|
+
print(f"wrote {path}" + (f" and {standalone}" if standalone.is_file() else ""))
|
|
230
237
|
return 0
|
|
231
238
|
|
|
232
239
|
|
|
@@ -256,7 +263,7 @@ def main(argv: Sequence[str] | None = None) -> int:
|
|
|
256
263
|
"update-expectations": "rewrite expectations from results.json files",
|
|
257
264
|
"accept": "store page scenario captures of a run as baselines: accept ARTIFACTS --baselines DIR",
|
|
258
265
|
"skill": "print where the agent skill for triaging page scenarios is, or copy it: skill [FOLDER]",
|
|
259
|
-
"report": "write a run's page scenario report (report.html) again: report ARTIFACTS",
|
|
266
|
+
"report": "write a run's page scenario report (report.html) again: report ARTIFACTS [--standalone]",
|
|
260
267
|
"svg-cards": "write the cards of the SVG test pages as page scenarios: svg-cards FOLDER",
|
|
261
268
|
"record-dump": "record a mitmproxy dump of the network scenarios: record-dump PATH [run options]",
|
|
262
269
|
}
|
{servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/diffing.py
RENAMED
|
@@ -49,6 +49,14 @@ EDGE_CONTRAST = 24
|
|
|
49
49
|
COLOUR_CHROMA = 48
|
|
50
50
|
COLOUR_AREA = 40
|
|
51
51
|
COLOUR_REACH = 2
|
|
52
|
+
# Whether a change moved toward another engine's rendering: how far, in CSS px, a region's content may lie from where
|
|
53
|
+
# the other engine has it and still count as found there (engines lay out text a few pixels apart), and the CSS px of
|
|
54
|
+
# context around the region, and the share by which the distance to it must change to count (10% made 11 of 163
|
|
55
|
+
# regions of pages with features turned off come out closer, 15% 7; 20% lost real changes). More context dilutes
|
|
56
|
+
# the change: with 8, 12 or 16 px, 88, 80 or 69 of those regions came out further, against 95 with 4.
|
|
57
|
+
REFERENCE_SEARCH = 24
|
|
58
|
+
REFERENCE_MARGIN = 4
|
|
59
|
+
REFERENCE_CHANGE = 0.15
|
|
52
60
|
HUES = (
|
|
53
61
|
"red",
|
|
54
62
|
"orange",
|
|
@@ -91,6 +99,16 @@ class Preset:
|
|
|
91
99
|
BASELINE = Preset(tolerance=12, tile=16, tile_share=0.05, css=False)
|
|
92
100
|
# Different engines: anti-aliasing and rasterization differ everywhere, so only clear differences count.
|
|
93
101
|
REFERENCE = Preset(tolerance=48, tile=16, tile_share=0.05, css=True)
|
|
102
|
+
# A change is minor, anti-aliasing or sub-pixel positions, when comparing the two captures in CSS px with
|
|
103
|
+
# MINOR_TOLERANCE finds nothing there, most of its differing pixels lie on edges both captures have, it didn't move
|
|
104
|
+
# by MINOR_MOVE device px or more and no colour changed. On GenUI pages on a DAYU200 (2026-11-18), that made minor
|
|
105
|
+
# 99% of the changed pixels of another text gamma or of whole captures moved by a quarter device pixel, 57% of edges
|
|
106
|
+
# blurred by 0.6 device px; none of 64 injected changes (hidden, resized, refonted, recoloured, spaced, shifted by a
|
|
107
|
+
# device pixel or more, underlined) and 18 of 178 regions, under 1% of the changed pixels, of pages with features
|
|
108
|
+
# turned off. A tolerance of 48 made as few changes minor that shouldn't be, but only 41% and 28% of the moved and
|
|
109
|
+
# blurred captures.
|
|
110
|
+
MINOR_TOLERANCE = 64
|
|
111
|
+
MINOR_MOVE = 0.75
|
|
94
112
|
|
|
95
113
|
|
|
96
114
|
@dataclass
|
|
@@ -103,7 +121,12 @@ class Region:
|
|
|
103
121
|
mean_difference: float
|
|
104
122
|
max_difference: int
|
|
105
123
|
elements: list[dict[str, object]] = field(default_factory=list)
|
|
124
|
+
# Whether the actual rendering is "closer" to another engine's than the expected one, "further" from it or the
|
|
125
|
+
# "same"; None where they can't be compared. `reference` says it in words, with why where they can't.
|
|
106
126
|
reference: str | None = None
|
|
127
|
+
toward_reference: str | None = None
|
|
128
|
+
# Whether the change is minor: anti-aliasing or sub-pixel positions (see `mark_minor`).
|
|
129
|
+
minor: bool = False
|
|
107
130
|
crop: str | None = None
|
|
108
131
|
# Pixel measurements (see `measure`) and what they mean.
|
|
109
132
|
measured: dict[str, object] | None = None
|
|
@@ -370,17 +393,41 @@ def stacked(images: Sequence[tuple[str, Image.Image]], dest: Path) -> Path:
|
|
|
370
393
|
return dest
|
|
371
394
|
|
|
372
395
|
|
|
396
|
+
def _aligned(shot: Shot, top: float, size: tuple[int, int], pixels: tuple[int, int]) -> Image.Image:
|
|
397
|
+
"""The `size` CSS pixels of the document from `top` down as `shot` shows them, scaled to `pixels`, grey where it
|
|
398
|
+
doesn't show them."""
|
|
399
|
+
with Image.open(shot.path) as image:
|
|
400
|
+
rgb = image.convert("RGB")
|
|
401
|
+
scale = rgb.width / shot.size[0]
|
|
402
|
+
canvas = Image.new("RGB", (round(size[0] * scale), round(size[1] * scale)), (128, 128, 128))
|
|
403
|
+
canvas.paste(rgb, (0, round((shot.scroll - top) * scale)))
|
|
404
|
+
return canvas if canvas.size == pixels else canvas.resize(pixels, Image.Resampling.BOX)
|
|
405
|
+
|
|
406
|
+
|
|
373
407
|
def write_review_images(
|
|
374
|
-
expected: Shot,
|
|
408
|
+
expected: Shot,
|
|
409
|
+
actual: Shot,
|
|
410
|
+
diff: Diff,
|
|
411
|
+
overlay: Path,
|
|
412
|
+
directory: Path,
|
|
413
|
+
name: str,
|
|
414
|
+
labels: tuple[str, str],
|
|
415
|
+
reference: Shot | None = None,
|
|
375
416
|
) -> dict[str, str]:
|
|
376
417
|
"""Writes the compared part of each capture, at its own resolution, to `<name>-expected.png` and
|
|
377
418
|
`<name>-actual.png`, `<name>-compare.png` (expected | actual | differing) and a crop of each of the largest regions
|
|
378
419
|
to `regions/<name>-<n>.png` (expected above actual above the differing pixels), whose paths go into the regions.
|
|
379
|
-
|
|
420
|
+
With a `reference`, another engine's capture of the same width in CSS px, its view of the compared part goes to
|
|
421
|
+
`<name>-reference.png`, at the actual one's resolution. Returns the names of the images by what they show, the
|
|
422
|
+
overlay of differing pixels included."""
|
|
380
423
|
common = diff.size
|
|
381
424
|
a, b = expected.load(diff.top, common), actual.load(diff.top, common)
|
|
382
425
|
a.save(directory / f"{name}-expected.png")
|
|
383
426
|
b.save(directory / f"{name}-actual.png")
|
|
427
|
+
written = {}
|
|
428
|
+
if reference is not None and reference.size[0] == actual.size[0]:
|
|
429
|
+
_aligned(reference, diff.top, common, b.size).save(directory / f"{name}-reference.png")
|
|
430
|
+
written["reference"] = f"{name}-reference.png"
|
|
384
431
|
with Image.open(overlay) as image:
|
|
385
432
|
marked = image.convert("RGB")
|
|
386
433
|
a, b = (image.resize(marked.size, Image.Resampling.BOX) if image.size != marked.size else image for image in (a, b))
|
|
@@ -409,6 +456,7 @@ def write_review_images(
|
|
|
409
456
|
"expected": f"{name}-expected.png",
|
|
410
457
|
"actual": f"{name}-actual.png",
|
|
411
458
|
"differing": overlay.name,
|
|
459
|
+
**written,
|
|
412
460
|
}
|
|
413
461
|
|
|
414
462
|
|
|
@@ -661,6 +709,19 @@ def measure(expected: Shot, actual: Shot, diff: Diff, region: Region) -> Measure
|
|
|
661
709
|
)
|
|
662
710
|
|
|
663
711
|
|
|
712
|
+
def mark_minor(expected: Shot, actual: Shot, diff: Diff, measurements: Sequence[Measurement]) -> None:
|
|
713
|
+
"""Marks the regions of `diff`, a comparison of `expected` and `actual`, that are minor (see MINOR_TOLERANCE), from
|
|
714
|
+
the `measurements` of each."""
|
|
715
|
+
coarse = compare(expected, actual, REFERENCE.updated({"tolerance": MINOR_TOLERANCE}))
|
|
716
|
+
for region, measurement in zip(diff.regions, measurements, strict=True):
|
|
717
|
+
left, top, right, bottom = region.box
|
|
718
|
+
if any(left < r.box[2] and right > r.box[0] and top < r.box[3] and bottom > r.box[1] for r in coarse.regions):
|
|
719
|
+
continue
|
|
720
|
+
moved = measurement.moved and max(map(abs, measurement.offset)) * diff.scale >= MINOR_MOVE
|
|
721
|
+
edges = measurement.on_edges is None or measurement.on_edges >= 0.5
|
|
722
|
+
region.minor = edges and not moved and not measurement.colours
|
|
723
|
+
|
|
724
|
+
|
|
664
725
|
@dataclass(frozen=True)
|
|
665
726
|
class Measurement:
|
|
666
727
|
"""How a region differs: `offset`, in CSS px, best aligns the actual rendering onto the expected one, with mean
|
|
@@ -679,6 +740,12 @@ class Measurement:
|
|
|
679
740
|
on_edges: float | None = None
|
|
680
741
|
colours: tuple[tuple[str, str, float], ...] = ()
|
|
681
742
|
|
|
743
|
+
@property
|
|
744
|
+
def moved(self) -> bool:
|
|
745
|
+
"""Whether the actual rendering is nearly the expected one moved by `offset`."""
|
|
746
|
+
aligned, unshifted = self.difference_aligned, self.difference_unshifted
|
|
747
|
+
return bool(self.offset[0] or self.offset[1]) and aligned <= 8 and aligned <= 0.5 * unshifted
|
|
748
|
+
|
|
682
749
|
def reading(self, text: bool = True) -> str:
|
|
683
750
|
"""What the measurements mean, in words, for an agent that can't see the crops. Without `text` in the region,
|
|
684
751
|
the width of the ink, e.g. of a textured background, says nothing about spacing or glyphs, and is left out."""
|
|
@@ -686,7 +753,7 @@ class Measurement:
|
|
|
686
753
|
aligned, unshifted, ink = self.difference_aligned, self.difference_unshifted, self.ink
|
|
687
754
|
before, after = self.ink_width
|
|
688
755
|
parts = []
|
|
689
|
-
moved =
|
|
756
|
+
moved = self.moved
|
|
690
757
|
if moved:
|
|
691
758
|
parts.append(f"moved by ({dx:+g}, {dy:+g}) CSS px, otherwise nearly the same")
|
|
692
759
|
elif unshifted < 3 and (self.on_edges is None or self.on_edges >= 0.5):
|
|
@@ -725,42 +792,98 @@ class Measurement:
|
|
|
725
792
|
def reference_hint(previous: Shot, current: Shot, reference: Shot, regions: Sequence[Region]) -> None:
|
|
726
793
|
"""Notes in each region whether `current` is closer to `reference`, another engine's rendering, than `previous`.
|
|
727
794
|
|
|
728
|
-
Regions are in `current`'s viewport.
|
|
729
|
-
|
|
795
|
+
Regions are in `current`'s viewport. How far two renderings are apart in a region: each one's part there where it
|
|
796
|
+
best matches the other within REFERENCE_SEARCH CSS px, in colour, the worse of the two. Engines lay out text a few
|
|
797
|
+
pixels apart: compared in place, text that moved a little is further from the reference's text than a blank is,
|
|
798
|
+
and a blank is found in any blank part of the reference, but not the reference's text in it. Beyond the edges of a
|
|
799
|
+
capture, and under masks, the search sees the part's median colour, so that regions at the edges can be aligned
|
|
800
|
+
too. Distances that differ by less than REFERENCE_CHANGE count as the same. On GenUI pages on a DAYU200
|
|
801
|
+
(2026-11-18), with features turned off, 98% of the changed pixels came out further from ArkWeb and 1% closer
|
|
802
|
+
(compared in place: 60% and 14%); of 23 hidden, resized or refonted elements, 14 further and 3 closer (in place: 7
|
|
803
|
+
and 7). The engines rasterize differently, so this is a hint for triage: a change toward the reference is more
|
|
804
|
+
likely an improvement, one away from it more likely a regression.
|
|
730
805
|
"""
|
|
731
|
-
|
|
732
|
-
|
|
806
|
+
if reference.size[0] != current.size[0]:
|
|
807
|
+
for region in regions:
|
|
808
|
+
region.reference = (
|
|
809
|
+
f"not comparable: the reference's viewport is {reference.size[0]} CSS px wide, this one's "
|
|
810
|
+
f"{current.size[0]}"
|
|
811
|
+
)
|
|
812
|
+
return
|
|
813
|
+
before, after, ref = (_Document(shot) for shot in (previous, current, reference))
|
|
733
814
|
for region in regions:
|
|
734
815
|
left, top, right, bottom = region.box
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
round(top + current.scroll - shot.scroll),
|
|
739
|
-
round(right),
|
|
740
|
-
round(bottom + current.scroll - shot.scroll),
|
|
741
|
-
)
|
|
742
|
-
for shot in shots
|
|
743
|
-
]
|
|
744
|
-
if any(
|
|
745
|
-
box[0] < 0 or box[1] < 0 or box[2] > shot.size[0] or box[3] > shot.size[1]
|
|
746
|
-
for box, shot in zip(boxes, shots, strict=True)
|
|
747
|
-
):
|
|
816
|
+
box = (left, top + current.scroll, right, bottom + current.scroll)
|
|
817
|
+
old, new = _apart(before, ref, box), _apart(after, ref, box)
|
|
818
|
+
if old is None or new is None:
|
|
748
819
|
region.reference = "not all in the reference's or the earlier capture's viewport"
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
old, new = _mean(_largest_channel(before, ref)), _mean(_largest_channel(after, ref))
|
|
752
|
-
if abs(new - old) < max(1.0, 0.1 * max(old, new)):
|
|
820
|
+
elif abs(new - old) < max(1.0, REFERENCE_CHANGE * max(old, new)):
|
|
821
|
+
region.toward_reference = "same"
|
|
753
822
|
region.reference = f"as far from the reference as before ({old:.1f} -> {new:.1f})"
|
|
754
823
|
elif new < old:
|
|
824
|
+
region.toward_reference = "closer"
|
|
755
825
|
region.reference = f"closer to the reference ({old:.1f} -> {new:.1f})"
|
|
756
826
|
else:
|
|
827
|
+
region.toward_reference = "further"
|
|
757
828
|
region.reference = f"further from the reference ({old:.1f} -> {new:.1f})"
|
|
758
829
|
|
|
759
830
|
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
831
|
+
class _Document:
|
|
832
|
+
"""A capture as an RGB array at one pixel per CSS pixel, placed in the document: NaN under its masks. `shown` is
|
|
833
|
+
the part of the document, from the top down, that it shows."""
|
|
834
|
+
|
|
835
|
+
def __init__(self, shot: Shot) -> None:
|
|
836
|
+
import numpy as np
|
|
837
|
+
|
|
838
|
+
self.top = shot.scroll
|
|
839
|
+
self.shown = (shot.scroll, shot.scroll + shot.size[1])
|
|
840
|
+
image = shot.load(shot.scroll, shot.size).resize(shot.size, Image.Resampling.BOX)
|
|
841
|
+
self.pixels = np.asarray(image, np.float32).copy()
|
|
842
|
+
for left, top, right, bottom in shot.masks:
|
|
843
|
+
self.pixels[max(0, round(top)) : max(0, round(bottom)), max(0, round(left)) : max(0, round(right))] = np.nan
|
|
844
|
+
|
|
845
|
+
def part(self, box: tuple[int, int, int, int]): # noqa: ANN201 - a NumPy array
|
|
846
|
+
"""The part `box`, in document CSS px, which may reach beyond what the capture shows, with what it doesn't
|
|
847
|
+
show in the median colour of the rest; None if it shows none of it."""
|
|
848
|
+
import numpy as np
|
|
849
|
+
|
|
850
|
+
left, top, right, bottom = box
|
|
851
|
+
part = np.full((bottom - top, right - left, 3), np.nan, np.float32)
|
|
852
|
+
offset = round(self.top)
|
|
853
|
+
rows = (max(top, offset), min(bottom, offset + self.pixels.shape[0]))
|
|
854
|
+
columns = (max(left, 0), min(right, self.pixels.shape[1]))
|
|
855
|
+
if rows[1] > rows[0] and columns[1] > columns[0]:
|
|
856
|
+
part[rows[0] - top : rows[1] - top, columns[0] - left : columns[1] - left] = self.pixels[
|
|
857
|
+
rows[0] - offset : rows[1] - offset, columns[0] : columns[1]
|
|
858
|
+
]
|
|
859
|
+
missing = np.isnan(part)
|
|
860
|
+
if missing.all():
|
|
861
|
+
return None
|
|
862
|
+
if missing.any():
|
|
863
|
+
part = np.where(missing, np.nanmedian(part, axis=(0, 1)), part)
|
|
864
|
+
return part
|
|
865
|
+
|
|
866
|
+
|
|
867
|
+
def _apart(a: _Document, b: _Document, box: CssBox) -> float | None:
|
|
868
|
+
"""How far renderings `a` and `b` are apart in `box`, in document CSS px (see `reference_hint`); None if `box`
|
|
869
|
+
isn't in what both show."""
|
|
870
|
+
import cv2
|
|
871
|
+
|
|
872
|
+
left, top, right, bottom = box
|
|
873
|
+
if any(top < document.shown[0] or bottom > document.shown[1] for document in (a, b)):
|
|
874
|
+
return None
|
|
875
|
+
margin, search = REFERENCE_MARGIN, REFERENCE_SEARCH
|
|
876
|
+
near = (round(left) - margin, round(top) - margin, round(right) + margin, round(bottom) + margin)
|
|
877
|
+
around = (near[0] - search, near[1] - search, near[2] + search, near[3] + search)
|
|
878
|
+
parts = [a.part(near), b.part(around), b.part(near), a.part(around)]
|
|
879
|
+
if any(part is None for part in parts):
|
|
880
|
+
return None
|
|
881
|
+
|
|
882
|
+
def found(template, area) -> float: # noqa: ANN001
|
|
883
|
+
best = float(cv2.matchTemplate(area, template, cv2.TM_SQDIFF).min())
|
|
884
|
+
return (max(0.0, best) / template.size) ** 0.5
|
|
885
|
+
|
|
886
|
+
return max(found(parts[0], parts[1]), found(parts[2], parts[3]))
|
|
764
887
|
|
|
765
888
|
|
|
766
889
|
# Computed style properties of DOM snapshot elements that a rendering difference often comes from.
|
{servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/page_scenarios.py
RENAMED
|
@@ -788,18 +788,28 @@ class Description:
|
|
|
788
788
|
broken = [element for element in elements if element.get("image") == "broken"]
|
|
789
789
|
for region in diff.regions:
|
|
790
790
|
region.elements = diffing.elements_in(elements, region.box)
|
|
791
|
-
|
|
791
|
+
# Comparisons of the same engine's captures, at their resolution, tell minor changes apart, from the
|
|
792
|
+
# measurements of all regions; between engines, everything differs in anti-aliasing.
|
|
793
|
+
minor = not diff.preset.css
|
|
794
|
+
measurements = []
|
|
795
|
+
for region in diff.regions if minor else diff.regions[: diffing.MAX_CROPS]:
|
|
792
796
|
measurement = diffing.measure(stored.shot, actual.shot, diff, region)
|
|
797
|
+
measurements.append(measurement)
|
|
793
798
|
region.measured = asdict(measurement)
|
|
794
799
|
region.reading = measurement.reading(text=any(element.get("text") for element in region.elements))
|
|
800
|
+
if minor:
|
|
801
|
+
diffing.mark_minor(stored.shot, actual.shot, diff, measurements)
|
|
802
|
+
for region in diff.regions[: len(measurements)]:
|
|
795
803
|
missing = sorted({str(element["src"]) for element in diffing.elements_in(broken, region.box, 3)})
|
|
796
804
|
if missing:
|
|
805
|
+
region.minor = False
|
|
797
806
|
region.reading = f"image {', '.join(missing)} did not load in {self.labels[1]}; {region.reading}"
|
|
798
|
-
if self.reference and name in self.reference
|
|
799
|
-
|
|
807
|
+
reference = self.reference[name].shot if self.reference and name in self.reference else None
|
|
808
|
+
if reference is not None:
|
|
809
|
+
diffing.reference_hint(stored.shot, actual.shot, reference, diff.regions)
|
|
800
810
|
overlay = _overlay(self.directory, name)
|
|
801
811
|
images = diffing.write_review_images(
|
|
802
|
-
stored.shot, actual.shot, diff, overlay, self.directory, name, self.labels
|
|
812
|
+
stored.shot, actual.shot, diff, overlay, self.directory, name, self.labels, reference
|
|
803
813
|
)
|
|
804
814
|
captures[name].update(
|
|
805
815
|
diff.to_dict(),
|
|
@@ -1011,9 +1021,11 @@ class PageRuns:
|
|
|
1011
1021
|
def store(self, engine: str) -> CaptureStore | None:
|
|
1012
1022
|
return CaptureStore(self.baselines, self.device, engine) if self.baselines else None
|
|
1013
1023
|
|
|
1014
|
-
def stored_reference(self, scenario: Scenario) -> dict[str, Capture]:
|
|
1024
|
+
def stored_reference(self, scenario: Scenario) -> tuple[dict[str, Capture], dict[str, object]]:
|
|
1025
|
+
"""The reference engine's stored captures of `scenario`, with how they were made; empty without them."""
|
|
1015
1026
|
store = self.store(self.reference) if self.reference else None
|
|
1016
|
-
|
|
1027
|
+
captures = store.captures(scenario.name) if store else {}
|
|
1028
|
+
return captures, store.origin(scenario.name) if store and captures else {}
|
|
1017
1029
|
|
|
1018
1030
|
def reference_captures(self, scenario: Scenario) -> tuple[dict[str, Capture], dict[str, object]] | None:
|
|
1019
1031
|
"""The reference engine's captures of `scenario`, stored or rendered now (and then stored), with how they
|
{servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/plugin.py
RENAMED
|
@@ -38,7 +38,15 @@ COMPARISON = re.compile(r"::test_matches_(baseline|reference)\[")
|
|
|
38
38
|
EXPECTED_KEY = pytest.StashKey[list[str]]()
|
|
39
39
|
# Marks a per-test artifacts folder, so that the next run can remove it.
|
|
40
40
|
TEST_ARTIFACTS_MARKER = "test-key.txt"
|
|
41
|
-
RUN_ARTIFACTS = (
|
|
41
|
+
RUN_ARTIFACTS = (
|
|
42
|
+
"results.json",
|
|
43
|
+
"hilog.txt",
|
|
44
|
+
"servoshell.log",
|
|
45
|
+
"mitmproxy.log",
|
|
46
|
+
"chromedriver.log",
|
|
47
|
+
"report.html",
|
|
48
|
+
"report-standalone.html",
|
|
49
|
+
)
|
|
42
50
|
TEST_KEY = pytest.StashKey[str]()
|
|
43
51
|
NEEDS_TEST_IME = "needs a test IME: pass --test-ime <bundle>"
|
|
44
52
|
|
|
@@ -108,6 +116,11 @@ def pytest_addoption(parser: pytest.Parser) -> None:
|
|
|
108
116
|
group.addoption(
|
|
109
117
|
"--update-references", action="store_true", help="render the references anew even where they are stored"
|
|
110
118
|
)
|
|
119
|
+
group.addoption(
|
|
120
|
+
"--standalone-report",
|
|
121
|
+
action="store_true",
|
|
122
|
+
help="also write the page scenario report with its images inlined, one file to share: report-standalone.html",
|
|
123
|
+
)
|
|
111
124
|
group.addoption("--chromedriver", type=Path, help="chromedriver matching ArkWeb's Chromium, for the arkweb engine")
|
|
112
125
|
|
|
113
126
|
|
|
@@ -309,7 +322,7 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int | pytest.ExitC
|
|
|
309
322
|
path.write_text(run.report.to_json(), encoding="utf-8")
|
|
310
323
|
from .report import write_report
|
|
311
324
|
|
|
312
|
-
run.page_report = write_report(run.artifacts, path)
|
|
325
|
+
run.page_report = write_report(run.artifacts, path, config.option.standalone_report)
|
|
313
326
|
if summary := os.environ.get("GITHUB_STEP_SUMMARY"):
|
|
314
327
|
with open(summary, "a", encoding="utf-8") as f:
|
|
315
328
|
f.write(_markdown_summary(run.report))
|
|
@@ -469,6 +482,9 @@ def pytest_terminal_summary(terminalreporter: pytest.TerminalReporter, config: p
|
|
|
469
482
|
terminalreporter.line("no unexpected results")
|
|
470
483
|
if run.page_report is not None:
|
|
471
484
|
terminalreporter.line(f"page scenario report: {run.page_report}")
|
|
485
|
+
standalone = run.page_report.with_name("report-standalone.html")
|
|
486
|
+
if config.option.standalone_report and standalone.is_file():
|
|
487
|
+
terminalreporter.line(f"as one file to share: {standalone}")
|
|
472
488
|
if failed := _failed_comparisons(run.report):
|
|
473
489
|
terminalreporter.line(f"{failed} page scenario comparison(s) failed; to triage them, read {SKILL / 'SKILL.md'}")
|
|
474
490
|
|