servo-embedder-tests 0.2.0__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/PKG-INFO +42 -18
  2. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/README.md +40 -17
  3. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/pyproject.toml +2 -1
  4. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/pyproject.toml.orig +2 -1
  5. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/cli.py +11 -4
  6. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/diffing.py +151 -28
  7. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/page_scenarios.py +18 -6
  8. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/plugin.py +18 -2
  9. servo_embedder_tests-0.3.0/src/servo_embedder_tests/report.py +932 -0
  10. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/scenarios/page_scenarios/test_pages.py +33 -26
  11. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/skills/compare-renderings/SKILL.md +17 -5
  12. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/skills/compare-renderings/describe_regions.py +4 -2
  13. servo_embedder_tests-0.2.0/src/servo_embedder_tests/report.py +0 -458
  14. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/LICENSE-APACHE +0 -0
  15. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/LICENSE-MIT +0 -0
  16. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/__init__.py +0 -0
  17. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/__main__.py +0 -0
  18. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/describe.py +0 -0
  19. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/engines.py +0 -0
  20. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/errors.py +0 -0
  21. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/expectations.py +0 -0
  22. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/hookspecs.py +0 -0
  23. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/oracles.py +0 -0
  24. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/basic.html +0 -0
  25. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/history-a.html +0 -0
  26. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/history-b.html +0 -0
  27. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/ime.html +0 -0
  28. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/report.js +0 -0
  29. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/10_coordinate_transforms.html +0 -0
  30. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/11_symbols_and_use.html +0 -0
  31. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/12_basic_text.html +0 -0
  32. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/13_text_paths.html +0 -0
  33. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/14_clipping_paths.html +0 -0
  34. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/15_masking.html +0 -0
  35. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/16_filter_effects_basic.html +0 -0
  36. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/1_basic_shapes.html +0 -0
  37. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/2_path_commands.html +0 -0
  38. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/3_path_curves.html +0 -0
  39. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/4_stroke_properties.html +0 -0
  40. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/5_linear_gradients.html +0 -0
  41. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/6_radial_gradients.html +0 -0
  42. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/7_patterns.html +0 -0
  43. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/8_markers.html +0 -0
  44. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/9_viewbox_scaling.html +0 -0
  45. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/LICENSE +0 -0
  46. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/README.md +0 -0
  47. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/security/layer1_xml_validation.html +0 -0
  48. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/security/layer2_css_injection.html +0 -0
  49. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/security/layer2_import_fetch.html +0 -0
  50. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/security/layer3_fetch_allowlist.html +0 -0
  51. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/security/layer4_expansion_limits.html +0 -0
  52. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/security/layer4_use_self_cycle.html +0 -0
  53. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/security/layer5_mask_recursion.html +0 -0
  54. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/svg/security/layer5_render_safety.html +0 -0
  55. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages/webview_apps/scroll.html +0 -0
  56. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/pages.py +0 -0
  57. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/platform/__init__.py +0 -0
  58. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/platform/desktop.py +0 -0
  59. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/platform/ohos.py +0 -0
  60. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/proxy.py +0 -0
  61. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/scenarios/__init__.py +0 -0
  62. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/scenarios/ime/__init__.py +0 -0
  63. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/scenarios/ime/test_text_input.py +0 -0
  64. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/scenarios/navigation/__init__.py +0 -0
  65. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/scenarios/navigation/test_local_pages.py +0 -0
  66. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/scenarios/page_scenarios/__init__.py +0 -0
  67. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/scenarios/sites/__init__.py +0 -0
  68. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/scenarios/sites/mossel.py +0 -0
  69. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/scenarios/sites/test_mossel.py +0 -0
  70. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/scenarios/sites/test_servo_org.py +0 -0
  71. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/scenarios/svg/__init__.py +0 -0
  72. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/scenarios/svg/test_security.py +0 -0
  73. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/scenarios/webview_apps/__init__.py +0 -0
  74. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/scenarios/webview_apps/apps/arkweb-test.toml +0 -0
  75. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/scenarios/webview_apps/flows.py +0 -0
  76. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/scenarios/webview_apps/test_apps.py +0 -0
  77. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/screenshots.py +0 -0
  78. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/servoshell.py +0 -0
  79. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/svg_cards.py +0 -0
  80. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/touch.py +0 -0
  81. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/ui.py +0 -0
  82. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/webdriver.py +0 -0
  83. {servo_embedder_tests-0.2.0 → servo_embedder_tests-0.3.0}/src/servo_embedder_tests/webview_app.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: servo-embedder-tests
3
- Version: 0.2.0
3
+ Version: 0.3.0
4
4
  Summary: Embedder tests for Servo: what servoshell and its users do beyond WPT, on OpenHarmony, HarmonyOS and desktop
5
5
  License-Expression: Apache-2.0 OR MIT
6
6
  License-File: LICENSE-APACHE
@@ -16,6 +16,7 @@ Requires-Dist: tomlkit>=0.13
16
16
  Requires-Dist: urllib3>=2
17
17
  Requires-Python: >=3.11
18
18
  Project-URL: Repository, https://github.com/openharmony-rs/servo-embedder-tests
19
+ Project-URL: Changelog, https://github.com/openharmony-rs/servo-embedder-tests/blob/main/CHANGELOG.md
19
20
  Description-Content-Type: text/markdown
20
21
 
21
22
  # servo-embedder-tests
@@ -160,8 +161,11 @@ Each scenario has three tests:
160
161
  - `test_matches_baseline`: each capture matches the engine's baseline;
161
162
  - `test_matches_reference`: each capture matches the reference engine's.
162
163
 
163
- The first is a regression check: it finds what changed since the baselines were stored, e.g. with an engine update.
164
- The second is an engine comparison: it finds the gaps between two engines. `--engine-version NAME` and
164
+ `test_matches_baseline` is a regression check: it finds what changed since the baselines were stored, e.g. with an
165
+ engine update. `test_matches_reference` is an engine comparison: it finds the gaps between two engines. Together they
166
+ make a three-way comparison: where a page changed since its baseline, the reference engine's stored rendering tells
167
+ whether each change brought it closer to the reference, likely an improvement, or took it further away, likely a
168
+ regression. `--engine-version NAME` and
165
169
  `--reference-engine-version NAME` name the builds, e.g. a commit or a release. The name is kept with their captures
166
170
  and baselines, so that `diff.json` and the report say which builds were compared. Stored references keep the name
167
171
  they were rendered with: `--reference-engine-version` names those rendered in the run, see `--update-references`.
@@ -170,7 +174,8 @@ The page is rendered once per engine for all three. Known differences between en
170
174
  `test_matches_reference`, so that they don't hide regressions. The captures of each engine are kept in
171
175
  `--baselines` under `<device>/<engine>/<scenario>/`; the device is its model, or `--reference-key`. The reference
172
176
  engine's captures are expectations like the baselines: with them stored, a run doesn't need the reference engine. It
173
- renders the missing ones if it has the engine, and `--update-references` renders all anew.
177
+ renders the missing ones if it has the engine, and `--update-references` renders all anew. The engine comparisons
178
+ run before the regression checks, so that references rendered in the run judge the changes too.
174
179
 
175
180
  A capture is a screenshot of the web view, taken by the OS the same way for every engine. It's taken after:
176
181
 
@@ -191,16 +196,22 @@ Each comparison test writes `diff.json` with, for each capture:
191
196
 
192
197
  - the differing regions in the actual capture's viewport, in CSS px;
193
198
  - the elements whose boxes best match each region, with their text, computed fonts and paint properties;
194
- - for the largest regions, a `reading` of pixel measurements: moved by how much, only anti-aliasing, wider with
195
- the same ink (spacing), less or more ink (stroke weight, or content missing), shapes that differ even when
196
- aligned, or a colour drawn in another colour (such as a colour emoji drawn black). It starts with the images in the
197
- region that did not load, if any;
199
+ - for the regions (of baselines all, else the largest), a `reading` of pixel measurements: moved by how much, only
200
+ anti-aliasing, wider with the same ink (spacing), less or more ink (stroke weight, or content missing), shapes that
201
+ differ even when aligned, or a colour drawn in another colour (such as a colour emoji drawn black). It starts with
202
+ the images in the region that did not load, if any;
198
203
  - `dom_changes`, the elements whose text, box, style or image load state changed;
199
- - for baselines, whether each region moved closer to the reference engine's rendering or further from it, a hint
200
- whether a change is an improvement.
201
-
202
- It also writes `<capture>-compare.png` (expected, actual and the differing pixels) and a crop of each region in
203
- `regions/`, expected above actual above the differing pixels. Those crops are what to send to an image-capable model
204
+ - for baselines, whether a region's change is `minor`: anti-aliasing or sub-pixel positions only, as when text is
205
+ rasterized with another gamma. It vanishes when the captures are compared in CSS px with a tolerance above what
206
+ engines differ by, lies on edges both captures have, didn't move by a device pixel or more and changed no colour;
207
+ - for baselines, with the reference engine's captures stored, whether each region moved closer to the reference
208
+ engine's rendering or further from it (`toward_reference`: closer, further or same), a hint whether a change is an
209
+ improvement. Each rendering's part of the region is looked for in the other's within 24 CSS px, so that content the
210
+ engines lay out a few pixels apart counts as found, and a blank as far from content.
211
+
212
+ It also writes `<capture>-compare.png` (expected, actual and the differing pixels), for baselines the reference
213
+ engine's view of the compared part as `<capture>-reference.png`, and a crop of each region in `regions/`, expected
214
+ above actual above the differing pixels. Those crops are what to send to an image-capable model
204
215
  for what the readings leave open: the `compare-renderings` skill describes how an agent triages a run, and its
205
216
  script asks any OpenAI-compatible endpoint about the regions.
206
217
 
@@ -208,11 +219,24 @@ A comparison test passes or fails as soon as the tiles are compared. Describing
208
219
  readings, DOM changes and images, takes seconds per capture, so it runs in up to four background processes while the
209
220
  run goes on to the next test; `diff.json` and the images are complete when the run ends.
210
221
 
211
- For people, a run writes `report.html` to the artifacts folder: per scenario, each differing capture next to what it
212
- was compared with (hover the actual one to see the expected one in its place), the differing pixels, the regions
213
- numbered with their enlarged crops and readings, and the DOM changes. Where an agent triaged the run, the report also
214
- shows the image model's answers and the agent's conclusion (`triage.json` in the test's folder, see the skill), so
215
- that you can check them against the pictures. `servo-embedder-tests report <artifacts>` writes it again.
222
+ For people, a run writes `report.html` to the artifacts folder. It starts with each page's verdict: unchanged since
223
+ its baseline, or, where it changed, a likely regression or improvement by whether the changes took it away from the
224
+ reference engine's rendering or toward it, mixed, or unclear. Minor changes are shown, with dashed boxes and folded
225
+ away, but left out of the verdict; a page that changed only in them has minor changes. Pages get a "Check" flag, with
226
+ links, for what their verdict leaves out: captures that weren't compared, such as those of a page that got shorter,
227
+ and regions that matched the reference engine's rendering before but no longer do, however small. For each changed capture it shows the old rendering,
228
+ the new one (hover it to see the old one in its place), the reference engine's and the changed pixels, then each
229
+ region numbered with its verdict, the renderings of it side by side and enlarged, its readings and the DOM changes.
230
+ The engine comparison, the gaps that remain to the reference engine, is folded away below; without baselines, it is
231
+ all the report shows. Where an agent triaged the run, the report also shows the image model's answers and the agent's
232
+ conclusion (`triage.json` in the test's folder, see the skill), so that you can check them against the pictures.
233
+ `servo-embedder-tests report <artifacts>` writes it again.
234
+
235
+ The report shows the images in the artifacts folder, so it needs the folder. To share one file instead, pass
236
+ `--standalone-report` to `run`, or `--standalone` to `report`: `report-standalone.html` has its images inlined, the
237
+ full captures at one pixel per CSS pixel and the region crops sharper, in WebP, each image once. For 8 GenUI pages it
238
+ is 13 MB, against 106 MB of PNGs, and takes about a second per page. It can be made later from the artifacts folder,
239
+ and `report` writes it again along with `report.html` if it's there.
216
240
 
217
241
  `page-scenarios.toml` in the folder sets defaults for all scenarios, `scenario.toml` in a scenario's folder
218
242
  overrides them. Changes to the page are opt-in, under `[page]`:
@@ -140,8 +140,11 @@ Each scenario has three tests:
140
140
  - `test_matches_baseline`: each capture matches the engine's baseline;
141
141
  - `test_matches_reference`: each capture matches the reference engine's.
142
142
 
143
- The first is a regression check: it finds what changed since the baselines were stored, e.g. with an engine update.
144
- The second is an engine comparison: it finds the gaps between two engines. `--engine-version NAME` and
143
+ `test_matches_baseline` is a regression check: it finds what changed since the baselines were stored, e.g. with an
144
+ engine update. `test_matches_reference` is an engine comparison: it finds the gaps between two engines. Together they
145
+ make a three-way comparison: where a page changed since its baseline, the reference engine's stored rendering tells
146
+ whether each change brought it closer to the reference, likely an improvement, or took it further away, likely a
147
+ regression. `--engine-version NAME` and
145
148
  `--reference-engine-version NAME` name the builds, e.g. a commit or a release. The name is kept with their captures
146
149
  and baselines, so that `diff.json` and the report say which builds were compared. Stored references keep the name
147
150
  they were rendered with: `--reference-engine-version` names those rendered in the run, see `--update-references`.
@@ -150,7 +153,8 @@ The page is rendered once per engine for all three. Known differences between en
150
153
  `test_matches_reference`, so that they don't hide regressions. The captures of each engine are kept in
151
154
  `--baselines` under `<device>/<engine>/<scenario>/`; the device is its model, or `--reference-key`. The reference
152
155
  engine's captures are expectations like the baselines: with them stored, a run doesn't need the reference engine. It
153
- renders the missing ones if it has the engine, and `--update-references` renders all anew.
156
+ renders the missing ones if it has the engine, and `--update-references` renders all anew. The engine comparisons
157
+ run before the regression checks, so that references rendered in the run judge the changes too.
154
158
 
155
159
  A capture is a screenshot of the web view, taken by the OS the same way for every engine. It's taken after:
156
160
 
@@ -171,16 +175,22 @@ Each comparison test writes `diff.json` with, for each capture:
171
175
 
172
176
  - the differing regions in the actual capture's viewport, in CSS px;
173
177
  - the elements whose boxes best match each region, with their text, computed fonts and paint properties;
174
- - for the largest regions, a `reading` of pixel measurements: moved by how much, only anti-aliasing, wider with
175
- the same ink (spacing), less or more ink (stroke weight, or content missing), shapes that differ even when
176
- aligned, or a colour drawn in another colour (such as a colour emoji drawn black). It starts with the images in the
177
- region that did not load, if any;
178
+ - for the regions (of baselines all, else the largest), a `reading` of pixel measurements: moved by how much, only
179
+ anti-aliasing, wider with the same ink (spacing), less or more ink (stroke weight, or content missing), shapes that
180
+ differ even when aligned, or a colour drawn in another colour (such as a colour emoji drawn black). It starts with
181
+ the images in the region that did not load, if any;
178
182
  - `dom_changes`, the elements whose text, box, style or image load state changed;
179
- - for baselines, whether each region moved closer to the reference engine's rendering or further from it, a hint
180
- whether a change is an improvement.
181
-
182
- It also writes `<capture>-compare.png` (expected, actual and the differing pixels) and a crop of each region in
183
- `regions/`, expected above actual above the differing pixels. Those crops are what to send to an image-capable model
183
+ - for baselines, whether a region's change is `minor`: anti-aliasing or sub-pixel positions only, as when text is
184
+ rasterized with another gamma. It vanishes when the captures are compared in CSS px with a tolerance above what
185
+ engines differ by, lies on edges both captures have, didn't move by a device pixel or more and changed no colour;
186
+ - for baselines, with the reference engine's captures stored, whether each region moved closer to the reference
187
+ engine's rendering or further from it (`toward_reference`: closer, further or same), a hint whether a change is an
188
+ improvement. Each rendering's part of the region is looked for in the other's within 24 CSS px, so that content the
189
+ engines lay out a few pixels apart counts as found, and a blank as far from content.
190
+
191
+ It also writes `<capture>-compare.png` (expected, actual and the differing pixels), for baselines the reference
192
+ engine's view of the compared part as `<capture>-reference.png`, and a crop of each region in `regions/`, expected
193
+ above actual above the differing pixels. Those crops are what to send to an image-capable model
184
194
  for what the readings leave open: the `compare-renderings` skill describes how an agent triages a run, and its
185
195
  script asks any OpenAI-compatible endpoint about the regions.
186
196
 
@@ -188,11 +198,24 @@ A comparison test passes or fails as soon as the tiles are compared. Describing
188
198
  readings, DOM changes and images, takes seconds per capture, so it runs in up to four background processes while the
189
199
  run goes on to the next test; `diff.json` and the images are complete when the run ends.
190
200
 
191
- For people, a run writes `report.html` to the artifacts folder: per scenario, each differing capture next to what it
192
- was compared with (hover the actual one to see the expected one in its place), the differing pixels, the regions
193
- numbered with their enlarged crops and readings, and the DOM changes. Where an agent triaged the run, the report also
194
- shows the image model's answers and the agent's conclusion (`triage.json` in the test's folder, see the skill), so
195
- that you can check them against the pictures. `servo-embedder-tests report <artifacts>` writes it again.
201
+ For people, a run writes `report.html` to the artifacts folder. It starts with each page's verdict: unchanged since
202
+ its baseline, or, where it changed, a likely regression or improvement by whether the changes took it away from the
203
+ reference engine's rendering or toward it, mixed, or unclear. Minor changes are shown, with dashed boxes and folded
204
+ away, but left out of the verdict; a page that changed only in them has minor changes. Pages get a "Check" flag, with
205
+ links, for what their verdict leaves out: captures that weren't compared, such as those of a page that got shorter,
206
+ and regions that matched the reference engine's rendering before but no longer do, however small. For each changed capture it shows the old rendering,
207
+ the new one (hover it to see the old one in its place), the reference engine's and the changed pixels, then each
208
+ region numbered with its verdict, the renderings of it side by side and enlarged, its readings and the DOM changes.
209
+ The engine comparison, the gaps that remain to the reference engine, is folded away below; without baselines, it is
210
+ all the report shows. Where an agent triaged the run, the report also shows the image model's answers and the agent's
211
+ conclusion (`triage.json` in the test's folder, see the skill), so that you can check them against the pictures.
212
+ `servo-embedder-tests report <artifacts>` writes it again.
213
+
214
+ The report shows the images in the artifacts folder, so it needs the folder. To share one file instead, pass
215
+ `--standalone-report` to `run`, or `--standalone` to `report`: `report-standalone.html` has its images inlined, the
216
+ full captures at one pixel per CSS pixel and the region crops sharper, in WebP, each image once. For 8 GenUI pages it
217
+ is 13 MB, against 106 MB of PNGs, and takes about a second per page. It can be made later from the artifacts folder,
218
+ and `report` writes it again along with `report.html` if it's there.
196
219
 
197
220
  `page-scenarios.toml` in the folder sets defaults for all scenarios, `scenario.toml` in a scenario's folder
198
221
  overrides them. Changes to the page are opt-in, under `[page]`:
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "servo-embedder-tests"
3
- version = "0.2.0"
3
+ version = "0.3.0"
4
4
  description = "Embedder tests for Servo: what servoshell and its users do beyond WPT, on OpenHarmony, HarmonyOS and desktop"
5
5
  readme = "README.md"
6
6
  requires-python = ">=3.11"
@@ -23,6 +23,7 @@ dependencies = [
23
23
 
24
24
  [project.urls]
25
25
  Repository = "https://github.com/openharmony-rs/servo-embedder-tests"
26
+ Changelog = "https://github.com/openharmony-rs/servo-embedder-tests/blob/main/CHANGELOG.md"
26
27
 
27
28
  [project.scripts]
28
29
  servo-embedder-tests = "servo_embedder_tests.cli:main"
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "servo-embedder-tests"
3
- version = "0.2.0"
3
+ version = "0.3.0"
4
4
  description = "Embedder tests for Servo: what servoshell and its users do beyond WPT, on OpenHarmony, HarmonyOS and desktop"
5
5
  readme = "README.md"
6
6
  requires-python = ">=3.11"
@@ -20,6 +20,7 @@ dependencies = [
20
20
 
21
21
  [project.urls]
22
22
  Repository = "https://github.com/openharmony-rs/servo-embedder-tests"
23
+ Changelog = "https://github.com/openharmony-rs/servo-embedder-tests/blob/main/CHANGELOG.md"
23
24
 
24
25
  [project.scripts]
25
26
  servo-embedder-tests = "servo_embedder_tests.cli:main"
@@ -17,7 +17,7 @@ from . import __version__, svg_cards
17
17
  from .expectations import Expectations, RunReport, update
18
18
  from .page_scenarios import Capture, CaptureStore, Rendering
19
19
  from .plugin import BUNDLED_SCENARIOS, SKILL
20
- from .report import write_report
20
+ from .report import STANDALONE, write_report
21
21
 
22
22
 
23
23
  def _bundled(arg: str) -> str | None:
@@ -221,12 +221,19 @@ def _report(argv: Sequence[str]) -> int:
221
221
  description="Write the page scenario report of a run again, e.g. after an agent added its conclusions.",
222
222
  )
223
223
  parser.add_argument("artifacts", type=Path, help="the run's --artifacts folder")
224
+ parser.add_argument(
225
+ "--standalone",
226
+ action="store_true",
227
+ help=f"also write {STANDALONE}, the report with its images inlined, one file to share (written anyway if "
228
+ "it's there)",
229
+ )
224
230
  options = parser.parse_args(argv)
225
- path = write_report(options.artifacts)
231
+ path = write_report(options.artifacts, standalone=options.standalone)
226
232
  if path is None:
227
233
  print(f"error: no page scenarios ran in {options.artifacts}")
228
234
  return 1
229
- print(f"wrote {path}")
235
+ standalone = path.with_name(STANDALONE)
236
+ print(f"wrote {path}" + (f" and {standalone}" if standalone.is_file() else ""))
230
237
  return 0
231
238
 
232
239
 
@@ -256,7 +263,7 @@ def main(argv: Sequence[str] | None = None) -> int:
256
263
  "update-expectations": "rewrite expectations from results.json files",
257
264
  "accept": "store page scenario captures of a run as baselines: accept ARTIFACTS --baselines DIR",
258
265
  "skill": "print where the agent skill for triaging page scenarios is, or copy it: skill [FOLDER]",
259
- "report": "write a run's page scenario report (report.html) again: report ARTIFACTS",
266
+ "report": "write a run's page scenario report (report.html) again: report ARTIFACTS [--standalone]",
260
267
  "svg-cards": "write the cards of the SVG test pages as page scenarios: svg-cards FOLDER",
261
268
  "record-dump": "record a mitmproxy dump of the network scenarios: record-dump PATH [run options]",
262
269
  }
@@ -49,6 +49,14 @@ EDGE_CONTRAST = 24
49
49
  COLOUR_CHROMA = 48
50
50
  COLOUR_AREA = 40
51
51
  COLOUR_REACH = 2
52
+ # Whether a change moved toward another engine's rendering: how far, in CSS px, a region's content may lie from where
53
+ # the other engine has it and still count as found there (engines lay out text a few pixels apart), and the CSS px of
54
+ # context around the region, and the share by which the distance to it must change to count (10% made 11 of 163
55
+ # regions of pages with features turned off come out closer, 15% 7; 20% lost real changes). More context dilutes
56
+ # the change: with 8, 12 or 16 px, 88, 80 or 69 of those regions came out further, against 95 with 4.
57
+ REFERENCE_SEARCH = 24
58
+ REFERENCE_MARGIN = 4
59
+ REFERENCE_CHANGE = 0.15
52
60
  HUES = (
53
61
  "red",
54
62
  "orange",
@@ -91,6 +99,16 @@ class Preset:
91
99
  BASELINE = Preset(tolerance=12, tile=16, tile_share=0.05, css=False)
92
100
  # Different engines: anti-aliasing and rasterization differ everywhere, so only clear differences count.
93
101
  REFERENCE = Preset(tolerance=48, tile=16, tile_share=0.05, css=True)
102
+ # A change is minor, anti-aliasing or sub-pixel positions, when comparing the two captures in CSS px with
103
+ # MINOR_TOLERANCE finds nothing there, most of its differing pixels lie on edges both captures have, it didn't move
104
+ # by MINOR_MOVE device px or more and no colour changed. On GenUI pages on a DAYU200 (2026-11-18), that made minor
105
+ # 99% of the changed pixels of another text gamma or of whole captures moved by a quarter device pixel, 57% of edges
106
+ # blurred by 0.6 device px; none of 64 injected changes (hidden, resized, refonted, recoloured, spaced, shifted by a
107
+ # device pixel or more, underlined) and 18 of 178 regions, under 1% of the changed pixels, of pages with features
108
+ # turned off. A tolerance of 48 made as few changes minor that shouldn't be, but only 41% and 28% of the moved and
109
+ # blurred captures.
110
+ MINOR_TOLERANCE = 64
111
+ MINOR_MOVE = 0.75
94
112
 
95
113
 
96
114
  @dataclass
@@ -103,7 +121,12 @@ class Region:
103
121
  mean_difference: float
104
122
  max_difference: int
105
123
  elements: list[dict[str, object]] = field(default_factory=list)
124
+ # Whether the actual rendering is "closer" to another engine's than the expected one, "further" from it or the
125
+ # "same"; None where they can't be compared. `reference` says it in words, with why where they can't.
106
126
  reference: str | None = None
127
+ toward_reference: str | None = None
128
+ # Whether the change is minor: anti-aliasing or sub-pixel positions (see `mark_minor`).
129
+ minor: bool = False
107
130
  crop: str | None = None
108
131
  # Pixel measurements (see `measure`) and what they mean.
109
132
  measured: dict[str, object] | None = None
@@ -370,17 +393,41 @@ def stacked(images: Sequence[tuple[str, Image.Image]], dest: Path) -> Path:
370
393
  return dest
371
394
 
372
395
 
396
+ def _aligned(shot: Shot, top: float, size: tuple[int, int], pixels: tuple[int, int]) -> Image.Image:
397
+ """The `size` CSS pixels of the document from `top` down as `shot` shows them, scaled to `pixels`, grey where it
398
+ doesn't show them."""
399
+ with Image.open(shot.path) as image:
400
+ rgb = image.convert("RGB")
401
+ scale = rgb.width / shot.size[0]
402
+ canvas = Image.new("RGB", (round(size[0] * scale), round(size[1] * scale)), (128, 128, 128))
403
+ canvas.paste(rgb, (0, round((shot.scroll - top) * scale)))
404
+ return canvas if canvas.size == pixels else canvas.resize(pixels, Image.Resampling.BOX)
405
+
406
+
373
407
  def write_review_images(
374
- expected: Shot, actual: Shot, diff: Diff, overlay: Path, directory: Path, name: str, labels: tuple[str, str]
408
+ expected: Shot,
409
+ actual: Shot,
410
+ diff: Diff,
411
+ overlay: Path,
412
+ directory: Path,
413
+ name: str,
414
+ labels: tuple[str, str],
415
+ reference: Shot | None = None,
375
416
  ) -> dict[str, str]:
376
417
  """Writes the compared part of each capture, at its own resolution, to `<name>-expected.png` and
377
418
  `<name>-actual.png`, `<name>-compare.png` (expected | actual | differing) and a crop of each of the largest regions
378
419
  to `regions/<name>-<n>.png` (expected above actual above the differing pixels), whose paths go into the regions.
379
- Returns the names of the images by what they show, the overlay of differing pixels included."""
420
+ With a `reference`, another engine's capture of the same width in CSS px, its view of the compared part goes to
421
+ `<name>-reference.png`, at the actual one's resolution. Returns the names of the images by what they show, the
422
+ overlay of differing pixels included."""
380
423
  common = diff.size
381
424
  a, b = expected.load(diff.top, common), actual.load(diff.top, common)
382
425
  a.save(directory / f"{name}-expected.png")
383
426
  b.save(directory / f"{name}-actual.png")
427
+ written = {}
428
+ if reference is not None and reference.size[0] == actual.size[0]:
429
+ _aligned(reference, diff.top, common, b.size).save(directory / f"{name}-reference.png")
430
+ written["reference"] = f"{name}-reference.png"
384
431
  with Image.open(overlay) as image:
385
432
  marked = image.convert("RGB")
386
433
  a, b = (image.resize(marked.size, Image.Resampling.BOX) if image.size != marked.size else image for image in (a, b))
@@ -409,6 +456,7 @@ def write_review_images(
409
456
  "expected": f"{name}-expected.png",
410
457
  "actual": f"{name}-actual.png",
411
458
  "differing": overlay.name,
459
+ **written,
412
460
  }
413
461
 
414
462
 
@@ -661,6 +709,19 @@ def measure(expected: Shot, actual: Shot, diff: Diff, region: Region) -> Measure
661
709
  )
662
710
 
663
711
 
712
+ def mark_minor(expected: Shot, actual: Shot, diff: Diff, measurements: Sequence[Measurement]) -> None:
713
+ """Marks the regions of `diff`, a comparison of `expected` and `actual`, that are minor (see MINOR_TOLERANCE), from
714
+ the `measurements` of each."""
715
+ coarse = compare(expected, actual, REFERENCE.updated({"tolerance": MINOR_TOLERANCE}))
716
+ for region, measurement in zip(diff.regions, measurements, strict=True):
717
+ left, top, right, bottom = region.box
718
+ if any(left < r.box[2] and right > r.box[0] and top < r.box[3] and bottom > r.box[1] for r in coarse.regions):
719
+ continue
720
+ moved = measurement.moved and max(map(abs, measurement.offset)) * diff.scale >= MINOR_MOVE
721
+ edges = measurement.on_edges is None or measurement.on_edges >= 0.5
722
+ region.minor = edges and not moved and not measurement.colours
723
+
724
+
664
725
  @dataclass(frozen=True)
665
726
  class Measurement:
666
727
  """How a region differs: `offset`, in CSS px, best aligns the actual rendering onto the expected one, with mean
@@ -679,6 +740,12 @@ class Measurement:
679
740
  on_edges: float | None = None
680
741
  colours: tuple[tuple[str, str, float], ...] = ()
681
742
 
743
+ @property
744
+ def moved(self) -> bool:
745
+ """Whether the actual rendering is nearly the expected one moved by `offset`."""
746
+ aligned, unshifted = self.difference_aligned, self.difference_unshifted
747
+ return bool(self.offset[0] or self.offset[1]) and aligned <= 8 and aligned <= 0.5 * unshifted
748
+
682
749
  def reading(self, text: bool = True) -> str:
683
750
  """What the measurements mean, in words, for an agent that can't see the crops. Without `text` in the region,
684
751
  the width of the ink, e.g. of a textured background, says nothing about spacing or glyphs, and is left out."""
@@ -686,7 +753,7 @@ class Measurement:
686
753
  aligned, unshifted, ink = self.difference_aligned, self.difference_unshifted, self.ink
687
754
  before, after = self.ink_width
688
755
  parts = []
689
- moved = bool(dx or dy) and aligned <= 8 and aligned <= 0.5 * unshifted
756
+ moved = self.moved
690
757
  if moved:
691
758
  parts.append(f"moved by ({dx:+g}, {dy:+g}) CSS px, otherwise nearly the same")
692
759
  elif unshifted < 3 and (self.on_edges is None or self.on_edges >= 0.5):
@@ -725,42 +792,98 @@ class Measurement:
725
792
  def reference_hint(previous: Shot, current: Shot, reference: Shot, regions: Sequence[Region]) -> None:
726
793
  """Notes in each region whether `current` is closer to `reference`, another engine's rendering, than `previous`.
727
794
 
728
- Regions are in `current`'s viewport. The engines rasterize differently, so this is a hint for triage: a change
729
- toward the reference is more likely an improvement, one away from it more likely a regression.
795
+ Regions are in `current`'s viewport. How far two renderings are apart in a region: each one's part there where it
796
+ best matches the other within REFERENCE_SEARCH CSS px, in colour, the worse of the two. Engines lay out text a few
797
+ pixels apart: compared in place, text that moved a little is further from the reference's text than a blank is,
798
+ and a blank is found in any blank part of the reference, but not the reference's text in it. Beyond the edges of a
799
+ capture, and under masks, the search sees the part's median colour, so that regions at the edges can be aligned
800
+ too. Distances that differ by less than REFERENCE_CHANGE count as the same. On GenUI pages on a DAYU200
801
+ (2026-11-18), with features turned off, 98% of the changed pixels came out further from ArkWeb and 1% closer
802
+ (compared in place: 60% and 14%); of 23 hidden, resized or refonted elements, 14 further and 3 closer (in place: 7
803
+ and 7). The engines rasterize differently, so this is a hint for triage: a change toward the reference is more
804
+ likely an improvement, one away from it more likely a regression.
730
805
  """
731
- shots = (previous, current, reference)
732
- images = [shot.load(shot.scroll, shot.size).resize(shot.size, Image.Resampling.BOX) for shot in shots]
806
+ if reference.size[0] != current.size[0]:
807
+ for region in regions:
808
+ region.reference = (
809
+ f"not comparable: the reference's viewport is {reference.size[0]} CSS px wide, this one's "
810
+ f"{current.size[0]}"
811
+ )
812
+ return
813
+ before, after, ref = (_Document(shot) for shot in (previous, current, reference))
733
814
  for region in regions:
734
815
  left, top, right, bottom = region.box
735
- boxes = [
736
- (
737
- round(left),
738
- round(top + current.scroll - shot.scroll),
739
- round(right),
740
- round(bottom + current.scroll - shot.scroll),
741
- )
742
- for shot in shots
743
- ]
744
- if any(
745
- box[0] < 0 or box[1] < 0 or box[2] > shot.size[0] or box[3] > shot.size[1]
746
- for box, shot in zip(boxes, shots, strict=True)
747
- ):
816
+ box = (left, top + current.scroll, right, bottom + current.scroll)
817
+ old, new = _apart(before, ref, box), _apart(after, ref, box)
818
+ if old is None or new is None:
748
819
  region.reference = "not all in the reference's or the earlier capture's viewport"
749
- continue
750
- before, after, ref = (image.crop(box) for image, box in zip(images, boxes, strict=True))
751
- old, new = _mean(_largest_channel(before, ref)), _mean(_largest_channel(after, ref))
752
- if abs(new - old) < max(1.0, 0.1 * max(old, new)):
820
+ elif abs(new - old) < max(1.0, REFERENCE_CHANGE * max(old, new)):
821
+ region.toward_reference = "same"
753
822
  region.reference = f"as far from the reference as before ({old:.1f} -> {new:.1f})"
754
823
  elif new < old:
824
+ region.toward_reference = "closer"
755
825
  region.reference = f"closer to the reference ({old:.1f} -> {new:.1f})"
756
826
  else:
827
+ region.toward_reference = "further"
757
828
  region.reference = f"further from the reference ({old:.1f} -> {new:.1f})"
758
829
 
759
830
 
760
- def _mean(image: Image.Image) -> float:
761
- histogram = image.histogram()
762
- total = sum(histogram)
763
- return sum(value * count for value, count in enumerate(histogram)) / total if total else 0.0
831
+ class _Document:
832
+ """A capture as an RGB array at one pixel per CSS pixel, placed in the document: NaN under its masks. `shown` is
833
+ the part of the document, from the top down, that it shows."""
834
+
835
+ def __init__(self, shot: Shot) -> None:
836
+ import numpy as np
837
+
838
+ self.top = shot.scroll
839
+ self.shown = (shot.scroll, shot.scroll + shot.size[1])
840
+ image = shot.load(shot.scroll, shot.size).resize(shot.size, Image.Resampling.BOX)
841
+ self.pixels = np.asarray(image, np.float32).copy()
842
+ for left, top, right, bottom in shot.masks:
843
+ self.pixels[max(0, round(top)) : max(0, round(bottom)), max(0, round(left)) : max(0, round(right))] = np.nan
844
+
845
+ def part(self, box: tuple[int, int, int, int]): # noqa: ANN201 - a NumPy array
846
+ """The part `box`, in document CSS px, which may reach beyond what the capture shows, with what it doesn't
847
+ show in the median colour of the rest; None if it shows none of it."""
848
+ import numpy as np
849
+
850
+ left, top, right, bottom = box
851
+ part = np.full((bottom - top, right - left, 3), np.nan, np.float32)
852
+ offset = round(self.top)
853
+ rows = (max(top, offset), min(bottom, offset + self.pixels.shape[0]))
854
+ columns = (max(left, 0), min(right, self.pixels.shape[1]))
855
+ if rows[1] > rows[0] and columns[1] > columns[0]:
856
+ part[rows[0] - top : rows[1] - top, columns[0] - left : columns[1] - left] = self.pixels[
857
+ rows[0] - offset : rows[1] - offset, columns[0] : columns[1]
858
+ ]
859
+ missing = np.isnan(part)
860
+ if missing.all():
861
+ return None
862
+ if missing.any():
863
+ part = np.where(missing, np.nanmedian(part, axis=(0, 1)), part)
864
+ return part
865
+
866
+
867
+ def _apart(a: _Document, b: _Document, box: CssBox) -> float | None:
868
+ """How far renderings `a` and `b` are apart in `box`, in document CSS px (see `reference_hint`); None if `box`
869
+ isn't in what both show."""
870
+ import cv2
871
+
872
+ left, top, right, bottom = box
873
+ if any(top < document.shown[0] or bottom > document.shown[1] for document in (a, b)):
874
+ return None
875
+ margin, search = REFERENCE_MARGIN, REFERENCE_SEARCH
876
+ near = (round(left) - margin, round(top) - margin, round(right) + margin, round(bottom) + margin)
877
+ around = (near[0] - search, near[1] - search, near[2] + search, near[3] + search)
878
+ parts = [a.part(near), b.part(around), b.part(near), a.part(around)]
879
+ if any(part is None for part in parts):
880
+ return None
881
+
882
+ def found(template, area) -> float: # noqa: ANN001
883
+ best = float(cv2.matchTemplate(area, template, cv2.TM_SQDIFF).min())
884
+ return (max(0.0, best) / template.size) ** 0.5
885
+
886
+ return max(found(parts[0], parts[1]), found(parts[2], parts[3]))
764
887
 
765
888
 
766
889
  # Computed style properties of DOM snapshot elements that a rendering difference often comes from.
@@ -788,18 +788,28 @@ class Description:
788
788
  broken = [element for element in elements if element.get("image") == "broken"]
789
789
  for region in diff.regions:
790
790
  region.elements = diffing.elements_in(elements, region.box)
791
- for region in diff.regions[: diffing.MAX_CROPS]:
791
+ # Comparisons of the same engine's captures, at their resolution, tell minor changes apart, from the
792
+ # measurements of all regions; between engines, everything differs in anti-aliasing.
793
+ minor = not diff.preset.css
794
+ measurements = []
795
+ for region in diff.regions if minor else diff.regions[: diffing.MAX_CROPS]:
792
796
  measurement = diffing.measure(stored.shot, actual.shot, diff, region)
797
+ measurements.append(measurement)
793
798
  region.measured = asdict(measurement)
794
799
  region.reading = measurement.reading(text=any(element.get("text") for element in region.elements))
800
+ if minor:
801
+ diffing.mark_minor(stored.shot, actual.shot, diff, measurements)
802
+ for region in diff.regions[: len(measurements)]:
795
803
  missing = sorted({str(element["src"]) for element in diffing.elements_in(broken, region.box, 3)})
796
804
  if missing:
805
+ region.minor = False
797
806
  region.reading = f"image {', '.join(missing)} did not load in {self.labels[1]}; {region.reading}"
798
- if self.reference and name in self.reference:
799
- diffing.reference_hint(stored.shot, actual.shot, self.reference[name].shot, diff.regions)
807
+ reference = self.reference[name].shot if self.reference and name in self.reference else None
808
+ if reference is not None:
809
+ diffing.reference_hint(stored.shot, actual.shot, reference, diff.regions)
800
810
  overlay = _overlay(self.directory, name)
801
811
  images = diffing.write_review_images(
802
- stored.shot, actual.shot, diff, overlay, self.directory, name, self.labels
812
+ stored.shot, actual.shot, diff, overlay, self.directory, name, self.labels, reference
803
813
  )
804
814
  captures[name].update(
805
815
  diff.to_dict(),
@@ -1011,9 +1021,11 @@ class PageRuns:
1011
1021
  def store(self, engine: str) -> CaptureStore | None:
1012
1022
  return CaptureStore(self.baselines, self.device, engine) if self.baselines else None
1013
1023
 
1014
- def stored_reference(self, scenario: Scenario) -> dict[str, Capture]:
1024
+ def stored_reference(self, scenario: Scenario) -> tuple[dict[str, Capture], dict[str, object]]:
1025
+ """The reference engine's stored captures of `scenario`, with how they were made; empty without them."""
1015
1026
  store = self.store(self.reference) if self.reference else None
1016
- return store.captures(scenario.name) if store else {}
1027
+ captures = store.captures(scenario.name) if store else {}
1028
+ return captures, store.origin(scenario.name) if store and captures else {}
1017
1029
 
1018
1030
  def reference_captures(self, scenario: Scenario) -> tuple[dict[str, Capture], dict[str, object]] | None:
1019
1031
  """The reference engine's captures of `scenario`, stored or rendered now (and then stored), with how they
@@ -38,7 +38,15 @@ COMPARISON = re.compile(r"::test_matches_(baseline|reference)\[")
38
38
  EXPECTED_KEY = pytest.StashKey[list[str]]()
39
39
  # Marks a per-test artifacts folder, so that the next run can remove it.
40
40
  TEST_ARTIFACTS_MARKER = "test-key.txt"
41
- RUN_ARTIFACTS = ("results.json", "hilog.txt", "servoshell.log", "mitmproxy.log", "chromedriver.log", "report.html")
41
+ RUN_ARTIFACTS = (
42
+ "results.json",
43
+ "hilog.txt",
44
+ "servoshell.log",
45
+ "mitmproxy.log",
46
+ "chromedriver.log",
47
+ "report.html",
48
+ "report-standalone.html",
49
+ )
42
50
  TEST_KEY = pytest.StashKey[str]()
43
51
  NEEDS_TEST_IME = "needs a test IME: pass --test-ime <bundle>"
44
52
 
@@ -108,6 +116,11 @@ def pytest_addoption(parser: pytest.Parser) -> None:
108
116
  group.addoption(
109
117
  "--update-references", action="store_true", help="render the references anew even where they are stored"
110
118
  )
119
+ group.addoption(
120
+ "--standalone-report",
121
+ action="store_true",
122
+ help="also write the page scenario report with its images inlined, one file to share: report-standalone.html",
123
+ )
111
124
  group.addoption("--chromedriver", type=Path, help="chromedriver matching ArkWeb's Chromium, for the arkweb engine")
112
125
 
113
126
 
@@ -309,7 +322,7 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int | pytest.ExitC
309
322
  path.write_text(run.report.to_json(), encoding="utf-8")
310
323
  from .report import write_report
311
324
 
312
- run.page_report = write_report(run.artifacts, path)
325
+ run.page_report = write_report(run.artifacts, path, config.option.standalone_report)
313
326
  if summary := os.environ.get("GITHUB_STEP_SUMMARY"):
314
327
  with open(summary, "a", encoding="utf-8") as f:
315
328
  f.write(_markdown_summary(run.report))
@@ -469,6 +482,9 @@ def pytest_terminal_summary(terminalreporter: pytest.TerminalReporter, config: p
469
482
  terminalreporter.line("no unexpected results")
470
483
  if run.page_report is not None:
471
484
  terminalreporter.line(f"page scenario report: {run.page_report}")
485
+ standalone = run.page_report.with_name("report-standalone.html")
486
+ if config.option.standalone_report and standalone.is_file():
487
+ terminalreporter.line(f"as one file to share: {standalone}")
472
488
  if failed := _failed_comparisons(run.report):
473
489
  terminalreporter.line(f"{failed} page scenario comparison(s) failed; to triage them, read {SKILL / 'SKILL.md'}")
474
490