servo-embedder-tests 0.1.2__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/PKG-INFO +8 -1
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/README.md +4 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/pyproject.toml +4 -1
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/pyproject.toml.orig +4 -1
- servo_embedder_tests-0.2.0/src/servo_embedder_tests/describe.py +11 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/diffing.py +67 -11
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/engines.py +1 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/page_scenarios.py +173 -29
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/plugin.py +30 -5
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/report.py +2 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/scenarios/page_scenarios/test_pages.py +4 -1
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/servoshell.py +8 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/skills/compare-renderings/SKILL.md +2 -1
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/webdriver.py +7 -3
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/LICENSE-APACHE +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/LICENSE-MIT +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/__init__.py +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/__main__.py +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/cli.py +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/errors.py +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/expectations.py +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/hookspecs.py +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/oracles.py +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/pages/basic.html +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/pages/history-a.html +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/pages/history-b.html +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/pages/ime.html +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/pages/report.js +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/pages/svg/10_coordinate_transforms.html +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/pages/svg/11_symbols_and_use.html +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/pages/svg/12_basic_text.html +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/pages/svg/13_text_paths.html +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/pages/svg/14_clipping_paths.html +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/pages/svg/15_masking.html +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/pages/svg/16_filter_effects_basic.html +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/pages/svg/1_basic_shapes.html +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/pages/svg/2_path_commands.html +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/pages/svg/3_path_curves.html +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/pages/svg/4_stroke_properties.html +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/pages/svg/5_linear_gradients.html +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/pages/svg/6_radial_gradients.html +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/pages/svg/7_patterns.html +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/pages/svg/8_markers.html +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/pages/svg/9_viewbox_scaling.html +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/pages/svg/LICENSE +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/pages/svg/README.md +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/pages/svg/security/layer1_xml_validation.html +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/pages/svg/security/layer2_css_injection.html +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/pages/svg/security/layer2_import_fetch.html +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/pages/svg/security/layer3_fetch_allowlist.html +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/pages/svg/security/layer4_expansion_limits.html +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/pages/svg/security/layer4_use_self_cycle.html +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/pages/svg/security/layer5_mask_recursion.html +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/pages/svg/security/layer5_render_safety.html +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/pages/webview_apps/scroll.html +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/pages.py +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/platform/__init__.py +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/platform/desktop.py +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/platform/ohos.py +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/proxy.py +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/scenarios/__init__.py +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/scenarios/ime/__init__.py +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/scenarios/ime/test_text_input.py +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/scenarios/navigation/__init__.py +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/scenarios/navigation/test_local_pages.py +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/scenarios/page_scenarios/__init__.py +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/scenarios/sites/__init__.py +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/scenarios/sites/mossel.py +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/scenarios/sites/test_mossel.py +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/scenarios/sites/test_servo_org.py +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/scenarios/svg/__init__.py +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/scenarios/svg/test_security.py +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/scenarios/webview_apps/__init__.py +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/scenarios/webview_apps/apps/arkweb-test.toml +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/scenarios/webview_apps/flows.py +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/scenarios/webview_apps/test_apps.py +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/screenshots.py +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/skills/compare-renderings/describe_regions.py +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/svg_cards.py +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/touch.py +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/ui.py +0 -0
- {servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/webview_app.py +0 -0
|
@@ -1,16 +1,19 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: servo-embedder-tests
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Embedder tests for Servo: what servoshell and its users do beyond WPT, on OpenHarmony, HarmonyOS and desktop
|
|
5
5
|
License-Expression: Apache-2.0 OR MIT
|
|
6
6
|
License-File: LICENSE-APACHE
|
|
7
7
|
License-File: LICENSE-MIT
|
|
8
8
|
Requires-Dist: hdc-py>=0.2.1,<0.3
|
|
9
|
+
Requires-Dist: numpy>=1.26
|
|
9
10
|
Requires-Dist: ohos-test-ime>=0.1,<0.2
|
|
11
|
+
Requires-Dist: opencv-python-headless>=4.10.0.84
|
|
10
12
|
Requires-Dist: pillow>=11
|
|
11
13
|
Requires-Dist: pytest>=8.3
|
|
12
14
|
Requires-Dist: selenium>=4.30
|
|
13
15
|
Requires-Dist: tomlkit>=0.13
|
|
16
|
+
Requires-Dist: urllib3>=2
|
|
14
17
|
Requires-Python: >=3.11
|
|
15
18
|
Project-URL: Repository, https://github.com/openharmony-rs/servo-embedder-tests
|
|
16
19
|
Description-Content-Type: text/markdown
|
|
@@ -201,6 +204,10 @@ It also writes `<capture>-compare.png` (expected, actual and the differing pixel
|
|
|
201
204
|
for what the readings leave open: the `compare-renderings` skill describes how an agent triages a run, and its
|
|
202
205
|
script asks any OpenAI-compatible endpoint about the regions.
|
|
203
206
|
|
|
207
|
+
A comparison test passes or fails as soon as the tiles are compared. Describing the differences, the elements,
|
|
208
|
+
readings, DOM changes and images, takes seconds per capture, so it runs in up to four background processes while the
|
|
209
|
+
run goes on to the next test; `diff.json` and the images are complete when the run ends.
|
|
210
|
+
|
|
204
211
|
For people, a run writes `report.html` to the artifacts folder: per scenario, each differing capture next to what it
|
|
205
212
|
was compared with (hover the actual one to see the expected one in its place), the differing pixels, the regions
|
|
206
213
|
numbered with their enlarged crops and readings, and the DOM changes. Where an agent triaged the run, the report also
|
|
@@ -184,6 +184,10 @@ It also writes `<capture>-compare.png` (expected, actual and the differing pixel
|
|
|
184
184
|
for what the readings leave open: the `compare-renderings` skill describes how an agent triages a run, and its
|
|
185
185
|
script asks any OpenAI-compatible endpoint about the regions.
|
|
186
186
|
|
|
187
|
+
A comparison test passes or fails as soon as the tiles are compared. Describing the differences, the elements,
|
|
188
|
+
readings, DOM changes and images, takes seconds per capture, so it runs in up to four background processes while the
|
|
189
|
+
run goes on to the next test; `diff.json` and the images are complete when the run ends.
|
|
190
|
+
|
|
187
191
|
For people, a run writes `report.html` to the artifacts folder: per scenario, each differing capture next to what it
|
|
188
192
|
was compared with (hover the actual one to see the expected one in its place), the differing pixels, the regions
|
|
189
193
|
numbered with their enlarged crops and readings, and the DOM changes. Where an agent triaged the run, the report also
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "servo-embedder-tests"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.2.0"
|
|
4
4
|
description = "Embedder tests for Servo: what servoshell and its users do beyond WPT, on OpenHarmony, HarmonyOS and desktop"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.11"
|
|
@@ -11,11 +11,14 @@ license-files = [
|
|
|
11
11
|
]
|
|
12
12
|
dependencies = [
|
|
13
13
|
"hdc-py>=0.2.1,<0.3",
|
|
14
|
+
"numpy>=1.26",
|
|
14
15
|
"ohos-test-ime>=0.1,<0.2",
|
|
16
|
+
"opencv-python-headless>=4.10.0.84",
|
|
15
17
|
"pillow>=11",
|
|
16
18
|
"pytest>=8.3",
|
|
17
19
|
"selenium>=4.30",
|
|
18
20
|
"tomlkit>=0.13",
|
|
21
|
+
"urllib3>=2",
|
|
19
22
|
]
|
|
20
23
|
|
|
21
24
|
[project.urls]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "servo-embedder-tests"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.2.0"
|
|
4
4
|
description = "Embedder tests for Servo: what servoshell and its users do beyond WPT, on OpenHarmony, HarmonyOS and desktop"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.11"
|
|
@@ -8,11 +8,14 @@ license = "Apache-2.0 OR MIT"
|
|
|
8
8
|
license-files = ["LICENSE-APACHE", "LICENSE-MIT"]
|
|
9
9
|
dependencies = [
|
|
10
10
|
"hdc-py>=0.2.1,<0.3",
|
|
11
|
+
"numpy>=1.26",
|
|
11
12
|
"ohos-test-ime>=0.1,<0.2",
|
|
13
|
+
"opencv-python-headless>=4.10.0.84",
|
|
12
14
|
"pillow>=11",
|
|
13
15
|
"pytest>=8.3",
|
|
14
16
|
"selenium>=4.30",
|
|
15
17
|
"tomlkit>=0.13",
|
|
18
|
+
"urllib3>=2",
|
|
16
19
|
]
|
|
17
20
|
|
|
18
21
|
[project.urls]
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""Describes the differences of a failed page scenario comparison: runs the `page_scenarios.Description` pickled on
|
|
2
|
+
stdin, at a lower priority, for a `page_scenarios.Describer`."""
|
|
3
|
+
|
|
4
|
+
import os
|
|
5
|
+
import pickle
|
|
6
|
+
import sys
|
|
7
|
+
|
|
8
|
+
if __name__ == "__main__":
|
|
9
|
+
if hasattr(os, "nice"):
|
|
10
|
+
os.nice(10)
|
|
11
|
+
pickle.load(sys.stdin.buffer).run()
|
{servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/diffing.py
RENAMED
|
@@ -32,6 +32,8 @@ MAX_OFFSET = 6
|
|
|
32
32
|
BACKGROUND_RADIUS = 10
|
|
33
33
|
# A pixel is ink when its luminance differs from the background by more than this (0-255).
|
|
34
34
|
INK_CONTRAST = 48
|
|
35
|
+
# Backgrounds closer than this in luminance (0-255) are the same; see `_inks`.
|
|
36
|
+
BACKGROUND_TOLERANCE = 24
|
|
35
37
|
# Strips, in CSS px, that are aligned one by one to tell letter spacing from other glyphs, and how far, in CSS px, each
|
|
36
38
|
# looks around the previous one's offset.
|
|
37
39
|
STRIP = 24
|
|
@@ -447,11 +449,67 @@ def _best_offset(a: Image.Image, b: Image.Image, search: int, steps: int = 4) ->
|
|
|
447
449
|
return fine[1] / steps, fine[2] / steps, fine[0], unshifted
|
|
448
450
|
|
|
449
451
|
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
452
|
+
# Pillow's MedianFilter, MinFilter and MaxFilter of non-empty "L" images with odd sizes, with the same results, by
|
|
453
|
+
# OpenCV: Pillow's take time that grows with the square of `size`, for a median of 31 px about 200 times as long.
|
|
454
|
+
# OpenCV is imported only here, so that a broken installation fails the descriptions that need it, not every run.
|
|
455
|
+
def _median(image: Image.Image, size: int) -> Image.Image:
|
|
456
|
+
import cv2
|
|
457
|
+
import numpy as np
|
|
458
|
+
|
|
459
|
+
return Image.fromarray(cv2.medianBlur(np.asarray(image), size))
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def _minimum(image: Image.Image, size: int) -> Image.Image:
|
|
463
|
+
import cv2
|
|
464
|
+
import numpy as np
|
|
465
|
+
|
|
466
|
+
kernel = np.ones((size, size), np.uint8)
|
|
467
|
+
return Image.fromarray(cv2.erode(np.asarray(image), kernel, borderType=cv2.BORDER_REPLICATE))
|
|
468
|
+
|
|
469
|
+
|
|
470
|
+
def _maximum(image: Image.Image, size: int) -> Image.Image:
|
|
471
|
+
import cv2
|
|
472
|
+
import numpy as np
|
|
473
|
+
|
|
474
|
+
kernel = np.ones((size, size), np.uint8)
|
|
475
|
+
return Image.fromarray(cv2.dilate(np.asarray(image), kernel, borderType=cv2.BORDER_REPLICATE))
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
def _inks(a: Image.Image, b: Image.Image, radius: int) -> tuple[tuple[int, int], tuple[int, int]]:
|
|
479
|
+
"""`_ink` of the grayscale images `a` and `b` against their backgrounds, the median over a window wider than
|
|
480
|
+
strokes, which keeps gradients but not text. Content wider than half the window, such as a thick stroke, is its
|
|
481
|
+
image's median too: where only the other image's median is the background both mostly show, over an area at least
|
|
482
|
+
`radius` across, that one is the background of both, so that the content counts as ink. Narrower areas are edges
|
|
483
|
+
that moved, or the median's skew at the image's border. When their backgrounds differ, each keeps its own."""
|
|
484
|
+
size = 2 * radius + 1
|
|
485
|
+
median_a, median_b = _median(a, size), _median(b, size)
|
|
486
|
+
histogram_a, histogram_b = a.histogram(), b.histogram()
|
|
487
|
+
mode_a = max(range(256), key=histogram_a.__getitem__)
|
|
488
|
+
mode_b = max(range(256), key=histogram_b.__getitem__)
|
|
489
|
+
if abs(mode_a - mode_b) > BACKGROUND_TOLERANCE:
|
|
490
|
+
return _ink(a, median_a), _ink(b, median_b)
|
|
491
|
+
shown = max(range(256), key=lambda value: histogram_a[value] + histogram_b[value])
|
|
492
|
+
page = Image.new("L", a.size, shown)
|
|
493
|
+
|
|
494
|
+
def near(median: Image.Image) -> Image.Image:
|
|
495
|
+
return ImageChops.difference(median, page).point(lambda value: 255 if value <= BACKGROUND_TOLERANCE else 0)
|
|
496
|
+
|
|
497
|
+
def background(own: Image.Image, other: Image.Image) -> Image.Image:
|
|
498
|
+
"""`own`, except where only `other` is the page background, over an area at least `radius` across."""
|
|
499
|
+
only_other = ImageChops.subtract(near(other), near(own))
|
|
500
|
+
if only_other.getbbox() is None:
|
|
501
|
+
return own
|
|
502
|
+
width = radius | 1
|
|
503
|
+
wide = _maximum(_minimum(only_other, width), width)
|
|
504
|
+
return Image.composite(other, own, wide)
|
|
505
|
+
|
|
506
|
+
return _ink(a, background(median_a, median_b)), _ink(b, background(median_b, median_a))
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
def _ink(gray: Image.Image, background: Image.Image) -> tuple[int, int]:
|
|
510
|
+
"""How much the image departs from `background`, and the horizontal extent of the ink in pixels, between the 2nd
|
|
511
|
+
and the 98th percentile of its columns, so that stray specks of a textured background don't count."""
|
|
512
|
+
contrast = ImageChops.difference(gray, background)
|
|
455
513
|
energy = sum(value * count for value, count in enumerate(contrast.histogram()))
|
|
456
514
|
mask = contrast.point(lambda value: 255 if value > INK_CONTRAST else 0)
|
|
457
515
|
columns = list(mask.resize((mask.width, 1), Image.Resampling.BOX).tobytes())
|
|
@@ -503,9 +561,7 @@ def _on_shared_edges(a: Image.Image, b: Image.Image) -> float | None:
|
|
|
503
561
|
have. None if none differ."""
|
|
504
562
|
differing = ImageChops.difference(a, b).point(lambda value: 255 if value > 4 else 0)
|
|
505
563
|
edges = [
|
|
506
|
-
image.filter(ImageFilter.FIND_EDGES)
|
|
507
|
-
.point(lambda value: 255 if value > EDGE_CONTRAST else 0)
|
|
508
|
-
.filter(ImageFilter.MaxFilter(3))
|
|
564
|
+
_maximum(image.filter(ImageFilter.FIND_EDGES).point(lambda value: 255 if value > EDGE_CONTRAST else 0), 3)
|
|
509
565
|
for image in (a, b)
|
|
510
566
|
]
|
|
511
567
|
total = differing.histogram()[255]
|
|
@@ -554,12 +610,12 @@ def _colours_lost(a: Image.Image, b: Image.Image, scale: float) -> list[tuple[in
|
|
|
554
610
|
if mask.histogram()[255] < floor:
|
|
555
611
|
continue
|
|
556
612
|
similar = ImageChops.lighter(ImageChops.lighter(masks_b[k - 1], masks_b[k]), masks_b[(k + 1) % bins])
|
|
557
|
-
gone = ImageChops.multiply(mask, ImageChops.invert(similar
|
|
613
|
+
gone = ImageChops.multiply(mask, ImageChops.invert(_maximum(similar, reach)))
|
|
558
614
|
count = gone.histogram()[255]
|
|
559
615
|
if count < floor:
|
|
560
616
|
continue
|
|
561
617
|
was, now = ImageStat.Stat(a, gone).mean, ImageStat.Stat(b, gone).mean
|
|
562
|
-
around = ImageChops.subtract(ImageChops.subtract(gone
|
|
618
|
+
around = ImageChops.subtract(ImageChops.subtract(_maximum(gone, reach), gone), mask)
|
|
563
619
|
background = ImageStat.Stat(a, around).median if around.getbbox() else ImageStat.Stat(a).median
|
|
564
620
|
apart = max(abs(n - g) for n, g in zip(now, background, strict=True))
|
|
565
621
|
if colour_name(was) != colour_name(now) and apart > 48:
|
|
@@ -592,7 +648,7 @@ def measure(expected: Shot, actual: Shot, diff: Diff, region: Region) -> Measure
|
|
|
592
648
|
gray_a, gray_b = a.crop(box).convert("L"), b.crop(box).convert("L")
|
|
593
649
|
dx, dy, residual, unshifted = _best_offset(gray_a, gray_b, round(MAX_OFFSET * scale))
|
|
594
650
|
radius = round(BACKGROUND_RADIUS * scale)
|
|
595
|
-
(ink_a, extent_a), (ink_b, extent_b) =
|
|
651
|
+
(ink_a, extent_a), (ink_b, extent_b) = _inks(gray_a, gray_b, radius)
|
|
596
652
|
return Measurement(
|
|
597
653
|
(round(dx / scale, 2), round(dy / scale, 2)),
|
|
598
654
|
round(residual, 1),
|
{servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/engines.py
RENAMED
|
@@ -254,6 +254,7 @@ class ArkWebEngine(Engine):
|
|
|
254
254
|
raise ScenarioError(f"no chromedriver for ArkWeb's Chromium {version}: {hint}")
|
|
255
255
|
options = webdriver.ChromeOptions()
|
|
256
256
|
options.debugger_address = f"127.0.0.1:{self.port}"
|
|
257
|
+
options.ignore_local_proxy_environment_variables()
|
|
257
258
|
try:
|
|
258
259
|
return webdriver.Chrome(service=Service(str(executable), log_output=str(self.log)), options=options)
|
|
259
260
|
except SessionNotCreatedException as error:
|
{servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/page_scenarios.py
RENAMED
|
@@ -13,12 +13,20 @@ import contextlib
|
|
|
13
13
|
import copy
|
|
14
14
|
import datetime
|
|
15
15
|
import json
|
|
16
|
+
import os
|
|
17
|
+
import pickle
|
|
16
18
|
import shutil
|
|
19
|
+
import subprocess
|
|
20
|
+
import sys
|
|
21
|
+
import threading
|
|
17
22
|
import time
|
|
18
23
|
import tomllib
|
|
24
|
+
import warnings
|
|
19
25
|
from collections.abc import Callable, Mapping
|
|
26
|
+
from concurrent.futures import CancelledError, Future, ThreadPoolExecutor
|
|
20
27
|
from dataclasses import asdict, dataclass, field
|
|
21
28
|
from pathlib import Path
|
|
29
|
+
from typing import cast
|
|
22
30
|
|
|
23
31
|
from PIL import Image, ImageChops, ImageDraw
|
|
24
32
|
from selenium.common.exceptions import TimeoutException
|
|
@@ -36,6 +44,10 @@ from .pages import PageServer
|
|
|
36
44
|
|
|
37
45
|
SUITE_CONFIG = "page-scenarios.toml"
|
|
38
46
|
SCENARIO_CONFIG = "scenario.toml"
|
|
47
|
+
# How many processes at most describe the differences of failed comparisons at a time, and how many seconds each may
|
|
48
|
+
# take.
|
|
49
|
+
DESCRIBERS = 4
|
|
50
|
+
DESCRIPTION_TIMEOUT = 300
|
|
39
51
|
# The configuration keys, with their defaults.
|
|
40
52
|
DEFAULTS: dict[str, object] = {
|
|
41
53
|
# The page to open, relative to the scenario's folder.
|
|
@@ -752,6 +764,144 @@ class Comparison:
|
|
|
752
764
|
return f"{len(self.failures)} capture(s) differ from {what}: " + "; ".join(self.failures) + "; see diff.json"
|
|
753
765
|
|
|
754
766
|
|
|
767
|
+
def _overlay(directory: Path, name: str) -> Path:
|
|
768
|
+
return directory / f"{name}-differing.png"
|
|
769
|
+
|
|
770
|
+
|
|
771
|
+
@dataclass
|
|
772
|
+
class Description:
|
|
773
|
+
"""What describing the differences of a failed comparison takes: `report`, its `diff.json` so far, and the
|
|
774
|
+
differing captures, by name, with what they were compared with and their `Diff`."""
|
|
775
|
+
|
|
776
|
+
report: dict[str, object]
|
|
777
|
+
differing: list[tuple[str, Capture, Capture, diffing.Diff]]
|
|
778
|
+
directory: Path
|
|
779
|
+
labels: tuple[str, str]
|
|
780
|
+
reference: dict[str, Capture] | None
|
|
781
|
+
|
|
782
|
+
def run(self) -> None:
|
|
783
|
+
"""Measures the differing regions, writes the review images to `directory` and adds both, with the DOM
|
|
784
|
+
changes, to the report, then writes it as `diff.json`."""
|
|
785
|
+
captures = cast(dict[str, dict[str, object]], self.report["captures"])
|
|
786
|
+
for name, stored, actual, diff in self.differing:
|
|
787
|
+
elements = actual.elements()
|
|
788
|
+
broken = [element for element in elements if element.get("image") == "broken"]
|
|
789
|
+
for region in diff.regions:
|
|
790
|
+
region.elements = diffing.elements_in(elements, region.box)
|
|
791
|
+
for region in diff.regions[: diffing.MAX_CROPS]:
|
|
792
|
+
measurement = diffing.measure(stored.shot, actual.shot, diff, region)
|
|
793
|
+
region.measured = asdict(measurement)
|
|
794
|
+
region.reading = measurement.reading(text=any(element.get("text") for element in region.elements))
|
|
795
|
+
missing = sorted({str(element["src"]) for element in diffing.elements_in(broken, region.box, 3)})
|
|
796
|
+
if missing:
|
|
797
|
+
region.reading = f"image {', '.join(missing)} did not load in {self.labels[1]}; {region.reading}"
|
|
798
|
+
if self.reference and name in self.reference:
|
|
799
|
+
diffing.reference_hint(stored.shot, actual.shot, self.reference[name].shot, diff.regions)
|
|
800
|
+
overlay = _overlay(self.directory, name)
|
|
801
|
+
images = diffing.write_review_images(
|
|
802
|
+
stored.shot, actual.shot, diff, overlay, self.directory, name, self.labels
|
|
803
|
+
)
|
|
804
|
+
captures[name].update(
|
|
805
|
+
diff.to_dict(),
|
|
806
|
+
compare=images["compare"],
|
|
807
|
+
images=images,
|
|
808
|
+
dom_changes=diffing.dom_changes(
|
|
809
|
+
_in_document(stored), _in_document(actual), paint=min(stored.dom_version, actual.dom_version) >= 2
|
|
810
|
+
),
|
|
811
|
+
)
|
|
812
|
+
diffing.write_json(self.directory / "diff.json", self.report)
|
|
813
|
+
|
|
814
|
+
|
|
815
|
+
class Describer:
|
|
816
|
+
"""Runs `Description`s in processes of their own at a lower priority, a few at a time, so that a run goes on to
|
|
817
|
+
the next capture meanwhile. Not a multiprocessing pool, whose workers would import the main module of whoever
|
|
818
|
+
called `run_tests` again."""
|
|
819
|
+
|
|
820
|
+
def __init__(self, workers: int = DESCRIBERS, timeout: float = DESCRIPTION_TIMEOUT) -> None:
|
|
821
|
+
self.timeout = timeout
|
|
822
|
+
self._pool = ThreadPoolExecutor(max(1, min(workers, os.cpu_count() or 1)), "describer")
|
|
823
|
+
self._pending: list[tuple[Path, Future[str | None]]] = []
|
|
824
|
+
self._lock = threading.Lock()
|
|
825
|
+
self._processes: set[subprocess.Popen[bytes]] = set()
|
|
826
|
+
self._stopped = False
|
|
827
|
+
|
|
828
|
+
def submit(self, description: Description) -> None:
|
|
829
|
+
future = self._pool.submit(self._run, pickle.dumps(description))
|
|
830
|
+
self._pending.append((description.directory / "diff.json", future))
|
|
831
|
+
|
|
832
|
+
def _run(self, payload: bytes) -> str | None:
|
|
833
|
+
"""Describes in a process of its own, which imports this package from where this process does. Returns why
|
|
834
|
+
that failed, if it did."""
|
|
835
|
+
with self._lock:
|
|
836
|
+
if self._stopped:
|
|
837
|
+
return "stopped before it started"
|
|
838
|
+
process = subprocess.Popen(
|
|
839
|
+
[sys.executable, "-P", "-m", "servo_embedder_tests.describe"],
|
|
840
|
+
stdin=subprocess.PIPE,
|
|
841
|
+
stdout=subprocess.DEVNULL,
|
|
842
|
+
stderr=subprocess.PIPE,
|
|
843
|
+
env={**os.environ, "PYTHONPATH": os.pathsep.join(sys.path)},
|
|
844
|
+
creationflags=getattr(subprocess, "BELOW_NORMAL_PRIORITY_CLASS", 0),
|
|
845
|
+
)
|
|
846
|
+
self._processes.add(process)
|
|
847
|
+
try:
|
|
848
|
+
_, stderr = process.communicate(payload, timeout=self.timeout)
|
|
849
|
+
except subprocess.TimeoutExpired:
|
|
850
|
+
process.kill()
|
|
851
|
+
process.communicate()
|
|
852
|
+
return f"took longer than {self.timeout:g}s"
|
|
853
|
+
finally:
|
|
854
|
+
with self._lock:
|
|
855
|
+
self._processes.discard(process)
|
|
856
|
+
if process.returncode == 0:
|
|
857
|
+
return None
|
|
858
|
+
if self._stopped:
|
|
859
|
+
return "stopped"
|
|
860
|
+
lines = stderr.decode(errors="replace").strip().splitlines()
|
|
861
|
+
return lines[-1] if lines else f"exit status {process.returncode}"
|
|
862
|
+
|
|
863
|
+
def stop(self) -> None:
|
|
864
|
+
"""Cancels the descriptions that haven't started and ends those that run."""
|
|
865
|
+
with self._lock:
|
|
866
|
+
self._stopped = True
|
|
867
|
+
for process in self._processes:
|
|
868
|
+
process.kill()
|
|
869
|
+
self._pool.shutdown(wait=False, cancel_futures=True)
|
|
870
|
+
|
|
871
|
+
def wait(self) -> None:
|
|
872
|
+
"""Waits for the descriptions, or stops them when interrupted. One that failed or was stopped leaves its
|
|
873
|
+
`diff.json` without it, but with why as `description_error`; failures end in one warning."""
|
|
874
|
+
pending, self._pending = self._pending, []
|
|
875
|
+
try:
|
|
876
|
+
failures = self._record(pending)
|
|
877
|
+
except BaseException:
|
|
878
|
+
self.stop()
|
|
879
|
+
self._record(pending)
|
|
880
|
+
raise
|
|
881
|
+
if failures:
|
|
882
|
+
warnings.warn("describing the differences failed: " + "; ".join(failures), stacklevel=2)
|
|
883
|
+
|
|
884
|
+
@staticmethod
|
|
885
|
+
def _record(pending: list[tuple[Path, Future[str | None]]]) -> list[str]:
|
|
886
|
+
"""Waits for `pending` and notes each failure in its `diff.json`; returns the failures."""
|
|
887
|
+
failures = []
|
|
888
|
+
for path, future in pending:
|
|
889
|
+
try:
|
|
890
|
+
failure = future.result()
|
|
891
|
+
except CancelledError:
|
|
892
|
+
failure = "stopped before it started"
|
|
893
|
+
except Exception as error:
|
|
894
|
+
failure = f"{type(error).__name__}: {error}"
|
|
895
|
+
if failure is None:
|
|
896
|
+
continue
|
|
897
|
+
with contextlib.suppress(OSError, ValueError):
|
|
898
|
+
report = json.loads(path.read_text(encoding="utf-8"))
|
|
899
|
+
report["description_error"] = failure
|
|
900
|
+
diffing.write_json(path, report)
|
|
901
|
+
failures.append(f"{path}: {failure}")
|
|
902
|
+
return failures
|
|
903
|
+
|
|
904
|
+
|
|
755
905
|
def compare_captures(
|
|
756
906
|
expected: dict[str, Capture],
|
|
757
907
|
rendering: Rendering,
|
|
@@ -760,13 +910,16 @@ def compare_captures(
|
|
|
760
910
|
labels: tuple[str, str],
|
|
761
911
|
reference: dict[str, Capture] | None = None,
|
|
762
912
|
describe: Callable[[], dict[str, object]] | None = None,
|
|
913
|
+
describer: Describer | None = None,
|
|
763
914
|
) -> Comparison:
|
|
764
|
-
"""Compares the captures of `rendering` with `expected`, those with the same names, and writes `diff.json`
|
|
765
|
-
|
|
766
|
-
|
|
915
|
+
"""Compares the captures of `rendering` with `expected`, those with the same names, and writes `diff.json` to
|
|
916
|
+
`directory`. The `Description` of differing captures adds their measurements and review images to it, before this
|
|
917
|
+
returns or, with a `describer`, by that while the run goes on. `reference` captures, from another engine, add a
|
|
918
|
+
hint per differing region whether the change moved toward them."""
|
|
767
919
|
directory.mkdir(parents=True, exist_ok=True)
|
|
768
920
|
failures = []
|
|
769
|
-
captures: dict[str, object] = {}
|
|
921
|
+
captures: dict[str, dict[str, object]] = {}
|
|
922
|
+
differing: list[tuple[str, Capture, Capture, diffing.Diff]] = []
|
|
770
923
|
for name in sorted(set(expected) - set(rendering.captures)):
|
|
771
924
|
failures.append(f"{name}: not captured")
|
|
772
925
|
captures[name] = {"matches": False, "summary": "not captured this time"}
|
|
@@ -780,42 +933,29 @@ def compare_captures(
|
|
|
780
933
|
"actual": str(actual.image),
|
|
781
934
|
}
|
|
782
935
|
continue
|
|
783
|
-
overlay = directory
|
|
936
|
+
overlay = _overlay(directory, name)
|
|
784
937
|
try:
|
|
785
938
|
diff = diffing.compare(stored.shot, actual.shot, preset, overlay=overlay)
|
|
786
939
|
except ValueError as error:
|
|
787
940
|
failures.append(f"{name}: {error}")
|
|
788
941
|
captures[name] = {"matches": False, "summary": str(error), "actual": str(actual.image)}
|
|
789
942
|
continue
|
|
790
|
-
|
|
791
|
-
if not diff.matches:
|
|
792
|
-
failures.append(f"{name}: {diff.summary()}")
|
|
793
|
-
elements = actual.elements()
|
|
794
|
-
broken = [element for element in elements if element.get("image") == "broken"]
|
|
795
|
-
for region in diff.regions:
|
|
796
|
-
region.elements = diffing.elements_in(elements, region.box)
|
|
797
|
-
for region in diff.regions[: diffing.MAX_CROPS]:
|
|
798
|
-
measurement = diffing.measure(stored.shot, actual.shot, diff, region)
|
|
799
|
-
region.measured = asdict(measurement)
|
|
800
|
-
region.reading = measurement.reading(text=any(element.get("text") for element in region.elements))
|
|
801
|
-
missing = sorted({str(element["src"]) for element in diffing.elements_in(broken, region.box, 3)})
|
|
802
|
-
if missing:
|
|
803
|
-
region.reading = f"image {', '.join(missing)} did not load in {labels[1]}; {region.reading}"
|
|
804
|
-
if reference and name in reference:
|
|
805
|
-
diffing.reference_hint(stored.shot, actual.shot, reference[name].shot, diff.regions)
|
|
806
|
-
images = diffing.write_review_images(stored.shot, actual.shot, diff, overlay, directory, name, labels)
|
|
807
|
-
entry["compare"] = images["compare"]
|
|
808
|
-
entry["images"] = images
|
|
809
|
-
entry["dom_changes"] = diffing.dom_changes(
|
|
810
|
-
_in_document(stored), _in_document(actual), paint=min(stored.dom_version, actual.dom_version) >= 2
|
|
811
|
-
)
|
|
812
|
-
else:
|
|
943
|
+
if diff.matches:
|
|
813
944
|
overlay.unlink(missing_ok=True)
|
|
814
|
-
|
|
945
|
+
else:
|
|
946
|
+
failures.append(f"{name}: {diff.summary()}")
|
|
947
|
+
differing.append((name, stored, actual, diff))
|
|
948
|
+
captures[name] = {**diff.to_dict(), "expected": str(stored.image), "actual": str(actual.image)}
|
|
815
949
|
report: dict[str, object] = {"labels": list(labels), "captures": captures}
|
|
816
950
|
if describe is not None:
|
|
817
951
|
report.update(describe())
|
|
818
952
|
diffing.write_json(directory / "diff.json", report)
|
|
953
|
+
if differing:
|
|
954
|
+
description = Description(report, differing, directory, labels, reference)
|
|
955
|
+
if describer is None:
|
|
956
|
+
description.run()
|
|
957
|
+
else:
|
|
958
|
+
describer.submit(description)
|
|
819
959
|
return Comparison(not failures, failures, report)
|
|
820
960
|
|
|
821
961
|
|
|
@@ -834,6 +974,7 @@ class PageRuns:
|
|
|
834
974
|
update_baselines: bool = False,
|
|
835
975
|
update_references: bool = False,
|
|
836
976
|
versions: Mapping[str, str] | None = None,
|
|
977
|
+
describer: Describer | None = None,
|
|
837
978
|
) -> None:
|
|
838
979
|
"""`versions` names the builds of engines, e.g. a commit, for whoever compares their renderings."""
|
|
839
980
|
if engine not in engines:
|
|
@@ -848,6 +989,7 @@ class PageRuns:
|
|
|
848
989
|
self.update_baselines = update_baselines
|
|
849
990
|
self.update_references = update_references
|
|
850
991
|
self.versions = dict(versions or {})
|
|
992
|
+
self.describer = describer or Describer()
|
|
851
993
|
self._renderings: dict[tuple[str, str], Rendering] = {}
|
|
852
994
|
|
|
853
995
|
def rendering(self, engine: str, scenario: Scenario) -> Rendering:
|
|
@@ -891,6 +1033,8 @@ class PageRuns:
|
|
|
891
1033
|
return rendering.captures, {"engine": rendering.info.get("engine"), "from": "rendered in this run"}
|
|
892
1034
|
|
|
893
1035
|
def stop(self) -> None:
|
|
1036
|
+
"""Stops the engines, then waits for the descriptions of differences."""
|
|
894
1037
|
for engine in self.engines.values():
|
|
895
1038
|
with contextlib.suppress(Exception):
|
|
896
1039
|
engine.stop()
|
|
1040
|
+
self.describer.wait()
|
{servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/plugin.py
RENAMED
|
@@ -14,6 +14,7 @@ from importlib import resources
|
|
|
14
14
|
from pathlib import Path
|
|
15
15
|
|
|
16
16
|
import pytest
|
|
17
|
+
import urllib3
|
|
17
18
|
from ohos_test_ime import TestIme
|
|
18
19
|
from selenium.common.exceptions import TimeoutException
|
|
19
20
|
from selenium.webdriver.remote.webdriver import WebDriver
|
|
@@ -21,7 +22,7 @@ from selenium.webdriver.remote.webdriver import WebDriver
|
|
|
21
22
|
from .engines import ArkWebEngine, Engine, ServoshellEngine
|
|
22
23
|
from .errors import AppCrashed, ScenarioError, ScenarioTimeout
|
|
23
24
|
from .expectations import Expectations, Result, RunReport
|
|
24
|
-
from .page_scenarios import PageRuns, Renderer
|
|
25
|
+
from .page_scenarios import Describer, PageRuns, Renderer
|
|
25
26
|
from .pages import PageServer
|
|
26
27
|
from .platform import DesktopHost, HilogStream, OhosDevice, Platform
|
|
27
28
|
from .proxy import MitmProxy, ProxyMode
|
|
@@ -130,6 +131,12 @@ def pytest_configure(config: pytest.Config) -> None:
|
|
|
130
131
|
raise pytest.UsageError("--reference-engine-version needs --reference-engine")
|
|
131
132
|
if getattr(config.option, "platform", None) is None or config.option.collectonly:
|
|
132
133
|
return
|
|
134
|
+
expectations = config.option.expectations
|
|
135
|
+
if expectations is not None and not expectations.is_file():
|
|
136
|
+
raise pytest.UsageError(
|
|
137
|
+
f"--expectations {expectations} does not exist; leave the option out to expect every test to pass, or "
|
|
138
|
+
"write the file with `servo-embedder-tests update-expectations`"
|
|
139
|
+
)
|
|
133
140
|
config.stash[RUN_KEY] = ScenarioRun(config)
|
|
134
141
|
|
|
135
142
|
|
|
@@ -142,6 +149,7 @@ class ScenarioRun:
|
|
|
142
149
|
report: RunReport = field(init=False)
|
|
143
150
|
stack: contextlib.ExitStack = field(default_factory=contextlib.ExitStack)
|
|
144
151
|
page_report: Path | None = None
|
|
152
|
+
describer: Describer = field(default_factory=Describer)
|
|
145
153
|
|
|
146
154
|
def __post_init__(self) -> None:
|
|
147
155
|
self.expectations = Expectations.load(self.config.option.expectations)
|
|
@@ -287,11 +295,15 @@ def _clear_artifacts(directory: Path) -> None:
|
|
|
287
295
|
shutil.rmtree(marker.parent)
|
|
288
296
|
|
|
289
297
|
|
|
290
|
-
def pytest_sessionfinish(session: pytest.Session) -> None:
|
|
298
|
+
def pytest_sessionfinish(session: pytest.Session, exitstatus: int | pytest.ExitCode) -> None:
|
|
291
299
|
config = session.config
|
|
292
300
|
if RUN_KEY not in config.stash:
|
|
293
301
|
return
|
|
294
302
|
run = _run(config)
|
|
303
|
+
# An interrupted run tears its fixtures down only after this, and shouldn't wait for descriptions anyway.
|
|
304
|
+
if exitstatus == pytest.ExitCode.INTERRUPTED:
|
|
305
|
+
run.describer.stop()
|
|
306
|
+
run.describer.wait()
|
|
295
307
|
path = config.option.results_json or run.artifacts / "results.json"
|
|
296
308
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
297
309
|
path.write_text(run.report.to_json(), encoding="utf-8")
|
|
@@ -340,7 +352,8 @@ def _status(call: pytest.CallInfo[None], item: pytest.Item) -> str:
|
|
|
340
352
|
return "PASS"
|
|
341
353
|
if call.excinfo.errisinstance(pytest.skip.Exception):
|
|
342
354
|
return "SKIP"
|
|
343
|
-
|
|
355
|
+
# urllib3's: a WebDriver command that got no answer within the client's timeout.
|
|
356
|
+
if call.excinfo.errisinstance((ScenarioTimeout, TimeoutException, urllib3.exceptions.TimeoutError)):
|
|
344
357
|
return "TIMEOUT"
|
|
345
358
|
if call.excinfo.errisinstance(AppCrashed):
|
|
346
359
|
return "CRASH"
|
|
@@ -355,6 +368,15 @@ def _is_running(app: Servoshell | WebViewApp) -> bool | None:
|
|
|
355
368
|
return None
|
|
356
369
|
|
|
357
370
|
|
|
371
|
+
def _exited(app: Servoshell) -> bool:
|
|
372
|
+
"""Whether servoshell was started and has exited, as opposed to not started or stopped; False if that can't be
|
|
373
|
+
told."""
|
|
374
|
+
try:
|
|
375
|
+
return app.exited()
|
|
376
|
+
except Exception:
|
|
377
|
+
return False
|
|
378
|
+
|
|
379
|
+
|
|
358
380
|
def _crashed(app: WebViewApp) -> bool:
|
|
359
381
|
"""Whether the app exited or one of its processes crashed; False if that can't be told."""
|
|
360
382
|
try:
|
|
@@ -401,8 +423,10 @@ def pytest_runtest_makereport(
|
|
|
401
423
|
funcargs = getattr(item, "funcargs", {})
|
|
402
424
|
session = funcargs.get("servo")
|
|
403
425
|
webview_apps = [value for value in funcargs.values() if isinstance(value, WebViewApp)]
|
|
404
|
-
|
|
405
|
-
if status in ("PASS", "FAIL") and
|
|
426
|
+
servoshells = [session.app] if isinstance(session, ServoSession) else []
|
|
427
|
+
if status in ("PASS", "FAIL") and (
|
|
428
|
+
any(_exited(app) for app in servoshells) or any(_is_running(app) is False for app in webview_apps)
|
|
429
|
+
):
|
|
406
430
|
status = "CRASH"
|
|
407
431
|
elif status in ("FAIL", "TIMEOUT") and any(_crashed(app) for app in webview_apps):
|
|
408
432
|
status = "CRASH"
|
|
@@ -713,6 +737,7 @@ def page_runs(scenario_run: ScenarioRun, platform: Platform, pages: PageServer)
|
|
|
713
737
|
)
|
|
714
738
|
if name and version
|
|
715
739
|
},
|
|
740
|
+
scenario_run.describer,
|
|
716
741
|
)
|
|
717
742
|
yield runs
|
|
718
743
|
runs.stop()
|
{servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/report.py
RENAMED
|
@@ -316,6 +316,8 @@ def _comparison(prefix: str, test: str, folder: Path | None, result: Result | No
|
|
|
316
316
|
head = f'<h3 id="{prefix}">{badge} {escape(labels[1])} against {escape(against)} {_status(result)}</h3>'
|
|
317
317
|
if sides := _sides(report):
|
|
318
318
|
head += f"<p class=muted>{sides}</p>"
|
|
319
|
+
if error := report.get("description_error"):
|
|
320
|
+
head += f"<p>Describing the differences failed: <code>{escape(str(error))}</code></p>"
|
|
319
321
|
triage = _load(folder / TRIAGE)
|
|
320
322
|
if triage:
|
|
321
323
|
agent, verdict, summary = (escape(str(triage.get(key, ""))) for key in ("agent", "verdict", "summary"))
|
|
@@ -4,7 +4,8 @@
|
|
|
4
4
|
the seeded `Math.random()`).
|
|
5
5
|
`test_matches_baseline` compares the captures of --engine with its baselines in --baselines, and
|
|
6
6
|
`test_matches_reference` with those of --reference-engine, stored in --baselines or rendered in the run. Each page is
|
|
7
|
-
rendered once per engine for all three. Differences are described in each test's `diff.json
|
|
7
|
+
rendered once per engine for all three. Differences are described in each test's `diff.json`: the comparison tests
|
|
8
|
+
decide at once, and describing the differences, the slow part, goes on in the background until the end of the run.
|
|
8
9
|
"""
|
|
9
10
|
|
|
10
11
|
import pytest
|
|
@@ -49,6 +50,7 @@ def test_matches_baseline(scenario, page_runs, artifacts):
|
|
|
49
50
|
("baseline", page_runs.engine),
|
|
50
51
|
page_runs.stored_reference(scenario),
|
|
51
52
|
lambda: {"expected": store.origin(scenario.name), "actual": rendering.info.get("engine")},
|
|
53
|
+
describer=page_runs.describer,
|
|
52
54
|
)
|
|
53
55
|
assert comparison.matches, comparison.message(f"the baseline in {store.directory(scenario.name)}")
|
|
54
56
|
|
|
@@ -69,5 +71,6 @@ def test_matches_reference(scenario, page_runs, artifacts):
|
|
|
69
71
|
artifacts,
|
|
70
72
|
(page_runs.reference, page_runs.engine),
|
|
71
73
|
describe=lambda: {"expected": origin, "actual": rendering.info.get("engine")},
|
|
74
|
+
describer=page_runs.describer,
|
|
72
75
|
)
|
|
73
76
|
assert comparison.matches, comparison.message(f"{page_runs.reference}'s rendering")
|
{servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/servoshell.py
RENAMED
|
@@ -89,6 +89,10 @@ class ServoshellOhos:
|
|
|
89
89
|
def is_running(self) -> bool:
|
|
90
90
|
return self.pid is not None and self.device.pid_of(self.bundle) == self.pid
|
|
91
91
|
|
|
92
|
+
def exited(self) -> bool:
|
|
93
|
+
"""Whether the app was started and its process is gone, though `stop` wasn't called."""
|
|
94
|
+
return self.pid is not None and self.device.pid_of(self.bundle) != self.pid
|
|
95
|
+
|
|
92
96
|
|
|
93
97
|
class ServoshellDesktop:
|
|
94
98
|
"""servoshell as a local binary."""
|
|
@@ -133,5 +137,9 @@ class ServoshellDesktop:
|
|
|
133
137
|
def is_running(self) -> bool:
|
|
134
138
|
return self.process is not None and self.process.poll() is None
|
|
135
139
|
|
|
140
|
+
def exited(self) -> bool:
|
|
141
|
+
"""Whether servoshell was started and its process is gone, though `stop` wasn't called."""
|
|
142
|
+
return self.process is not None and self.process.poll() is not None
|
|
143
|
+
|
|
136
144
|
|
|
137
145
|
Servoshell = ServoshellOhos | ServoshellDesktop
|
|
@@ -15,7 +15,8 @@ doesn't settle the question.
|
|
|
15
15
|
- `<artifacts>/results.json`: failed tests with their message. The message names each differing capture and its
|
|
16
16
|
largest region.
|
|
17
17
|
- Each failed test's folder (`<artifacts>/page_scenarios_test_pages.py_test_matches_<kind>_<scenario>_/`) has:
|
|
18
|
-
- `diff.json`;
|
|
18
|
+
- `diff.json`; one with `description_error` lacks the readings, DOM changes and images below, because the run was
|
|
19
|
+
interrupted or describing the differences failed: run the scenario again rather than triage without them;
|
|
19
20
|
- `<capture>-compare.png`: expected | actual | differing pixels;
|
|
20
21
|
- `regions/<capture>-<n>.png`: the crop of each region, largest first.
|
|
21
22
|
- The renderings are in `<artifacts>/page-scenarios_<engine>_<scenario>/`:
|
{servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/webdriver.py
RENAMED
|
@@ -8,10 +8,11 @@ from collections.abc import Callable
|
|
|
8
8
|
|
|
9
9
|
from selenium import webdriver
|
|
10
10
|
from selenium.webdriver.common.options import ArgOptions
|
|
11
|
+
from selenium.webdriver.common.proxy import Proxy, ProxyType
|
|
11
12
|
from selenium.webdriver.remote.client_config import ClientConfig
|
|
12
13
|
from selenium.webdriver.remote.remote_connection import RemoteConnection
|
|
13
14
|
|
|
14
|
-
from .errors import ScenarioError
|
|
15
|
+
from .errors import AppCrashed, ScenarioError
|
|
15
16
|
|
|
16
17
|
|
|
17
18
|
def _port_open(port: int) -> bool:
|
|
@@ -44,13 +45,14 @@ def connect(
|
|
|
44
45
|
raise ScenarioError(f"WebDriver server on port {port} did not come up within {timeout}s")
|
|
45
46
|
if alive is not None and time.monotonic() > next_alive_check:
|
|
46
47
|
if not alive():
|
|
47
|
-
raise
|
|
48
|
+
raise AppCrashed("servoshell exited before its WebDriver server came up")
|
|
48
49
|
next_alive_check = time.monotonic() + 2
|
|
49
50
|
time.sleep(0.3)
|
|
50
51
|
time.sleep(grace)
|
|
51
52
|
options = ArgOptions()
|
|
52
53
|
options.set_capability("browserName", "servo")
|
|
53
|
-
|
|
54
|
+
direct = Proxy({"proxyType": ProxyType.DIRECT})
|
|
55
|
+
config = ClientConfig(remote_server_addr=f"http://127.0.0.1:{port}", timeout=120, proxy=direct)
|
|
54
56
|
last_error: Exception | None = None
|
|
55
57
|
while time.monotonic() < deadline:
|
|
56
58
|
try:
|
|
@@ -58,4 +60,6 @@ def connect(
|
|
|
58
60
|
except Exception as error:
|
|
59
61
|
last_error = error
|
|
60
62
|
time.sleep(0.5)
|
|
63
|
+
if alive is not None and not alive():
|
|
64
|
+
raise AppCrashed(f"servoshell exited while a WebDriver session was being created: {last_error}")
|
|
61
65
|
raise ScenarioError(f"Could not create a WebDriver session on port {port}: {last_error}")
|
|
File without changes
|
|
File without changes
|
{servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/__init__.py
RENAMED
|
File without changes
|
{servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/__main__.py
RENAMED
|
File without changes
|
|
File without changes
|
{servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/errors.py
RENAMED
|
File without changes
|
{servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/expectations.py
RENAMED
|
File without changes
|
{servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/hookspecs.py
RENAMED
|
File without changes
|
{servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/oracles.py
RENAMED
|
File without changes
|
{servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/pages/basic.html
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/pages/ime.html
RENAMED
|
File without changes
|
{servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/pages/report.js
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/pages/svg/LICENSE
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/platform/ohos.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/screenshots.py
RENAMED
|
File without changes
|
|
File without changes
|
{servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/svg_cards.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{servo_embedder_tests-0.1.2 → servo_embedder_tests-0.2.0}/src/servo_embedder_tests/webview_app.py
RENAMED
|
File without changes
|