rustwright 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (165) hide show
  1. rustwright-0.1.0/.dockerignore +34 -0
  2. rustwright-0.1.0/.github/ISSUE_TEMPLATE/bug_report.md +34 -0
  3. rustwright-0.1.0/.github/ISSUE_TEMPLATE/config.yml +5 -0
  4. rustwright-0.1.0/.github/ISSUE_TEMPLATE/feature_request.md +18 -0
  5. rustwright-0.1.0/.github/PULL_REQUEST_TEMPLATE.md +16 -0
  6. rustwright-0.1.0/.github/dependabot.yml +26 -0
  7. rustwright-0.1.0/.github/prompts/code-review.md +56 -0
  8. rustwright-0.1.0/.github/sync.yml +62 -0
  9. rustwright-0.1.0/.github/workflows/auto-merge-sync.yml +48 -0
  10. rustwright-0.1.0/.github/workflows/benchmark-testbox.yml +49 -0
  11. rustwright-0.1.0/.github/workflows/benchmark.yml +296 -0
  12. rustwright-0.1.0/.github/workflows/claude-code-review.yml +87 -0
  13. rustwright-0.1.0/.github/workflows/fingerprint.yml +198 -0
  14. rustwright-0.1.0/.github/workflows/release-npm.yml +280 -0
  15. rustwright-0.1.0/.github/workflows/release-pypi.yml +190 -0
  16. rustwright-0.1.0/.github/workflows/sync-rustwright-cloud.yml +62 -0
  17. rustwright-0.1.0/.github/workflows/test.yml +115 -0
  18. rustwright-0.1.0/.gitignore +32 -0
  19. rustwright-0.1.0/AGENTS.md +56 -0
  20. rustwright-0.1.0/BENCHMARK.md +202 -0
  21. rustwright-0.1.0/CODE_ARCHITECTURE.md +166 -0
  22. rustwright-0.1.0/CODE_OF_CONDUCT.md +116 -0
  23. rustwright-0.1.0/CONTRIBUTING.md +60 -0
  24. rustwright-0.1.0/Cargo.lock +1888 -0
  25. rustwright-0.1.0/Cargo.toml +51 -0
  26. rustwright-0.1.0/Dockerfile +126 -0
  27. rustwright-0.1.0/LICENSE +21 -0
  28. rustwright-0.1.0/LIMITATIONS.md +23 -0
  29. rustwright-0.1.0/MEMORY_BENCH.md +52 -0
  30. rustwright-0.1.0/PKG-INFO +239 -0
  31. rustwright-0.1.0/QUICKSTART.md +139 -0
  32. rustwright-0.1.0/README.md +214 -0
  33. rustwright-0.1.0/SECURITY.md +14 -0
  34. rustwright-0.1.0/benchmarks/README.md +38 -0
  35. rustwright-0.1.0/benchmarks/async_concurrency_load.py +615 -0
  36. rustwright-0.1.0/benchmarks/automation_cases.py +21869 -0
  37. rustwright-0.1.0/benchmarks/basemark/README.md +22 -0
  38. rustwright-0.1.0/benchmarks/basemark/benchmark.json +9 -0
  39. rustwright-0.1.0/benchmarks/basemark/run.py +28 -0
  40. rustwright-0.1.0/benchmarks/browser_speed/README.md +34 -0
  41. rustwright-0.1.0/benchmarks/browser_speed/list.py +33 -0
  42. rustwright-0.1.0/benchmarks/browser_speed_lib.py +131 -0
  43. rustwright-0.1.0/benchmarks/browsertime/README.md +25 -0
  44. rustwright-0.1.0/benchmarks/browsertime/benchmark.json +9 -0
  45. rustwright-0.1.0/benchmarks/browsertime/run.py +77 -0
  46. rustwright-0.1.0/benchmarks/browsertime/scripts/basic_journey.cjs +9 -0
  47. rustwright-0.1.0/benchmarks/browsertime/urls/default.txt +2 -0
  48. rustwright-0.1.0/benchmarks/crossbench/README.md +30 -0
  49. rustwright-0.1.0/benchmarks/crossbench/benchmark.json +9 -0
  50. rustwright-0.1.0/benchmarks/crossbench/run.py +114 -0
  51. rustwright-0.1.0/benchmarks/jetstream/README.md +20 -0
  52. rustwright-0.1.0/benchmarks/jetstream/benchmark.json +9 -0
  53. rustwright-0.1.0/benchmarks/jetstream/run.py +42 -0
  54. rustwright-0.1.0/benchmarks/motionmark/README.md +17 -0
  55. rustwright-0.1.0/benchmarks/motionmark/benchmark.json +9 -0
  56. rustwright-0.1.0/benchmarks/motionmark/run.py +42 -0
  57. rustwright-0.1.0/benchmarks/process_tree_memory.py +180 -0
  58. rustwright-0.1.0/benchmarks/run_benchmarks.py +1586 -0
  59. rustwright-0.1.0/benchmarks/speedometer/README.md +20 -0
  60. rustwright-0.1.0/benchmarks/speedometer/benchmark.json +9 -0
  61. rustwright-0.1.0/benchmarks/speedometer/run.py +42 -0
  62. rustwright-0.1.0/benchmarks/tachometer/README.md +23 -0
  63. rustwright-0.1.0/benchmarks/tachometer/benchmark.json +9 -0
  64. rustwright-0.1.0/benchmarks/tachometer/cases/dom_append.html +17 -0
  65. rustwright-0.1.0/benchmarks/tachometer/cases/layout_read.html +35 -0
  66. rustwright-0.1.0/benchmarks/tachometer/cases/selector_query.html +28 -0
  67. rustwright-0.1.0/benchmarks/tachometer/run.py +52 -0
  68. rustwright-0.1.0/benchmarks/telescope/README.md +23 -0
  69. rustwright-0.1.0/benchmarks/telescope/benchmark.json +9 -0
  70. rustwright-0.1.0/benchmarks/telescope/run.py +78 -0
  71. rustwright-0.1.0/benchmarks/webpagetest/README.md +23 -0
  72. rustwright-0.1.0/benchmarks/webpagetest/benchmark.json +9 -0
  73. rustwright-0.1.0/benchmarks/webpagetest/run.py +84 -0
  74. rustwright-0.1.0/benchmarks/webpagetest/scripts/basic_navigation.wpt +2 -0
  75. rustwright-0.1.0/docs/PARITY.md +819 -0
  76. rustwright-0.1.0/docs/RELEASING.md +93 -0
  77. rustwright-0.1.0/docs/api-surface/playwright-async-api-classifications.json +62 -0
  78. rustwright-0.1.0/docs/api-surface/playwright-async-api.json +1865 -0
  79. rustwright-0.1.0/docs/api-surface/playwright-async-api.md +9 -0
  80. rustwright-0.1.0/docs/api-surface/playwright-sync-api-classifications.json +90 -0
  81. rustwright-0.1.0/docs/api-surface/playwright-sync-api.json +1865 -0
  82. rustwright-0.1.0/docs/api-surface/playwright-sync-api.md +9 -0
  83. rustwright-0.1.0/docs/assets/banner.png +0 -0
  84. rustwright-0.1.0/docs/assets/logo.png +0 -0
  85. rustwright-0.1.0/docs/assets/mascot.png +0 -0
  86. rustwright-0.1.0/docs/async-design.md +233 -0
  87. rustwright-0.1.0/examples/quickstart.py +13 -0
  88. rustwright-0.1.0/pyproject.toml +48 -0
  89. rustwright-0.1.0/python/rustwright/__init__.py +56 -0
  90. rustwright-0.1.0/python/rustwright/__main__.py +5 -0
  91. rustwright-0.1.0/python/rustwright/_backend.py +58 -0
  92. rustwright-0.1.0/python/rustwright/_compat/__init__.py +83 -0
  93. rustwright-0.1.0/python/rustwright/_compat/cloakbrowser/__init__.py +92 -0
  94. rustwright-0.1.0/python/rustwright/_compat/patchright/__init__.py +2 -0
  95. rustwright-0.1.0/python/rustwright/_compat/patchright/__main__.py +5 -0
  96. rustwright-0.1.0/python/rustwright/_compat/patchright/_impl/__init__.py +1 -0
  97. rustwright-0.1.0/python/rustwright/_compat/patchright/_impl/_errors.py +4 -0
  98. rustwright-0.1.0/python/rustwright/_compat/patchright/async_api.py +2 -0
  99. rustwright-0.1.0/python/rustwright/_compat/patchright/pytest_plugin.py +2 -0
  100. rustwright-0.1.0/python/rustwright/_compat/patchright/sync_api.py +2 -0
  101. rustwright-0.1.0/python/rustwright/_compat/playwright/__init__.py +2 -0
  102. rustwright-0.1.0/python/rustwright/_compat/playwright/__main__.py +5 -0
  103. rustwright-0.1.0/python/rustwright/_compat/playwright/_impl/__init__.py +2 -0
  104. rustwright-0.1.0/python/rustwright/_compat/playwright/_impl/_errors.py +4 -0
  105. rustwright-0.1.0/python/rustwright/_compat/playwright/async_api.py +2 -0
  106. rustwright-0.1.0/python/rustwright/_compat/playwright/pytest_plugin.py +1 -0
  107. rustwright-0.1.0/python/rustwright/_compat/playwright/sync_api.py +2 -0
  108. rustwright-0.1.0/python/rustwright/_compat/pytest_playwright/__init__.py +3 -0
  109. rustwright-0.1.0/python/rustwright/_compat/pytest_playwright/py.typed +1 -0
  110. rustwright-0.1.0/python/rustwright/_compat/pytest_playwright/pytest_playwright.py +57 -0
  111. rustwright-0.1.0/python/rustwright/_devices.py +1034 -0
  112. rustwright-0.1.0/python/rustwright/async_api.py +5745 -0
  113. rustwright-0.1.0/python/rustwright/cli.py +1538 -0
  114. rustwright-0.1.0/python/rustwright/pytest_plugin.py +409 -0
  115. rustwright-0.1.0/python/rustwright/sync_api.py +29198 -0
  116. rustwright-0.1.0/src/lib.rs +13713 -0
  117. rustwright-0.1.0/tests/fixtures/mind2web_sample.json +57 -0
  118. rustwright-0.1.0/tests/test_adversarial_review.py +276 -0
  119. rustwright-0.1.0/tests/test_antibot_benchmarks.py +399 -0
  120. rustwright-0.1.0/tests/test_benchmark_memory.py +89 -0
  121. rustwright-0.1.0/tests/test_mind2web_benchmark.py +121 -0
  122. rustwright-0.1.0/tests/test_playwright_compat_optin.py +173 -0
  123. rustwright-0.1.0/tests/test_playwright_parity_cases.py +70 -0
  124. rustwright-0.1.0/tests/test_project_status_tools.py +1299 -0
  125. rustwright-0.1.0/tests/test_rustwright_sync_api.py +36374 -0
  126. rustwright-0.1.0/tests/test_skyvern_alias_command.py +90 -0
  127. rustwright-0.1.0/tests/test_skyvern_cloud_overlay_tests.py +108 -0
  128. rustwright-0.1.0/tests/test_skyvern_prompt_overlay_smoke.py +103 -0
  129. rustwright-0.1.0/tests/test_skyvern_replacement_smoke.py +361 -0
  130. rustwright-0.1.0/tools/adversarial_review.py +438 -0
  131. rustwright-0.1.0/tools/api_surface_audit.py +445 -0
  132. rustwright-0.1.0/tools/audit_skyvern_playwright_usage.py +856 -0
  133. rustwright-0.1.0/tools/check_benchmark_artifacts.py +164 -0
  134. rustwright-0.1.0/tools/check_cross_library_speed_goal.py +225 -0
  135. rustwright-0.1.0/tools/check_external_reliability_goal.py +274 -0
  136. rustwright-0.1.0/tools/check_launch_latency_claim.py +460 -0
  137. rustwright-0.1.0/tools/check_native_extension.py +60 -0
  138. rustwright-0.1.0/tools/check_phase1_gate.py +308 -0
  139. rustwright-0.1.0/tools/check_phase2_benchmark.py +235 -0
  140. rustwright-0.1.0/tools/check_testbox_visibility.py +294 -0
  141. rustwright-0.1.0/tools/docker_test.sh +208 -0
  142. rustwright-0.1.0/tools/docker_verify.sh +439 -0
  143. rustwright-0.1.0/tools/download_mind2web.py +92 -0
  144. rustwright-0.1.0/tools/generate_parity_map.py +999 -0
  145. rustwright-0.1.0/tools/import_mind2web.py +379 -0
  146. rustwright-0.1.0/tools/import_webvoyager.py +178 -0
  147. rustwright-0.1.0/tools/install_hooks.py +106 -0
  148. rustwright-0.1.0/tools/query_project_state.py +1046 -0
  149. rustwright-0.1.0/tools/render_benchmark_matrix.py +173 -0
  150. rustwright-0.1.0/tools/render_project_tables.py +286 -0
  151. rustwright-0.1.0/tools/run_antibot_benchmarks.py +1260 -0
  152. rustwright-0.1.0/tools/run_benchmark_matrix.py +571 -0
  153. rustwright-0.1.0/tools/run_benchmark_testbox.sh +223 -0
  154. rustwright-0.1.0/tools/run_mind2web_benchmark.py +1248 -0
  155. rustwright-0.1.0/tools/run_mind2web_matrix.py +286 -0
  156. rustwright-0.1.0/tools/run_mind2web_sharded.py +514 -0
  157. rustwright-0.1.0/tools/run_parity_cases.py +151 -0
  158. rustwright-0.1.0/tools/run_remote_docker_test.py +562 -0
  159. rustwright-0.1.0/tools/run_skyvern_alias_command.py +339 -0
  160. rustwright-0.1.0/tools/run_skyvern_cloud_overlay_tests.py +434 -0
  161. rustwright-0.1.0/tools/run_skyvern_prompt_overlay_smoke.py +210 -0
  162. rustwright-0.1.0/tools/run_skyvern_replacement_smoke.py +425 -0
  163. rustwright-0.1.0/tools/run_webvoyager_benchmark.py +604 -0
  164. rustwright-0.1.0/tools/run_webvoyager_matrix.py +299 -0
  165. rustwright-0.1.0/tools/rustwright_ts_binding.js +327 -0
@@ -0,0 +1,34 @@
1
+ .git
2
+ .audit-playwright
3
+ .venv
4
+ .pytest_cache
5
+ .mypy_cache
6
+ .ruff_cache
7
+ target
8
+ dist
9
+ build
10
+ *.egg-info
11
+ __pycache__
12
+ *.py[cod]
13
+ *.so
14
+ *.dylib
15
+ *.dll
16
+ *.log
17
+ .DS_Store
18
+ downloads
19
+ test-results
20
+ playwright-report
21
+ videos
22
+ traces
23
+ screenshots
24
+ .benchmark-data
25
+ benchmark-data
26
+ datasets
27
+ data/Mind2Web
28
+ data/mind2web
29
+ data/WebVoyager
30
+ data/webvoyager
31
+ data/WebVoyager_data.jsonl
32
+ data/GAIA_web.jsonl
33
+ data/reference_answer.json
34
+ benchmarks/external
@@ -0,0 +1,34 @@
1
+ ---
2
+ name: Bug report
3
+ about: Report a reproducible Rustwright bug
4
+ title: "[Bug]: "
5
+ labels: bug
6
+ assignees: ""
7
+ ---
8
+
9
+ ## What happened?
10
+
11
+
12
+ ## Reproduction
13
+
14
+ ```python
15
+ # Minimal code or commands that reproduce the issue
16
+ ```
17
+
18
+ ## Expected behavior
19
+
20
+
21
+ ## Environment
22
+
23
+ - OS:
24
+ - Python version:
25
+ - Rust version:
26
+ - Node version, if relevant:
27
+ - Browser:
28
+ - Rustwright commit:
29
+
30
+ ## Logs
31
+
32
+ ```text
33
+
34
+ ```
@@ -0,0 +1,5 @@
1
+ blank_issues_enabled: false
2
+ contact_links:
3
+ - name: Questions and setup help
4
+ url: https://discord.gg/fG2XXEuQX3
5
+ about: Use the Skyvern Discord for questions, troubleshooting, and discussion.
@@ -0,0 +1,18 @@
1
+ ---
2
+ name: Feature request
3
+ about: Suggest a Rustwright capability or compatibility improvement
4
+ title: "[Feature]: "
5
+ labels: enhancement
6
+ assignees: ""
7
+ ---
8
+
9
+ ## What should Rustwright support?
10
+
11
+
12
+ ## Why is this useful?
13
+
14
+
15
+ ## Playwright behavior or reference
16
+
17
+
18
+ ## Additional context
@@ -0,0 +1,16 @@
1
+ ## What
2
+
3
+
4
+ ## Why
5
+
6
+
7
+ ## Testing
8
+
9
+ - [ ] `cargo check --locked`
10
+ - [ ] Targeted pytest, using the focused `-k` subset from `CONTRIBUTING.md`
11
+
12
+ ## Checklist
13
+
14
+ - [ ] I kept this change focused.
15
+ - [ ] I added or updated tests for behavior changes.
16
+ - [ ] I did not include private credentials, tokens, or internal-only artifacts.
@@ -0,0 +1,26 @@
1
+ version: 2
2
+ updates:
3
+ - package-ecosystem: cargo
4
+ directory: "/"
5
+ schedule:
6
+ interval: weekly
7
+
8
+ - package-ecosystem: cargo
9
+ directory: "/node"
10
+ schedule:
11
+ interval: weekly
12
+
13
+ - package-ecosystem: pip
14
+ directory: "/"
15
+ schedule:
16
+ interval: weekly
17
+
18
+ - package-ecosystem: npm
19
+ directory: "/node"
20
+ schedule:
21
+ interval: weekly
22
+
23
+ - package-ecosystem: github-actions
24
+ directory: "/"
25
+ schedule:
26
+ interval: weekly
@@ -0,0 +1,56 @@
1
+ Please review this pull request for the **Rustwright** repository and give concise, actionable feedback.
2
+
3
+ Rustwright is a Rust engine (PyO3 for Python, napi-rs for Node) that speaks raw Chrome DevTools Protocol and exposes a **Playwright-compatible** API. It is alpha, Chromium-only, MIT-licensed. The Rust core lives in `src/lib.rs`; the Python API in `python/rustwright/`; the Node bindings in `node/`.
4
+
5
+ Review for:
6
+
7
+ - **Correctness & bugs** — logic errors, unhandled edge cases, error handling. Pay special attention to:
8
+ - the FFI boundaries — PyO3 `Bound<'py, T>` usage and GIL handling (`py.detach` around blocking work), and napi async/threadsafe-function usage;
9
+ - the CDP layer — session/frame routing (OOPIF), the tokio-tungstenite WebSocket transport, and message dispatch/correlation.
10
+ - **Memory & concurrency safety** — any new `unsafe`; `unwrap()`/`expect()`/`panic!` on fallible paths that could crash a user's process across the FFI boundary; races or deadlocks in the async CDP client; holding the GIL across I/O.
11
+ - **Security** (this is browser-automation software) — untrusted-input handling, JS/CDP injection, and anything that silently changes the **trusted-input-by-default** behavior. Rustwright must **not overclaim on anti-detection**: flag any change that adds "undetectable" / "bypass Cloudflare" / "stealth" style claims, or that weakens the honest signal-hygiene framing (it currently passes 3/4 public fingerprint targets and says so plainly).
12
+ - **API compatibility** — the Python/Node surface mirrors Playwright. Flag divergence from Playwright's shape/behavior that isn't justified, and missing parity coverage.
13
+ - **Performance** — avoidable allocations/copies on the per-CDP-message hot path; blocking calls inside async paths.
14
+ - **Tests** — is the change covered by the pytest suite (`tests/`)? Does it warrant a parity test against real Playwright?
15
+ - **Honesty of claims** — if the PR touches README/docs/benchmarks, verify the claims match what the code and tests actually support (no inflated numbers, no unverified platform/behavior claims, MSRV kept accurate).
16
+
17
+ Use `CONTRIBUTING.md` (and `CLAUDE.md` if present) for conventions. Be constructive.
18
+
19
+ Keep the review **concise and scannable**:
20
+ - Start with a 1–2 sentence overall assessment.
21
+ - Group feedback by severity using GitHub `<details>`/`<summary>` collapsible sections.
22
+ - Expand 🔴 Critical by default; keep 🟡 Suggestions and 📝 Minor collapsed.
23
+ - If there are no issues, post a brief LGTM (with the marker).
24
+
25
+ Format:
26
+
27
+ ```
28
+ <!-- claude-code-review -->
29
+ ## Summary
30
+ Brief overall assessment.
31
+
32
+ <details open>
33
+ <summary>🔴 Critical Issues (X)</summary>
34
+
35
+ - ...
36
+
37
+ </details>
38
+
39
+ <details>
40
+ <summary>🟡 Suggestions (X)</summary>
41
+
42
+ - ...
43
+
44
+ </details>
45
+
46
+ <details>
47
+ <summary>📝 Minor / Style (X)</summary>
48
+
49
+ - ...
50
+
51
+ </details>
52
+ ```
53
+
54
+ IMPORTANT: Start your comment with the exact marker `<!-- claude-code-review -->` on the first line — this lets a later commit replace the previous review instead of stacking comments.
55
+
56
+ Use `gh pr comment` (via your Bash tool) to post the review as a single comment on the PR.
@@ -0,0 +1,62 @@
1
+ # Shared Rustwright files mirrored from rustwright to rustwright-cloud.
2
+ # Repository-specific GitHub workflows and metadata are intentionally excluded.
3
+ # Cloud-only additions must live outside the mirrored paths below.
4
+ Skyvern-AI/rustwright-cloud:
5
+ - source: benchmarks/
6
+ dest: benchmarks/
7
+ deleteOrphaned: true
8
+ - source: docs/
9
+ dest: docs/
10
+ deleteOrphaned: true
11
+ - source: node/
12
+ dest: node/
13
+ deleteOrphaned: true
14
+ - source: python/
15
+ dest: python/
16
+ deleteOrphaned: true
17
+ - source: src/
18
+ dest: src/
19
+ deleteOrphaned: true
20
+ - source: tests/
21
+ dest: tests/
22
+ deleteOrphaned: true
23
+ - source: tools/
24
+ dest: tools/
25
+ deleteOrphaned: true
26
+ - source: .dockerignore
27
+ dest: .dockerignore
28
+ - source: .gitignore
29
+ dest: .gitignore
30
+ - source: BENCHMARK.md
31
+ dest: BENCHMARK.md
32
+ - source: CODE_ARCHITECTURE.md
33
+ dest: CODE_ARCHITECTURE.md
34
+ - source: CODE_OF_CONDUCT.md
35
+ dest: CODE_OF_CONDUCT.md
36
+ - source: CONTRIBUTING.md
37
+ dest: CONTRIBUTING.md
38
+ - source: Cargo.lock
39
+ dest: Cargo.lock
40
+ - source: Cargo.toml
41
+ dest: Cargo.toml
42
+ - source: Dockerfile
43
+ dest: Dockerfile
44
+ - source: LICENSE
45
+ dest: LICENSE
46
+ - source: LIMITATIONS.md
47
+ dest: LIMITATIONS.md
48
+ - source: README.md
49
+ dest: README.md
50
+ - source: SECURITY.md
51
+ dest: SECURITY.md
52
+ - source: pyproject.toml
53
+ dest: pyproject.toml
54
+ - source: examples/
55
+ dest: examples/
56
+ deleteOrphaned: true
57
+ - source: AGENTS.md
58
+ dest: AGENTS.md
59
+ - source: QUICKSTART.md
60
+ dest: QUICKSTART.md
61
+ - source: MEMORY_BENCH.md
62
+ dest: MEMORY_BENCH.md
@@ -0,0 +1,48 @@
1
+ name: Auto-merge sync PRs
2
+
3
+ on:
4
+ pull_request:
5
+ types: [opened, reopened, ready_for_review, labeled, synchronize]
6
+
7
+ permissions:
8
+ contents: write
9
+ pull-requests: write
10
+ checks: read
11
+
12
+ jobs:
13
+ merge-when-sync:
14
+ # Auto-merge only machine-generated mirror PRs:
15
+ # - carry the 'sync' label applied by repo-file-sync-action
16
+ # - branch created by the sync action (repo-sync/*)
17
+ # - opened by the configured sync actor
18
+ if: >-
19
+ contains(join(github.event.pull_request.labels.*.name, ','), 'sync') &&
20
+ startsWith(github.event.pull_request.head.ref, 'repo-sync/') &&
21
+ github.event.pull_request.user.login == vars.RUSTWRIGHT_SYNC_ACTOR
22
+ runs-on: ubuntu-latest
23
+ steps:
24
+ - name: Wait for required checks to pass
25
+ uses: lewagon/wait-on-check-action@74049309dfeff245fe8009a0137eacf28136cb3c # v1.5.0
26
+ with:
27
+ ref: ${{ github.event.pull_request.head.sha }}
28
+ repo-token: ${{ secrets.GITHUB_TOKEN }}
29
+ wait-interval: 15
30
+ allowed-conclusions: success,skipped,neutral
31
+ ignore-checks: merge-when-sync,claude-review
32
+ - name: Verify PR is still open
33
+ env:
34
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
35
+ run: |
36
+ STATE=$(gh pr view ${{ github.event.pull_request.number }} --repo "${{ github.repository }}" --json state --jq '.state')
37
+ if [ "$STATE" != "OPEN" ]; then
38
+ echo "::error::PR is $STATE — aborting merge"
39
+ exit 1
40
+ fi
41
+ - name: Merge sync PR with admin bypass
42
+ env:
43
+ GH_TOKEN: ${{ secrets.RUSTWRIGHT_SYNC_GH_PAT }}
44
+ run: |-
45
+ gh pr merge ${{ github.event.pull_request.number }} \
46
+ --squash \
47
+ --admin \
48
+ --repo "${{ github.repository }}"
@@ -0,0 +1,49 @@
1
+ name: Rustwright Benchmark Testbox
2
+
3
+ on:
4
+ workflow_dispatch:
5
+ inputs:
6
+ testbox_id:
7
+ type: string
8
+ description: "Testbox session ID"
9
+ required: true
10
+
11
+ jobs:
12
+ benchmark:
13
+ if: github.repository == 'Skyvern-AI/rustwright'
14
+ runs-on: blacksmith-4vcpu-ubuntu-2404
15
+ timeout-minutes: 120
16
+ env:
17
+ RUSTWRIGHT_DOCKER_IMAGE: rustwright-verify-testbox
18
+ TEST_DOCKER_MEMORY_LIMIT: 8g
19
+ INSTALL_PUPPETEER: "1"
20
+ DOCKER_BUILDKIT: "1"
21
+ steps:
22
+ - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
23
+
24
+ - name: Begin Testbox
25
+ uses: useblacksmith/begin-testbox@233448af4bfdc6fca509a7f0974411ac6d8a8043 # v2
26
+ with:
27
+ testbox_id: ${{ inputs.testbox_id }}
28
+
29
+ - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
30
+ with:
31
+ python-version: "3.13"
32
+
33
+ - name: Verify Docker
34
+ run: |
35
+ docker version
36
+ docker info
37
+
38
+ - name: Build benchmark Docker image
39
+ run: |
40
+ tools/docker_test.sh build .
41
+
42
+ - name: Run Testbox
43
+ uses: useblacksmith/run-testbox@5ca05834db1d3813554d1dd109e5f2087a8d7cbc # v2
44
+ if: always()
45
+ env:
46
+ RUSTWRIGHT_DOCKER_IMAGE: rustwright-verify-testbox
47
+ TEST_DOCKER_MEMORY_LIMIT: 8g
48
+ INSTALL_PUPPETEER: "1"
49
+ DOCKER_BUILDKIT: "1"
@@ -0,0 +1,296 @@
1
+ name: Rustwright Benchmarks
2
+
3
+ on:
4
+ pull_request:
5
+ types:
6
+ - opened
7
+ - synchronize
8
+ - reopened
9
+ - ready_for_review
10
+ - labeled
11
+ workflow_dispatch:
12
+ inputs:
13
+ benchmark:
14
+ type: choice
15
+ description: "Benchmark workload to run"
16
+ required: true
17
+ default: strict
18
+ options:
19
+ - strict
20
+ - equivalent
21
+ - mind2web-sharded
22
+ - defensible-speed
23
+ implementations:
24
+ type: string
25
+ description: "Comma-separated implementations, or all"
26
+ required: true
27
+ default: "rustwright,playwright"
28
+ repetitions:
29
+ type: string
30
+ description: "Benchmark repetitions"
31
+ required: true
32
+ default: "1"
33
+ iterations:
34
+ type: string
35
+ description: "Iterations per benchmark run"
36
+ required: true
37
+ default: "1"
38
+ lifecycle:
39
+ type: choice
40
+ description: "bench-full lifecycle"
41
+ required: true
42
+ default: warm-browser
43
+ options:
44
+ - warm-browser
45
+ - warm-page
46
+ - cold-browser
47
+ - cold-container
48
+ case:
49
+ type: string
50
+ description: "Optional strict/equivalent case name"
51
+ required: false
52
+ default: ""
53
+ max_tasks:
54
+ type: string
55
+ description: "Optional Mind2Web task cap"
56
+ required: false
57
+ default: ""
58
+ shard_size:
59
+ type: string
60
+ description: "Mind2Web shard size"
61
+ required: false
62
+ default: "25"
63
+ defensible_lifecycles:
64
+ type: string
65
+ description: "Comma-separated lifecycles for defensible-speed"
66
+ required: false
67
+ default: "warm-browser,warm-page,cold-browser"
68
+ strict_repetitions:
69
+ type: string
70
+ description: "Strict-suite repetitions for defensible-speed; defaults to repetitions"
71
+ required: false
72
+ default: ""
73
+ strict_iterations:
74
+ type: string
75
+ description: "Strict-suite iterations for defensible-speed"
76
+ required: false
77
+ default: "1"
78
+ enforce_phase2:
79
+ type: boolean
80
+ description: "Fail the workflow if strict Phase 2 acceptance checks fail"
81
+ required: false
82
+ default: false
83
+ runner_label:
84
+ type: choice
85
+ description: "Runner label for provenance/debug runs; Blacksmith remains the default"
86
+ required: false
87
+ default: blacksmith-4vcpu-ubuntu-2404
88
+ options:
89
+ - blacksmith-4vcpu-ubuntu-2404
90
+ - ubuntu-latest
91
+
92
+ jobs:
93
+ benchmark:
94
+ if: >-
95
+ ${{
96
+ github.repository == 'Skyvern-AI/rustwright' &&
97
+ (
98
+ github.event_name != 'pull_request' ||
99
+ github.event.pull_request.head.repo.full_name == github.repository
100
+ ) &&
101
+ (
102
+ github.event_name == 'workflow_dispatch' ||
103
+ vars.RUSTWRIGHT_RUN_PR_BENCHMARKS == '1' ||
104
+ (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-benchmarks'))
105
+ )
106
+ }}
107
+ runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.runner_label || vars.RUSTWRIGHT_BENCHMARK_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }}
108
+ timeout-minutes: 480
109
+ env:
110
+ RUSTWRIGHT_DOCKER_IMAGE: rustwright-verify-github-benchmark
111
+ TEST_DOCKER_MEMORY_LIMIT: 8g
112
+ INSTALL_PUPPETEER: "1"
113
+ DOCKER_BUILDKIT: "1"
114
+ BENCHMARK_KIND: ${{ github.event_name == 'workflow_dispatch' && inputs.benchmark || vars.RUSTWRIGHT_PR_BENCHMARK || 'equivalent' }}
115
+ IMPLEMENTATIONS: ${{ github.event_name == 'workflow_dispatch' && inputs.implementations || vars.RUSTWRIGHT_PR_BENCHMARK_IMPLEMENTATIONS || 'all' }}
116
+ REPETITIONS: ${{ github.event_name == 'workflow_dispatch' && inputs.repetitions || vars.RUSTWRIGHT_PR_BENCHMARK_REPETITIONS || '1' }}
117
+ ITERATIONS: ${{ github.event_name == 'workflow_dispatch' && inputs.iterations || vars.RUSTWRIGHT_PR_BENCHMARK_ITERATIONS || '1' }}
118
+ BENCHMARK_FULL_ITERATIONS: ${{ github.event_name == 'workflow_dispatch' && inputs.iterations || vars.RUSTWRIGHT_PR_BENCHMARK_ITERATIONS || '1' }}
119
+ LIFECYCLE: ${{ github.event_name == 'workflow_dispatch' && inputs.lifecycle || vars.RUSTWRIGHT_PR_BENCHMARK_LIFECYCLE || 'warm-browser' }}
120
+ CASE_NAME: ${{ github.event_name == 'workflow_dispatch' && inputs.case || vars.RUSTWRIGHT_PR_BENCHMARK_CASE || '' }}
121
+ MAX_TASKS: ${{ github.event_name == 'workflow_dispatch' && inputs.max_tasks || vars.RUSTWRIGHT_PR_MIND2WEB_MAX_TASKS || '' }}
122
+ SHARD_SIZE: ${{ github.event_name == 'workflow_dispatch' && inputs.shard_size || vars.RUSTWRIGHT_PR_MIND2WEB_SHARD_SIZE || '25' }}
123
+ DEFENSIBLE_LIFECYCLES: ${{ github.event_name == 'workflow_dispatch' && inputs.defensible_lifecycles || vars.RUSTWRIGHT_BENCHMARK_DEFENSIBLE_LIFECYCLES || 'warm-browser,warm-page,cold-browser' }}
124
+ DEFENSIBLE_STRICT_REPETITIONS: ${{ github.event_name == 'workflow_dispatch' && inputs.strict_repetitions || vars.RUSTWRIGHT_BENCHMARK_DEFENSIBLE_STRICT_REPETITIONS || '' }}
125
+ DEFENSIBLE_STRICT_ITERATIONS: ${{ github.event_name == 'workflow_dispatch' && inputs.strict_iterations || vars.RUSTWRIGHT_BENCHMARK_DEFENSIBLE_STRICT_ITERATIONS || '1' }}
126
+ ENFORCE_PHASE2: ${{ github.event_name == 'workflow_dispatch' && inputs.enforce_phase2 || vars.RUSTWRIGHT_BENCHMARK_ENFORCE_PHASE2 || '0' }}
127
+ BENCHMARK_RUNNER_LABEL: ${{ github.event_name == 'workflow_dispatch' && inputs.runner_label || vars.RUSTWRIGHT_BENCHMARK_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }}
128
+ steps:
129
+ - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
130
+
131
+ - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
132
+ with:
133
+ python-version: "3.13"
134
+
135
+ - name: Verify Docker
136
+ run: |
137
+ docker version
138
+ docker info
139
+
140
+ - name: Print benchmark configuration
141
+ run: |
142
+ {
143
+ echo "event=${GITHUB_EVENT_NAME}"
144
+ echo "benchmark=${BENCHMARK_KIND}"
145
+ echo "implementations=${IMPLEMENTATIONS}"
146
+ echo "repetitions=${REPETITIONS}"
147
+ echo "iterations=${ITERATIONS}"
148
+ echo "lifecycle=${LIFECYCLE}"
149
+ echo "case=${CASE_NAME}"
150
+ echo "max_tasks=${MAX_TASKS}"
151
+ echo "shard_size=${SHARD_SIZE}"
152
+ echo "defensible_lifecycles=${DEFENSIBLE_LIFECYCLES}"
153
+ echo "defensible_strict_repetitions=${DEFENSIBLE_STRICT_REPETITIONS}"
154
+ echo "defensible_strict_iterations=${DEFENSIBLE_STRICT_ITERATIONS}"
155
+ echo "enforce_phase2=${ENFORCE_PHASE2}"
156
+ echo "runner_label=${BENCHMARK_RUNNER_LABEL}"
157
+ echo "docker_memory=${TEST_DOCKER_MEMORY_LIMIT}"
158
+ }
159
+
160
+ - name: Build benchmark Docker image
161
+ run: |
162
+ tools/docker_test.sh build .
163
+
164
+ - name: Prepare Mind2Web manifest
165
+ if: ${{ env.BENCHMARK_KIND == 'mind2web-sharded' }}
166
+ run: |
167
+ set -euo pipefail
168
+ python tools/download_mind2web.py --all-train --retries 5 --timeout 300 --json
169
+ python tools/import_mind2web.py \
170
+ --source .benchmark-data/raw/mind2web/data/train \
171
+ --output .benchmark-data/manifests/mind2web-train-100pct-action-fixtures.json \
172
+ --include-action-fixtures \
173
+ --json
174
+
175
+ - name: Run benchmark
176
+ run: |
177
+ set -euo pipefail
178
+ mkdir -p .benchmark-data/results
179
+ timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
180
+
181
+ impl_args=()
182
+ if [ "$IMPLEMENTATIONS" != "all" ]; then
183
+ IFS=',' read -ra impls <<< "$IMPLEMENTATIONS"
184
+ for impl in "${impls[@]}"; do
185
+ impl="$(echo "$impl" | xargs)"
186
+ if [ -n "$impl" ]; then
187
+ impl_args+=(--impl "$impl")
188
+ fi
189
+ done
190
+ fi
191
+
192
+ case_args=()
193
+ if [ -n "$CASE_NAME" ]; then
194
+ case_args+=(--case "$CASE_NAME")
195
+ fi
196
+
197
+ if [ "$BENCHMARK_KIND" = "defensible-speed" ]; then
198
+ if [ -n "$CASE_NAME" ]; then
199
+ echo "defensible-speed uses the full predeclared case set; do not pass case filters" >&2
200
+ exit 2
201
+ fi
202
+
203
+ IFS=',' read -ra lifecycles <<< "$DEFENSIBLE_LIFECYCLES"
204
+ for benchmark_lifecycle in "${lifecycles[@]}"; do
205
+ benchmark_lifecycle="$(echo "$benchmark_lifecycle" | xargs)"
206
+ if [ -z "$benchmark_lifecycle" ]; then
207
+ continue
208
+ fi
209
+ case "$benchmark_lifecycle" in
210
+ warm-browser|warm-page|cold-browser|cold-container)
211
+ ;;
212
+ *)
213
+ echo "Unsupported defensible lifecycle: $benchmark_lifecycle" >&2
214
+ exit 2
215
+ ;;
216
+ esac
217
+
218
+ BENCHMARK_FULL_ITERATIONS="$ITERATIONS" tools/docker_test.sh bench-full \
219
+ "${impl_args[@]}" \
220
+ --suite equivalent \
221
+ --lifecycle "$benchmark_lifecycle" \
222
+ --repetitions "$REPETITIONS" \
223
+ --output ".benchmark-data/results/defensible-speed-equivalent-${benchmark_lifecycle}-${REPETITIONS}x${ITERATIONS}-${timestamp}.json" \
224
+ --json
225
+ done
226
+
227
+ strict_repetitions="$DEFENSIBLE_STRICT_REPETITIONS"
228
+ if [ -z "$strict_repetitions" ]; then
229
+ strict_repetitions="$REPETITIONS"
230
+ fi
231
+ BENCHMARK_FULL_ITERATIONS="$DEFENSIBLE_STRICT_ITERATIONS" tools/docker_test.sh bench-full \
232
+ --impl rustwright \
233
+ --impl playwright \
234
+ --suite strict \
235
+ --lifecycle warm-browser \
236
+ --repetitions "$strict_repetitions" \
237
+ --output ".benchmark-data/results/defensible-speed-strict-warm-browser-${strict_repetitions}x${DEFENSIBLE_STRICT_ITERATIONS}-${timestamp}.json" \
238
+ --json
239
+ elif [ "$BENCHMARK_KIND" = "mind2web-sharded" ]; then
240
+ max_task_args=()
241
+ if [ -n "$MAX_TASKS" ]; then
242
+ max_task_args+=(--max-tasks "$MAX_TASKS")
243
+ fi
244
+ python tools/run_mind2web_sharded.py \
245
+ "${impl_args[@]}" \
246
+ --shard-size "$SHARD_SIZE" \
247
+ --repetitions "$REPETITIONS" \
248
+ --iterations "$ITERATIONS" \
249
+ "${max_task_args[@]}" \
250
+ --output ".benchmark-data/results/mind2web-sharded-${timestamp}.json" \
251
+ --json
252
+ else
253
+ tools/docker_test.sh bench-full \
254
+ "${impl_args[@]}" \
255
+ --suite "$BENCHMARK_KIND" \
256
+ --lifecycle "$LIFECYCLE" \
257
+ --repetitions "$REPETITIONS" \
258
+ "${case_args[@]}" \
259
+ --output ".benchmark-data/results/bench-full-${BENCHMARK_KIND}-${LIFECYCLE}-${timestamp}.json" \
260
+ --json
261
+ fi
262
+
263
+ - name: Check strict Phase 2 benchmark artifacts
264
+ if: always()
265
+ run: |
266
+ set -euo pipefail
267
+ mkdir -p .benchmark-data/reports
268
+
269
+ enforce=0
270
+ case "$ENFORCE_PHASE2" in
271
+ true|True|TRUE|1|yes|YES)
272
+ enforce=1
273
+ ;;
274
+ esac
275
+
276
+ args=(
277
+ --source github-actions
278
+ --runner "$BENCHMARK_RUNNER_LABEL"
279
+ --artifact rustwright-benchmark-results
280
+ --run-url "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
281
+ --json
282
+ )
283
+ if [ "$enforce" = "1" ]; then
284
+ args+=(--enforce-phase2)
285
+ fi
286
+ python tools/check_benchmark_artifacts.py "${args[@]}"
287
+
288
+ - name: Upload benchmark results
289
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
290
+ if: always()
291
+ with:
292
+ name: rustwright-benchmark-results
293
+ path: |
294
+ .benchmark-data/results/
295
+ .benchmark-data/reports/
296
+ if-no-files-found: warn