rustwright 0.1.0a2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (151) hide show
  1. rustwright-0.1.0a2/.dockerignore +35 -0
  2. rustwright-0.1.0a2/.github/ISSUE_TEMPLATE/bug_report.md +34 -0
  3. rustwright-0.1.0a2/.github/ISSUE_TEMPLATE/config.yml +5 -0
  4. rustwright-0.1.0a2/.github/ISSUE_TEMPLATE/feature_request.md +18 -0
  5. rustwright-0.1.0a2/.github/PULL_REQUEST_TEMPLATE.md +16 -0
  6. rustwright-0.1.0a2/.github/dependabot.yml +26 -0
  7. rustwright-0.1.0a2/.github/prompts/code-review.md +56 -0
  8. rustwright-0.1.0a2/.github/workflows/benchmark-testbox.yml +51 -0
  9. rustwright-0.1.0a2/.github/workflows/benchmark.yml +297 -0
  10. rustwright-0.1.0a2/.github/workflows/claude-code-review.yml +87 -0
  11. rustwright-0.1.0a2/.github/workflows/release-npm.yml +281 -0
  12. rustwright-0.1.0a2/.github/workflows/release-pypi.yml +200 -0
  13. rustwright-0.1.0a2/.github/workflows/test.yml +115 -0
  14. rustwright-0.1.0a2/.gitignore +32 -0
  15. rustwright-0.1.0a2/BENCHMARK.md +202 -0
  16. rustwright-0.1.0a2/CODE_ARCHITECTURE.md +166 -0
  17. rustwright-0.1.0a2/CODE_OF_CONDUCT.md +116 -0
  18. rustwright-0.1.0a2/CONTRIBUTING.md +60 -0
  19. rustwright-0.1.0a2/Cargo.lock +1888 -0
  20. rustwright-0.1.0a2/Cargo.toml +47 -0
  21. rustwright-0.1.0a2/Dockerfile +126 -0
  22. rustwright-0.1.0a2/LICENSE +21 -0
  23. rustwright-0.1.0a2/LIMITATIONS.md +23 -0
  24. rustwright-0.1.0a2/PKG-INFO +224 -0
  25. rustwright-0.1.0a2/README.md +199 -0
  26. rustwright-0.1.0a2/SECURITY.md +14 -0
  27. rustwright-0.1.0a2/benchmarks/README.md +38 -0
  28. rustwright-0.1.0a2/benchmarks/async_concurrency_load.py +615 -0
  29. rustwright-0.1.0a2/benchmarks/automation_cases.py +21869 -0
  30. rustwright-0.1.0a2/benchmarks/basemark/README.md +22 -0
  31. rustwright-0.1.0a2/benchmarks/basemark/benchmark.json +9 -0
  32. rustwright-0.1.0a2/benchmarks/basemark/run.py +28 -0
  33. rustwright-0.1.0a2/benchmarks/browser_speed/README.md +34 -0
  34. rustwright-0.1.0a2/benchmarks/browser_speed/list.py +33 -0
  35. rustwright-0.1.0a2/benchmarks/browser_speed_lib.py +131 -0
  36. rustwright-0.1.0a2/benchmarks/browsertime/README.md +25 -0
  37. rustwright-0.1.0a2/benchmarks/browsertime/benchmark.json +9 -0
  38. rustwright-0.1.0a2/benchmarks/browsertime/run.py +77 -0
  39. rustwright-0.1.0a2/benchmarks/browsertime/scripts/basic_journey.cjs +9 -0
  40. rustwright-0.1.0a2/benchmarks/browsertime/urls/default.txt +2 -0
  41. rustwright-0.1.0a2/benchmarks/crossbench/README.md +30 -0
  42. rustwright-0.1.0a2/benchmarks/crossbench/benchmark.json +9 -0
  43. rustwright-0.1.0a2/benchmarks/crossbench/run.py +114 -0
  44. rustwright-0.1.0a2/benchmarks/jetstream/README.md +20 -0
  45. rustwright-0.1.0a2/benchmarks/jetstream/benchmark.json +9 -0
  46. rustwright-0.1.0a2/benchmarks/jetstream/run.py +42 -0
  47. rustwright-0.1.0a2/benchmarks/motionmark/README.md +17 -0
  48. rustwright-0.1.0a2/benchmarks/motionmark/benchmark.json +9 -0
  49. rustwright-0.1.0a2/benchmarks/motionmark/run.py +42 -0
  50. rustwright-0.1.0a2/benchmarks/run_benchmarks.py +1984 -0
  51. rustwright-0.1.0a2/benchmarks/speedometer/README.md +20 -0
  52. rustwright-0.1.0a2/benchmarks/speedometer/benchmark.json +9 -0
  53. rustwright-0.1.0a2/benchmarks/speedometer/run.py +42 -0
  54. rustwright-0.1.0a2/benchmarks/tachometer/README.md +23 -0
  55. rustwright-0.1.0a2/benchmarks/tachometer/benchmark.json +9 -0
  56. rustwright-0.1.0a2/benchmarks/tachometer/cases/dom_append.html +17 -0
  57. rustwright-0.1.0a2/benchmarks/tachometer/cases/layout_read.html +35 -0
  58. rustwright-0.1.0a2/benchmarks/tachometer/cases/selector_query.html +28 -0
  59. rustwright-0.1.0a2/benchmarks/tachometer/run.py +52 -0
  60. rustwright-0.1.0a2/benchmarks/telescope/README.md +23 -0
  61. rustwright-0.1.0a2/benchmarks/telescope/benchmark.json +9 -0
  62. rustwright-0.1.0a2/benchmarks/telescope/run.py +78 -0
  63. rustwright-0.1.0a2/benchmarks/webpagetest/README.md +23 -0
  64. rustwright-0.1.0a2/benchmarks/webpagetest/benchmark.json +9 -0
  65. rustwright-0.1.0a2/benchmarks/webpagetest/run.py +84 -0
  66. rustwright-0.1.0a2/benchmarks/webpagetest/scripts/basic_navigation.wpt +2 -0
  67. rustwright-0.1.0a2/docs/RELEASING.md +93 -0
  68. rustwright-0.1.0a2/docs/api-surface/playwright-async-api-classifications.json +62 -0
  69. rustwright-0.1.0a2/docs/api-surface/playwright-async-api.json +1865 -0
  70. rustwright-0.1.0a2/docs/api-surface/playwright-async-api.md +9 -0
  71. rustwright-0.1.0a2/docs/api-surface/playwright-sync-api-classifications.json +90 -0
  72. rustwright-0.1.0a2/docs/api-surface/playwright-sync-api.json +1865 -0
  73. rustwright-0.1.0a2/docs/api-surface/playwright-sync-api.md +9 -0
  74. rustwright-0.1.0a2/docs/assets/banner.png +0 -0
  75. rustwright-0.1.0a2/docs/assets/logo.png +0 -0
  76. rustwright-0.1.0a2/docs/assets/mascot.png +0 -0
  77. rustwright-0.1.0a2/docs/async-design.md +177 -0
  78. rustwright-0.1.0a2/examples/quickstart.py +13 -0
  79. rustwright-0.1.0a2/pyproject.toml +51 -0
  80. rustwright-0.1.0a2/python/rustwright/__init__.py +56 -0
  81. rustwright-0.1.0a2/python/rustwright/__main__.py +5 -0
  82. rustwright-0.1.0a2/python/rustwright/_backend.py +58 -0
  83. rustwright-0.1.0a2/python/rustwright/_compat/__init__.py +83 -0
  84. rustwright-0.1.0a2/python/rustwright/_compat/cloakbrowser/__init__.py +92 -0
  85. rustwright-0.1.0a2/python/rustwright/_compat/patchright/__init__.py +2 -0
  86. rustwright-0.1.0a2/python/rustwright/_compat/patchright/__main__.py +5 -0
  87. rustwright-0.1.0a2/python/rustwright/_compat/patchright/_impl/__init__.py +1 -0
  88. rustwright-0.1.0a2/python/rustwright/_compat/patchright/_impl/_errors.py +4 -0
  89. rustwright-0.1.0a2/python/rustwright/_compat/patchright/async_api.py +2 -0
  90. rustwright-0.1.0a2/python/rustwright/_compat/patchright/pytest_plugin.py +2 -0
  91. rustwright-0.1.0a2/python/rustwright/_compat/patchright/sync_api.py +2 -0
  92. rustwright-0.1.0a2/python/rustwright/_compat/playwright/__init__.py +2 -0
  93. rustwright-0.1.0a2/python/rustwright/_compat/playwright/__main__.py +5 -0
  94. rustwright-0.1.0a2/python/rustwright/_compat/playwright/_impl/__init__.py +2 -0
  95. rustwright-0.1.0a2/python/rustwright/_compat/playwright/_impl/_errors.py +4 -0
  96. rustwright-0.1.0a2/python/rustwright/_compat/playwright/async_api.py +2 -0
  97. rustwright-0.1.0a2/python/rustwright/_compat/playwright/pytest_plugin.py +1 -0
  98. rustwright-0.1.0a2/python/rustwright/_compat/playwright/sync_api.py +2 -0
  99. rustwright-0.1.0a2/python/rustwright/_compat/pytest_playwright/__init__.py +3 -0
  100. rustwright-0.1.0a2/python/rustwright/_compat/pytest_playwright/py.typed +1 -0
  101. rustwright-0.1.0a2/python/rustwright/_compat/pytest_playwright/pytest_playwright.py +57 -0
  102. rustwright-0.1.0a2/python/rustwright/_devices.py +1034 -0
  103. rustwright-0.1.0a2/python/rustwright/async_api.py +5745 -0
  104. rustwright-0.1.0a2/python/rustwright/cli.py +1538 -0
  105. rustwright-0.1.0a2/python/rustwright/pytest_plugin.py +409 -0
  106. rustwright-0.1.0a2/python/rustwright/sync_api.py +29131 -0
  107. rustwright-0.1.0a2/src/lib.rs +12203 -0
  108. rustwright-0.1.0a2/tests/fixtures/mind2web_sample.json +57 -0
  109. rustwright-0.1.0a2/tests/test_antibot_benchmarks.py +399 -0
  110. rustwright-0.1.0a2/tests/test_mind2web_benchmark.py +121 -0
  111. rustwright-0.1.0a2/tests/test_playwright_compat_optin.py +173 -0
  112. rustwright-0.1.0a2/tests/test_playwright_parity_cases.py +70 -0
  113. rustwright-0.1.0a2/tests/test_project_status_tools.py +1299 -0
  114. rustwright-0.1.0a2/tests/test_rustwright_sync_api.py +36034 -0
  115. rustwright-0.1.0a2/tests/test_skyvern_alias_command.py +90 -0
  116. rustwright-0.1.0a2/tests/test_skyvern_cloud_overlay_tests.py +108 -0
  117. rustwright-0.1.0a2/tests/test_skyvern_prompt_overlay_smoke.py +103 -0
  118. rustwright-0.1.0a2/tests/test_skyvern_replacement_smoke.py +361 -0
  119. rustwright-0.1.0a2/tools/api_surface_audit.py +445 -0
  120. rustwright-0.1.0a2/tools/audit_skyvern_playwright_usage.py +856 -0
  121. rustwright-0.1.0a2/tools/check_benchmark_artifacts.py +164 -0
  122. rustwright-0.1.0a2/tools/check_cross_library_speed_goal.py +225 -0
  123. rustwright-0.1.0a2/tools/check_external_reliability_goal.py +274 -0
  124. rustwright-0.1.0a2/tools/check_launch_latency_claim.py +460 -0
  125. rustwright-0.1.0a2/tools/check_native_extension.py +60 -0
  126. rustwright-0.1.0a2/tools/check_phase1_gate.py +308 -0
  127. rustwright-0.1.0a2/tools/check_phase2_benchmark.py +235 -0
  128. rustwright-0.1.0a2/tools/check_testbox_visibility.py +294 -0
  129. rustwright-0.1.0a2/tools/docker_test.sh +211 -0
  130. rustwright-0.1.0a2/tools/docker_verify.sh +439 -0
  131. rustwright-0.1.0a2/tools/download_mind2web.py +92 -0
  132. rustwright-0.1.0a2/tools/import_mind2web.py +379 -0
  133. rustwright-0.1.0a2/tools/import_webvoyager.py +178 -0
  134. rustwright-0.1.0a2/tools/query_project_state.py +1046 -0
  135. rustwright-0.1.0a2/tools/render_benchmark_matrix.py +173 -0
  136. rustwright-0.1.0a2/tools/render_project_tables.py +307 -0
  137. rustwright-0.1.0a2/tools/run_antibot_benchmarks.py +1598 -0
  138. rustwright-0.1.0a2/tools/run_benchmark_matrix.py +561 -0
  139. rustwright-0.1.0a2/tools/run_benchmark_testbox.sh +223 -0
  140. rustwright-0.1.0a2/tools/run_mind2web_benchmark.py +1533 -0
  141. rustwright-0.1.0a2/tools/run_mind2web_matrix.py +286 -0
  142. rustwright-0.1.0a2/tools/run_mind2web_sharded.py +514 -0
  143. rustwright-0.1.0a2/tools/run_parity_cases.py +151 -0
  144. rustwright-0.1.0a2/tools/run_remote_docker_test.py +563 -0
  145. rustwright-0.1.0a2/tools/run_skyvern_alias_command.py +339 -0
  146. rustwright-0.1.0a2/tools/run_skyvern_cloud_overlay_tests.py +434 -0
  147. rustwright-0.1.0a2/tools/run_skyvern_prompt_overlay_smoke.py +210 -0
  148. rustwright-0.1.0a2/tools/run_skyvern_replacement_smoke.py +425 -0
  149. rustwright-0.1.0a2/tools/run_webvoyager_benchmark.py +604 -0
  150. rustwright-0.1.0a2/tools/run_webvoyager_matrix.py +299 -0
  151. rustwright-0.1.0a2/tools/rustwright_ts_binding.js +327 -0
@@ -0,0 +1,35 @@
1
+ .git
2
+ .audit-playwright
3
+ .venv
4
+ .venv_browser_harness_compare
5
+ .pytest_cache
6
+ .mypy_cache
7
+ .ruff_cache
8
+ target
9
+ dist
10
+ build
11
+ *.egg-info
12
+ __pycache__
13
+ *.py[cod]
14
+ *.so
15
+ *.dylib
16
+ *.dll
17
+ *.log
18
+ .DS_Store
19
+ downloads
20
+ test-results
21
+ playwright-report
22
+ videos
23
+ traces
24
+ screenshots
25
+ .benchmark-data
26
+ benchmark-data
27
+ datasets
28
+ data/Mind2Web
29
+ data/mind2web
30
+ data/WebVoyager
31
+ data/webvoyager
32
+ data/WebVoyager_data.jsonl
33
+ data/GAIA_web.jsonl
34
+ data/reference_answer.json
35
+ benchmarks/external
@@ -0,0 +1,34 @@
1
+ ---
2
+ name: Bug report
3
+ about: Report a reproducible Rustwright bug
4
+ title: "[Bug]: "
5
+ labels: bug
6
+ assignees: ""
7
+ ---
8
+
9
+ ## What happened?
10
+
11
+
12
+ ## Reproduction
13
+
14
+ ```python
15
+ # Minimal code or commands that reproduce the issue
16
+ ```
17
+
18
+ ## Expected behavior
19
+
20
+
21
+ ## Environment
22
+
23
+ - OS:
24
+ - Python version:
25
+ - Rust version:
26
+ - Node version, if relevant:
27
+ - Browser:
28
+ - Rustwright commit:
29
+
30
+ ## Logs
31
+
32
+ ```text
33
+
34
+ ```
@@ -0,0 +1,5 @@
1
+ blank_issues_enabled: false
2
+ contact_links:
3
+ - name: Questions and setup help
4
+ url: https://discord.gg/fG2XXEuQX3
5
+ about: Use the Skyvern Discord for questions, troubleshooting, and discussion.
@@ -0,0 +1,18 @@
1
+ ---
2
+ name: Feature request
3
+ about: Suggest a Rustwright capability or compatibility improvement
4
+ title: "[Feature]: "
5
+ labels: enhancement
6
+ assignees: ""
7
+ ---
8
+
9
+ ## What should Rustwright support?
10
+
11
+
12
+ ## Why is this useful?
13
+
14
+
15
+ ## Playwright behavior or reference
16
+
17
+
18
+ ## Additional context
@@ -0,0 +1,16 @@
1
+ ## What
2
+
3
+
4
+ ## Why
5
+
6
+
7
+ ## Testing
8
+
9
+ - [ ] `cargo check --locked`
10
+ - [ ] Targeted pytest, using the focused `-k` subset from `CONTRIBUTING.md`
11
+
12
+ ## Checklist
13
+
14
+ - [ ] I kept this change focused.
15
+ - [ ] I added or updated tests for behavior changes.
16
+ - [ ] I did not include private credentials, tokens, or internal-only artifacts.
@@ -0,0 +1,26 @@
1
+ version: 2
2
+ updates:
3
+ - package-ecosystem: cargo
4
+ directory: "/"
5
+ schedule:
6
+ interval: weekly
7
+
8
+ - package-ecosystem: cargo
9
+ directory: "/node"
10
+ schedule:
11
+ interval: weekly
12
+
13
+ - package-ecosystem: pip
14
+ directory: "/"
15
+ schedule:
16
+ interval: weekly
17
+
18
+ - package-ecosystem: npm
19
+ directory: "/node"
20
+ schedule:
21
+ interval: weekly
22
+
23
+ - package-ecosystem: github-actions
24
+ directory: "/"
25
+ schedule:
26
+ interval: weekly
@@ -0,0 +1,56 @@
1
+ Please review this pull request for the **Rustwright** repository and give concise, actionable feedback.
2
+
3
+ Rustwright is a Rust engine (PyO3 for Python, napi-rs for Node) that speaks raw Chrome DevTools Protocol and exposes a **Playwright-compatible** API. It is alpha, Chromium-only, MIT-licensed. The Rust core lives in `src/lib.rs`; the Python API in `python/rustwright/`; the Node bindings in `node/`.
4
+
5
+ Review for:
6
+
7
+ - **Correctness & bugs** — logic errors, unhandled edge cases, error handling. Pay special attention to:
8
+ - the FFI boundaries — PyO3 `Bound<'py, T>` usage and GIL handling (`py.detach` around blocking work), and napi async/threadsafe-function usage;
9
+ - the CDP layer — session/frame routing (OOPIF), the tokio-tungstenite WebSocket transport, and message dispatch/correlation.
10
+ - **Memory & concurrency safety** — any new `unsafe`; `unwrap()`/`expect()`/`panic!` on fallible paths that could crash a user's process across the FFI boundary; races or deadlocks in the async CDP client; holding the GIL across I/O.
11
+ - **Security** (this is browser-automation software) — untrusted-input handling, JS/CDP injection, and anything that silently changes the **trusted-input-by-default** behavior. Rustwright must **not overclaim on anti-detection**: flag any change that adds "undetectable" / "bypass Cloudflare" / "stealth" style claims, or that weakens the honest signal-hygiene framing (it currently passes 3/4 public fingerprint targets and says so plainly).
12
+ - **API compatibility** — the Python/Node surface mirrors Playwright. Flag divergence from Playwright's shape/behavior that isn't justified, and missing parity coverage.
13
+ - **Performance** — avoidable allocations/copies on the per-CDP-message hot path; blocking calls inside async paths.
14
+ - **Tests** — is the change covered by the pytest suite (`tests/`)? Does it warrant a parity test against real Playwright?
15
+ - **Honesty of claims** — if the PR touches README/docs/benchmarks, verify the claims match what the code and tests actually support (no inflated numbers, no unverified platform/behavior claims, MSRV kept accurate).
16
+
17
+ Use `CONTRIBUTING.md` (and `CLAUDE.md` if present) for conventions. Be constructive.
18
+
19
+ Keep the review **concise and scannable**:
20
+ - Start with a 1–2 sentence overall assessment.
21
+ - Group feedback by severity using GitHub `<details>`/`<summary>` collapsible sections.
22
+ - Expand 🔴 Critical by default; keep 🟡 Suggestions and 📝 Minor collapsed.
23
+ - If there are no issues, post a brief LGTM (with the marker).
24
+
25
+ Format:
26
+
27
+ ```
28
+ <!-- claude-code-review -->
29
+ ## Summary
30
+ Brief overall assessment.
31
+
32
+ <details open>
33
+ <summary>🔴 Critical Issues (X)</summary>
34
+
35
+ - ...
36
+
37
+ </details>
38
+
39
+ <details>
40
+ <summary>🟡 Suggestions (X)</summary>
41
+
42
+ - ...
43
+
44
+ </details>
45
+
46
+ <details>
47
+ <summary>📝 Minor / Style (X)</summary>
48
+
49
+ - ...
50
+
51
+ </details>
52
+ ```
53
+
54
+ IMPORTANT: Start your comment with the exact marker `<!-- claude-code-review -->` on the first line — this lets a later commit replace the previous review instead of stacking comments.
55
+
56
+ Use `gh pr comment` (via your Bash tool) to post the review as a single comment on the PR.
@@ -0,0 +1,51 @@
1
+ name: Rustwright Benchmark Testbox
2
+
3
+ on:
4
+ workflow_dispatch:
5
+ inputs:
6
+ testbox_id:
7
+ type: string
8
+ description: "Testbox session ID"
9
+ required: true
10
+
11
+ jobs:
12
+ benchmark:
13
+ if: github.repository == 'Skyvern-AI/rustwright'
14
+ runs-on: blacksmith-4vcpu-ubuntu-2404
15
+ timeout-minutes: 120
16
+ env:
17
+ RUSTWRIGHT_DOCKER_IMAGE: rustwright-verify-testbox
18
+ TEST_DOCKER_MEMORY_LIMIT: 8g
19
+ INSTALL_BROWSER_HARNESS: "1"
20
+ INSTALL_PUPPETEER: "1"
21
+ DOCKER_BUILDKIT: "1"
22
+ steps:
23
+ - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
24
+
25
+ - name: Begin Testbox
26
+ uses: useblacksmith/begin-testbox@233448af4bfdc6fca509a7f0974411ac6d8a8043 # v2
27
+ with:
28
+ testbox_id: ${{ inputs.testbox_id }}
29
+
30
+ - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
31
+ with:
32
+ python-version: "3.13"
33
+
34
+ - name: Verify Docker
35
+ run: |
36
+ docker version
37
+ docker info
38
+
39
+ - name: Build benchmark Docker image
40
+ run: |
41
+ tools/docker_test.sh build .
42
+
43
+ - name: Run Testbox
44
+ uses: useblacksmith/run-testbox@5ca05834db1d3813554d1dd109e5f2087a8d7cbc # v2
45
+ if: always()
46
+ env:
47
+ RUSTWRIGHT_DOCKER_IMAGE: rustwright-verify-testbox
48
+ TEST_DOCKER_MEMORY_LIMIT: 8g
49
+ INSTALL_BROWSER_HARNESS: "1"
50
+ INSTALL_PUPPETEER: "1"
51
+ DOCKER_BUILDKIT: "1"
@@ -0,0 +1,297 @@
1
+ name: Rustwright Benchmarks
2
+
3
+ on:
4
+ pull_request:
5
+ types:
6
+ - opened
7
+ - synchronize
8
+ - reopened
9
+ - ready_for_review
10
+ - labeled
11
+ workflow_dispatch:
12
+ inputs:
13
+ benchmark:
14
+ type: choice
15
+ description: "Benchmark workload to run"
16
+ required: true
17
+ default: strict
18
+ options:
19
+ - strict
20
+ - equivalent
21
+ - mind2web-sharded
22
+ - defensible-speed
23
+ implementations:
24
+ type: string
25
+ description: "Comma-separated implementations, or all"
26
+ required: true
27
+ default: "rustwright,playwright"
28
+ repetitions:
29
+ type: string
30
+ description: "Benchmark repetitions"
31
+ required: true
32
+ default: "1"
33
+ iterations:
34
+ type: string
35
+ description: "Iterations per benchmark run"
36
+ required: true
37
+ default: "1"
38
+ lifecycle:
39
+ type: choice
40
+ description: "bench-full lifecycle"
41
+ required: true
42
+ default: warm-browser
43
+ options:
44
+ - warm-browser
45
+ - warm-page
46
+ - cold-browser
47
+ - cold-container
48
+ case:
49
+ type: string
50
+ description: "Optional strict/equivalent case name"
51
+ required: false
52
+ default: ""
53
+ max_tasks:
54
+ type: string
55
+ description: "Optional Mind2Web task cap"
56
+ required: false
57
+ default: ""
58
+ shard_size:
59
+ type: string
60
+ description: "Mind2Web shard size"
61
+ required: false
62
+ default: "25"
63
+ defensible_lifecycles:
64
+ type: string
65
+ description: "Comma-separated lifecycles for defensible-speed"
66
+ required: false
67
+ default: "warm-browser,warm-page,cold-browser"
68
+ strict_repetitions:
69
+ type: string
70
+ description: "Strict-suite repetitions for defensible-speed; defaults to repetitions"
71
+ required: false
72
+ default: ""
73
+ strict_iterations:
74
+ type: string
75
+ description: "Strict-suite iterations for defensible-speed"
76
+ required: false
77
+ default: "1"
78
+ enforce_phase2:
79
+ type: boolean
80
+ description: "Fail the workflow if strict Phase 2 acceptance checks fail"
81
+ required: false
82
+ default: false
83
+ runner_label:
84
+ type: choice
85
+ description: "Runner label for provenance/debug runs; Blacksmith remains the default"
86
+ required: false
87
+ default: blacksmith-4vcpu-ubuntu-2404
88
+ options:
89
+ - blacksmith-4vcpu-ubuntu-2404
90
+ - ubuntu-latest
91
+
92
+ jobs:
93
+ benchmark:
94
+ if: >-
95
+ ${{
96
+ github.repository == 'Skyvern-AI/rustwright' &&
97
+ (
98
+ github.event_name != 'pull_request' ||
99
+ github.event.pull_request.head.repo.full_name == github.repository
100
+ ) &&
101
+ (
102
+ github.event_name == 'workflow_dispatch' ||
103
+ vars.RUSTWRIGHT_RUN_PR_BENCHMARKS == '1' ||
104
+ (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-benchmarks'))
105
+ )
106
+ }}
107
+ runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.runner_label || vars.RUSTWRIGHT_BENCHMARK_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }}
108
+ timeout-minutes: 480
109
+ env:
110
+ RUSTWRIGHT_DOCKER_IMAGE: rustwright-verify-github-benchmark
111
+ TEST_DOCKER_MEMORY_LIMIT: 8g
112
+ INSTALL_BROWSER_HARNESS: "1"
113
+ INSTALL_PUPPETEER: "1"
114
+ DOCKER_BUILDKIT: "1"
115
+ BENCHMARK_KIND: ${{ github.event_name == 'workflow_dispatch' && inputs.benchmark || vars.RUSTWRIGHT_PR_BENCHMARK || 'equivalent' }}
116
+ IMPLEMENTATIONS: ${{ github.event_name == 'workflow_dispatch' && inputs.implementations || vars.RUSTWRIGHT_PR_BENCHMARK_IMPLEMENTATIONS || 'all' }}
117
+ REPETITIONS: ${{ github.event_name == 'workflow_dispatch' && inputs.repetitions || vars.RUSTWRIGHT_PR_BENCHMARK_REPETITIONS || '1' }}
118
+ ITERATIONS: ${{ github.event_name == 'workflow_dispatch' && inputs.iterations || vars.RUSTWRIGHT_PR_BENCHMARK_ITERATIONS || '1' }}
119
+ BENCHMARK_FULL_ITERATIONS: ${{ github.event_name == 'workflow_dispatch' && inputs.iterations || vars.RUSTWRIGHT_PR_BENCHMARK_ITERATIONS || '1' }}
120
+ LIFECYCLE: ${{ github.event_name == 'workflow_dispatch' && inputs.lifecycle || vars.RUSTWRIGHT_PR_BENCHMARK_LIFECYCLE || 'warm-browser' }}
121
+ CASE_NAME: ${{ github.event_name == 'workflow_dispatch' && inputs.case || vars.RUSTWRIGHT_PR_BENCHMARK_CASE || '' }}
122
+ MAX_TASKS: ${{ github.event_name == 'workflow_dispatch' && inputs.max_tasks || vars.RUSTWRIGHT_PR_MIND2WEB_MAX_TASKS || '' }}
123
+ SHARD_SIZE: ${{ github.event_name == 'workflow_dispatch' && inputs.shard_size || vars.RUSTWRIGHT_PR_MIND2WEB_SHARD_SIZE || '25' }}
124
+ DEFENSIBLE_LIFECYCLES: ${{ github.event_name == 'workflow_dispatch' && inputs.defensible_lifecycles || vars.RUSTWRIGHT_BENCHMARK_DEFENSIBLE_LIFECYCLES || 'warm-browser,warm-page,cold-browser' }}
125
+ DEFENSIBLE_STRICT_REPETITIONS: ${{ github.event_name == 'workflow_dispatch' && inputs.strict_repetitions || vars.RUSTWRIGHT_BENCHMARK_DEFENSIBLE_STRICT_REPETITIONS || '' }}
126
+ DEFENSIBLE_STRICT_ITERATIONS: ${{ github.event_name == 'workflow_dispatch' && inputs.strict_iterations || vars.RUSTWRIGHT_BENCHMARK_DEFENSIBLE_STRICT_ITERATIONS || '1' }}
127
+ ENFORCE_PHASE2: ${{ github.event_name == 'workflow_dispatch' && inputs.enforce_phase2 || vars.RUSTWRIGHT_BENCHMARK_ENFORCE_PHASE2 || '0' }}
128
+ BENCHMARK_RUNNER_LABEL: ${{ github.event_name == 'workflow_dispatch' && inputs.runner_label || vars.RUSTWRIGHT_BENCHMARK_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }}
129
+ steps:
130
+ - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
131
+
132
+ - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
133
+ with:
134
+ python-version: "3.13"
135
+
136
+ - name: Verify Docker
137
+ run: |
138
+ docker version
139
+ docker info
140
+
141
+ - name: Print benchmark configuration
142
+ run: |
143
+ {
144
+ echo "event=${GITHUB_EVENT_NAME}"
145
+ echo "benchmark=${BENCHMARK_KIND}"
146
+ echo "implementations=${IMPLEMENTATIONS}"
147
+ echo "repetitions=${REPETITIONS}"
148
+ echo "iterations=${ITERATIONS}"
149
+ echo "lifecycle=${LIFECYCLE}"
150
+ echo "case=${CASE_NAME}"
151
+ echo "max_tasks=${MAX_TASKS}"
152
+ echo "shard_size=${SHARD_SIZE}"
153
+ echo "defensible_lifecycles=${DEFENSIBLE_LIFECYCLES}"
154
+ echo "defensible_strict_repetitions=${DEFENSIBLE_STRICT_REPETITIONS}"
155
+ echo "defensible_strict_iterations=${DEFENSIBLE_STRICT_ITERATIONS}"
156
+ echo "enforce_phase2=${ENFORCE_PHASE2}"
157
+ echo "runner_label=${BENCHMARK_RUNNER_LABEL}"
158
+ echo "docker_memory=${TEST_DOCKER_MEMORY_LIMIT}"
159
+ }
160
+
161
+ - name: Build benchmark Docker image
162
+ run: |
163
+ tools/docker_test.sh build .
164
+
165
+ - name: Prepare Mind2Web manifest
166
+ if: ${{ env.BENCHMARK_KIND == 'mind2web-sharded' }}
167
+ run: |
168
+ set -euo pipefail
169
+ python tools/download_mind2web.py --all-train --retries 5 --timeout 300 --json
170
+ python tools/import_mind2web.py \
171
+ --source .benchmark-data/raw/mind2web/data/train \
172
+ --output .benchmark-data/manifests/mind2web-train-100pct-action-fixtures.json \
173
+ --include-action-fixtures \
174
+ --json
175
+
176
+ - name: Run benchmark
177
+ run: |
178
+ set -euo pipefail
179
+ mkdir -p .benchmark-data/results
180
+ timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
181
+
182
+ impl_args=()
183
+ if [ "$IMPLEMENTATIONS" != "all" ]; then
184
+ IFS=',' read -ra impls <<< "$IMPLEMENTATIONS"
185
+ for impl in "${impls[@]}"; do
186
+ impl="$(echo "$impl" | xargs)"
187
+ if [ -n "$impl" ]; then
188
+ impl_args+=(--impl "$impl")
189
+ fi
190
+ done
191
+ fi
192
+
193
+ case_args=()
194
+ if [ -n "$CASE_NAME" ]; then
195
+ case_args+=(--case "$CASE_NAME")
196
+ fi
197
+
198
+ if [ "$BENCHMARK_KIND" = "defensible-speed" ]; then
199
+ if [ -n "$CASE_NAME" ]; then
200
+ echo "defensible-speed uses the full predeclared case set; do not pass case filters" >&2
201
+ exit 2
202
+ fi
203
+
204
+ IFS=',' read -ra lifecycles <<< "$DEFENSIBLE_LIFECYCLES"
205
+ for benchmark_lifecycle in "${lifecycles[@]}"; do
206
+ benchmark_lifecycle="$(echo "$benchmark_lifecycle" | xargs)"
207
+ if [ -z "$benchmark_lifecycle" ]; then
208
+ continue
209
+ fi
210
+ case "$benchmark_lifecycle" in
211
+ warm-browser|warm-page|cold-browser|cold-container)
212
+ ;;
213
+ *)
214
+ echo "Unsupported defensible lifecycle: $benchmark_lifecycle" >&2
215
+ exit 2
216
+ ;;
217
+ esac
218
+
219
+ BENCHMARK_FULL_ITERATIONS="$ITERATIONS" tools/docker_test.sh bench-full \
220
+ "${impl_args[@]}" \
221
+ --suite equivalent \
222
+ --lifecycle "$benchmark_lifecycle" \
223
+ --repetitions "$REPETITIONS" \
224
+ --output ".benchmark-data/results/defensible-speed-equivalent-${benchmark_lifecycle}-${REPETITIONS}x${ITERATIONS}-${timestamp}.json" \
225
+ --json
226
+ done
227
+
228
+ strict_repetitions="$DEFENSIBLE_STRICT_REPETITIONS"
229
+ if [ -z "$strict_repetitions" ]; then
230
+ strict_repetitions="$REPETITIONS"
231
+ fi
232
+ BENCHMARK_FULL_ITERATIONS="$DEFENSIBLE_STRICT_ITERATIONS" tools/docker_test.sh bench-full \
233
+ --impl rustwright \
234
+ --impl playwright \
235
+ --suite strict \
236
+ --lifecycle warm-browser \
237
+ --repetitions "$strict_repetitions" \
238
+ --output ".benchmark-data/results/defensible-speed-strict-warm-browser-${strict_repetitions}x${DEFENSIBLE_STRICT_ITERATIONS}-${timestamp}.json" \
239
+ --json
240
+ elif [ "$BENCHMARK_KIND" = "mind2web-sharded" ]; then
241
+ max_task_args=()
242
+ if [ -n "$MAX_TASKS" ]; then
243
+ max_task_args+=(--max-tasks "$MAX_TASKS")
244
+ fi
245
+ python tools/run_mind2web_sharded.py \
246
+ "${impl_args[@]}" \
247
+ --shard-size "$SHARD_SIZE" \
248
+ --repetitions "$REPETITIONS" \
249
+ --iterations "$ITERATIONS" \
250
+ "${max_task_args[@]}" \
251
+ --output ".benchmark-data/results/mind2web-sharded-${timestamp}.json" \
252
+ --json
253
+ else
254
+ tools/docker_test.sh bench-full \
255
+ "${impl_args[@]}" \
256
+ --suite "$BENCHMARK_KIND" \
257
+ --lifecycle "$LIFECYCLE" \
258
+ --repetitions "$REPETITIONS" \
259
+ "${case_args[@]}" \
260
+ --output ".benchmark-data/results/bench-full-${BENCHMARK_KIND}-${LIFECYCLE}-${timestamp}.json" \
261
+ --json
262
+ fi
263
+
264
+ - name: Check strict Phase 2 benchmark artifacts
265
+ if: always()
266
+ run: |
267
+ set -euo pipefail
268
+ mkdir -p .benchmark-data/reports
269
+
270
+ enforce=0
271
+ case "$ENFORCE_PHASE2" in
272
+ true|True|TRUE|1|yes|YES)
273
+ enforce=1
274
+ ;;
275
+ esac
276
+
277
+ args=(
278
+ --source github-actions
279
+ --runner "$BENCHMARK_RUNNER_LABEL"
280
+ --artifact rustwright-benchmark-results
281
+ --run-url "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
282
+ --json
283
+ )
284
+ if [ "$enforce" = "1" ]; then
285
+ args+=(--enforce-phase2)
286
+ fi
287
+ python tools/check_benchmark_artifacts.py "${args[@]}"
288
+
289
+ - name: Upload benchmark results
290
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
291
+ if: always()
292
+ with:
293
+ name: rustwright-benchmark-results
294
+ path: |
295
+ .benchmark-data/results/
296
+ .benchmark-data/reports/
297
+ if-no-files-found: warn
@@ -0,0 +1,87 @@
1
+ name: Claude Code Review
2
+
3
+ # Automated PR review by Claude, on every opened/updated pull request.
4
+ # Requires a repository secret CLAUDE_CODE_OAUTH_TOKEN (create with `claude setup-token`).
5
+ # If the secret is not set, the review is skipped (the job still succeeds) instead of failing.
6
+ # The review prompt is loaded from the BASE branch so untrusted PRs cannot alter
7
+ # review behavior by editing the prompt file in the same PR.
8
+
9
+ on:
10
+ pull_request:
11
+ types: [opened, synchronize]
12
+
13
+ jobs:
14
+ claude-review:
15
+ # Skip PRs opened by Claude itself or by GitHub Actions to avoid review loops.
16
+ if: |
17
+ !contains(fromJSON('["claude","claude[bot]","github-actions[bot]"]'), github.event.pull_request.user.login)
18
+ runs-on: ubuntu-latest
19
+ permissions:
20
+ contents: read
21
+ pull-requests: write
22
+ issues: read
23
+ id-token: write
24
+ steps:
25
+ - name: Check for Claude token
26
+ id: gate
27
+ env:
28
+ CLAUDE_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
29
+ run: |
30
+ if [ -n "$CLAUDE_TOKEN" ]; then
31
+ echo "enabled=true" >> "$GITHUB_OUTPUT"
32
+ else
33
+ echo "enabled=false" >> "$GITHUB_OUTPUT"
34
+ echo "::notice::CLAUDE_CODE_OAUTH_TOKEN is not set — skipping Claude review. Add it with 'gh secret set CLAUDE_CODE_OAUTH_TOKEN'."
35
+ fi
36
+
37
+ - name: Checkout repository
38
+ if: steps.gate.outputs.enabled == 'true'
39
+ uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
40
+ with:
41
+ fetch-depth: 1
42
+
43
+ - name: Delete previous Claude review comment
44
+ if: steps.gate.outputs.enabled == 'true'
45
+ env:
46
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
47
+ run: |
48
+ # Remove any prior review comment (identified by the marker) so each
49
+ # push replaces the review instead of stacking comments.
50
+ gh api "repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments" \
51
+ --jq '.[] | select(.body | contains("<!-- claude-code-review -->")) | .id' \
52
+ | while read -r id; do
53
+ if [ -n "$id" ]; then
54
+ echo "Deleting previous Claude review comment: $id"
55
+ gh api "repos/${{ github.repository }}/issues/comments/$id" -X DELETE
56
+ fi
57
+ done
58
+
59
+ - name: Load review prompt from base branch
60
+ if: steps.gate.outputs.enabled == 'true'
61
+ id: prompt
62
+ run: |
63
+ # Load the prompt from the PR's BASE commit (trusted), not the PR head.
64
+ git fetch --no-tags --depth=1 origin "${{ github.event.pull_request.base.sha }}"
65
+ BASE="${{ github.event.pull_request.base.sha }}"
66
+ DELIM="PROMPT_EOF_$(openssl rand -hex 16)"
67
+ {
68
+ printf 'content<<%s\n' "$DELIM"
69
+ git show "${BASE}:.github/prompts/code-review.md"
70
+ printf '%s\n' "$DELIM"
71
+ } >> "$GITHUB_OUTPUT"
72
+
73
+ - name: Run Claude Code Review
74
+ if: steps.gate.outputs.enabled == 'true'
75
+ uses: anthropics/claude-code-action@ff9acae5886d41a99ed4ec14b7dc147d55834722 # v1.0.77
76
+ with:
77
+ claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
78
+ allowed_bots: "dependabot[bot],github-actions[bot],renovate[bot]"
79
+ prompt: |
80
+ REPO: ${{ github.repository }}
81
+ PR NUMBER: ${{ github.event.pull_request.number }}
82
+
83
+ ${{ steps.prompt.outputs.content }}
84
+ # Read-only review: Claude may inspect the repo and the PR, and post one comment.
85
+ claude_args: >-
86
+ --max-turns 30
87
+ --allowed-tools "Read,Glob,Grep,Bash(gh pr comment:*),Bash(gh pr diff:*),Bash(gh pr view:*),Bash(gh pr list:*),Bash(gh issue view:*)"