webstone 0.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (488) hide show
  1. harness/benchmark_execution.py +445 -0
  2. harness/benchmarks/__init__.py +1 -0
  3. harness/benchmarks/cards/expanded-swde.md +45 -0
  4. harness/benchmarks/cards/headless-browser.md +112 -0
  5. harness/benchmarks/cards/languages.json +1018 -0
  6. harness/benchmarks/cards/liveweb-ie.md +49 -0
  7. harness/benchmarks/cards/metadata.json +485 -0
  8. harness/benchmarks/cards/newsxlm.md +47 -0
  9. harness/benchmarks/cards/simpleqa-verified.md +75 -0
  10. harness/benchmarks/cards/swde.json +47 -0
  11. harness/benchmarks/cards/trajectory-replay.md +82 -0
  12. harness/benchmarks/cards/wcxb.md +116 -0
  13. harness/benchmarks/cards/web-extract.md +119 -0
  14. harness/benchmarks/cards/webarena-replay.md +91 -0
  15. harness/benchmarks/cards/webfetch.md +297 -0
  16. harness/benchmarks/cards/webmainbench.md +63 -0
  17. harness/benchmarks/cards/websearch.md +98 -0
  18. harness/benchmarks/cards/zyte-product-extraction.json +42 -0
  19. harness/benchmarks/code-migration.json +4554 -0
  20. harness/benchmarks/data-lock.json +129219 -0
  21. harness/benchmarks/data.py +397 -0
  22. harness/benchmarks/pinning.py +328 -0
  23. harness/benchmarks/publication.py +780 -0
  24. harness/benchmarks/scoring-regressions.json +90025 -0
  25. harness/browser_use/__init__.py +5 -0
  26. harness/browser_use/__main__.py +4 -0
  27. harness/browser_use/assets/evidence-viewer.js +1 -0
  28. harness/browser_use/assets/report.css +63 -0
  29. harness/browser_use/assets/report.js +12 -0
  30. harness/browser_use/assets/webarena-autonomous-evaluation-flow.png +0 -0
  31. harness/browser_use/cli.py +46 -0
  32. harness/browser_use/failure_cause_review.py +160 -0
  33. harness/browser_use/messages.json +699 -0
  34. harness/browser_use/model.py +258 -0
  35. harness/browser_use/publication.py +191 -0
  36. harness/browser_use/report.py +4261 -0
  37. harness/browser_use/report_check.py +982 -0
  38. harness/browser_use/report_i18n.py +175 -0
  39. harness/browser_use/report_package.py +100 -0
  40. harness/browser_use/report_publish.py +194 -0
  41. harness/browser_use/report_visual.py +265 -0
  42. harness/browser_use/review.py +170 -0
  43. harness/browser_use/runner.py +811 -0
  44. harness/browser_use/selection.py +346 -0
  45. harness/container_isolation.py +46 -0
  46. harness/data_home.py +15 -0
  47. harness/distribution.py +46 -0
  48. harness/distribution_data/configs/providers.headless.json +120 -0
  49. harness/distribution_data/evaluation/browser-use-public/adoption-evidence.json +122 -0
  50. harness/distribution_data/evaluation/browser-use-public/benchmark-catalog.json +244 -0
  51. harness/distribution_data/evaluation/browser-use-public/report-changes.json +55 -0
  52. harness/distribution_data/evaluation/webmainbench-full/scoring-regressions.json +43 -0
  53. harness/distribution_data/pyproject.toml +209 -0
  54. harness/distribution_data/uv.lock +5088 -0
  55. harness/env_file.py +125 -0
  56. harness/environments/__init__.py +1 -0
  57. harness/environments/browser.py +150 -0
  58. harness/environments/webarena/__init__.py +1 -0
  59. harness/environments/webarena/backend.py +108 -0
  60. harness/environments/webarena/cli.py +88 -0
  61. harness/environments/webarena/config.py +280 -0
  62. harness/environments/webarena/har.py +835 -0
  63. harness/environments/webarena/manager.py +204 -0
  64. harness/environments/webarena/map_archives.py +60 -0
  65. harness/environments/webarena/map_check.py +103 -0
  66. harness/environments/webarena/map_detail.py +149 -0
  67. harness/environments/webarena/map_setup.py +537 -0
  68. harness/environments/webarena/rainbond.py +1185 -0
  69. harness/environments/webarena/runtime/php-fpm-qemu.ini +12 -0
  70. harness/environments/webarena/runtime.py +736 -0
  71. harness/environments/webarena/topology.json +81 -0
  72. harness/environments/webarena/topology.py +238 -0
  73. harness/fix_log.py +305 -0
  74. harness/lexbench/__init__.py +1 -0
  75. harness/lexbench/__main__.py +4 -0
  76. harness/lexbench/assets/bundle-loader.js +75 -0
  77. harness/lexbench/assets/render_contract.cjs +382 -0
  78. harness/lexbench/assets/ui_check.js +636 -0
  79. harness/lexbench/charts.py +469 -0
  80. harness/lexbench/cli.py +45 -0
  81. harness/lexbench/pack.py +136 -0
  82. harness/lexbench/report.py +2132 -0
  83. harness/lexbench/reproduction.py +234 -0
  84. harness/lexbench/resource_evidence.py +199 -0
  85. harness/lexbench/resource_matrix.py +298 -0
  86. harness/lexbench/resource_profiles.py +315 -0
  87. harness/lexbench/resource_summary.py +277 -0
  88. harness/lexbench/template/README.md +11 -0
  89. harness/lexbench/template/app.js +1607 -0
  90. harness/lexbench/template/index.html +51 -0
  91. harness/lexbench/template/messages.json +744 -0
  92. harness/lexbench/template/style.css +905 -0
  93. harness/lexbench/verification.py +1239 -0
  94. harness/licenses/Lexbench-Apache-2.0.txt +201 -0
  95. harness/licenses/WCXB-CC-BY-4.0.txt +25 -0
  96. harness/numbers.py +14 -0
  97. harness/package_gate.py +84 -0
  98. harness/platform_capabilities.py +37 -0
  99. harness/record_io.py +107 -0
  100. harness/report_languages.py +246 -0
  101. harness/reporting/__init__.py +1 -0
  102. harness/reporting/assets.py +182 -0
  103. harness/reporting/certified_cli.py +24 -0
  104. harness/reporting/links.py +168 -0
  105. harness/reporting/locales.py +30 -0
  106. harness/reporting/mermaid.py +28 -0
  107. harness/reporting/packs.py +336 -0
  108. harness/reporting/presentation.py +176 -0
  109. harness/reporting/pricing.py +29 -0
  110. harness/reporting/privacy.py +95 -0
  111. harness/reporting/publication.py +561 -0
  112. harness/reporting/publication_target.py +92 -0
  113. harness/reporting/publish.py +894 -0
  114. harness/reporting/release_check.py +51 -0
  115. harness/reporting/source_policy.py +64 -0
  116. harness/reporting/templates.py +109 -0
  117. harness/reporting/versions.py +135 -0
  118. harness/reporting/visual.py +215 -0
  119. harness/resource_accounting.py +256 -0
  120. harness/resource_calibration.py +287 -0
  121. harness/results/__init__.py +5 -0
  122. harness/results/__main__.py +3 -0
  123. harness/results/cli.py +123 -0
  124. harness/results/client.py +783 -0
  125. harness/results/versions.py +72 -0
  126. harness/review_runner.py +122 -0
  127. harness/trajectory_replay/__init__.py +5 -0
  128. harness/trajectory_replay/__main__.py +4 -0
  129. harness/trajectory_replay/assets/compact-data-fetch-adapter.js +25 -0
  130. harness/trajectory_replay/assets/compact-detail-loader.js +11 -0
  131. harness/trajectory_replay/assets/compact-evidence-loader.js +11 -0
  132. harness/trajectory_replay/assets/curated-execution-filter.js +38 -0
  133. harness/trajectory_replay/assets/curated-report.css +6 -0
  134. harness/trajectory_replay/assets/evidence-dialog.js +40 -0
  135. harness/trajectory_replay/assets/public-webarena-benchmark-tabs.js +4 -0
  136. harness/trajectory_replay/assets/public-webarena-comparison.js +4 -0
  137. harness/trajectory_replay/assets/public-webarena-execution-filter.js +39 -0
  138. harness/trajectory_replay/assets/public-webarena-report.css +13 -0
  139. harness/trajectory_replay/assets/suite-filter.js +7 -0
  140. harness/trajectory_replay/assets/webarena-article.css +1 -0
  141. harness/trajectory_replay/assets/webarena-catalog.css +12 -0
  142. harness/trajectory_replay/assets/webarena-report.css +52 -0
  143. harness/trajectory_replay/assets/webarena-report.js +280 -0
  144. harness/trajectory_replay/browser_use_seeds.py +375 -0
  145. harness/trajectory_replay/catalog_acceptance.py +717 -0
  146. harness/trajectory_replay/catalog_action_audit.py +190 -0
  147. harness/trajectory_replay/catalog_binding.py +198 -0
  148. harness/trajectory_replay/catalog_comparison.py +423 -0
  149. harness/trajectory_replay/catalog_consolidation.py +330 -0
  150. harness/trajectory_replay/catalog_contracts.py +833 -0
  151. harness/trajectory_replay/catalog_release.py +80 -0
  152. harness/trajectory_replay/catalog_runner.py +832 -0
  153. harness/trajectory_replay/cli.py +309 -0
  154. harness/trajectory_replay/core/__init__.py +1 -0
  155. harness/trajectory_replay/core/diagnostics.py +263 -0
  156. harness/trajectory_replay/core/docker_runner.py +374 -0
  157. harness/trajectory_replay/core/executor.py +72 -0
  158. harness/trajectory_replay/core/metrics.py +365 -0
  159. harness/trajectory_replay/core/report.py +113 -0
  160. harness/trajectory_replay/core/suite.py +190 -0
  161. harness/trajectory_replay/diagnostics.py +685 -0
  162. harness/trajectory_replay/evaluator_transition.py +434 -0
  163. harness/trajectory_replay/executor.py +165 -0
  164. harness/trajectory_replay/expert_seeds.py +188 -0
  165. harness/trajectory_replay/failure_matrix.py +280 -0
  166. harness/trajectory_replay/interpretation.py +186 -0
  167. harness/trajectory_replay/messages.json +1002 -0
  168. harness/trajectory_replay/messages.py +11 -0
  169. harness/trajectory_replay/metrics.py +35 -0
  170. harness/trajectory_replay/participants.py +97 -0
  171. harness/trajectory_replay/playwright_tool.py +1331 -0
  172. harness/trajectory_replay/public_webarena/__init__.py +5 -0
  173. harness/trajectory_replay/public_webarena/cli.py +204 -0
  174. harness/trajectory_replay/public_webarena/config.py +79 -0
  175. harness/trajectory_replay/public_webarena/diagnostics.py +1163 -0
  176. harness/trajectory_replay/public_webarena/evaluators.py +155 -0
  177. harness/trajectory_replay/public_webarena/executor.py +695 -0
  178. harness/trajectory_replay/public_webarena/har.py +393 -0
  179. harness/trajectory_replay/public_webarena/manifest_repair.py +393 -0
  180. harness/trajectory_replay/public_webarena/metrics.py +47 -0
  181. harness/trajectory_replay/public_webarena/publication.py +492 -0
  182. harness/trajectory_replay/public_webarena/qualification.py +2765 -0
  183. harness/trajectory_replay/public_webarena/release.py +171 -0
  184. harness/trajectory_replay/public_webarena/report.py +1903 -0
  185. harness/trajectory_replay/public_webarena/response_builder.py +327 -0
  186. harness/trajectory_replay/public_webarena/result_audit.py +185 -0
  187. harness/trajectory_replay/public_webarena/runner.py +908 -0
  188. harness/trajectory_replay/public_webarena/suite.py +410 -0
  189. harness/trajectory_replay/public_webarena/targets.py +93 -0
  190. harness/trajectory_replay/public_webarena/webarena.py +735 -0
  191. harness/trajectory_replay/public_webarena/webarena_environment.py +735 -0
  192. harness/trajectory_replay/remote_execution.py +1581 -0
  193. harness/trajectory_replay/remote_runtime.py +1071 -0
  194. harness/trajectory_replay/report.py +1236 -0
  195. harness/trajectory_replay/report_compaction.py +217 -0
  196. harness/trajectory_replay/report_inputs.py +170 -0
  197. harness/trajectory_replay/report_languages.py +14 -0
  198. harness/trajectory_replay/resources/fixtures/app.html +11 -0
  199. harness/trajectory_replay/resources/fixtures/catalog.html +9 -0
  200. harness/trajectory_replay/resources/fixtures/core.html +18 -0
  201. harness/trajectory_replay/resources/fixtures/dynamic.html +11 -0
  202. harness/trajectory_replay/resources/fixtures/forms.html +11 -0
  203. harness/trajectory_replay/resources/fixtures/interaction-space.html +142 -0
  204. harness/trajectory_replay/resources/fixtures/page2.html +3 -0
  205. harness/trajectory_replay/resources/fixtures/page3.html +2 -0
  206. harness/trajectory_replay/retrieval_acceptance.py +1080 -0
  207. harness/trajectory_replay/runner.py +304 -0
  208. harness/trajectory_replay/runtime_identity.py +131 -0
  209. harness/trajectory_replay/suite.py +83 -0
  210. harness/trajectory_replay/webarena_corrections.py +1088 -0
  211. harness/trajectory_replay/webarena_dataset.py +684 -0
  212. harness/trajectory_replay/webarena_plan.py +658 -0
  213. harness/trajectory_replay/webarena_replay.py +1657 -0
  214. harness/trajectory_replay/webarena_report.py +2290 -0
  215. harness/trajectory_replay/webarena_report_check.py +944 -0
  216. harness/trajectory_replay/webarena_report_package.py +155 -0
  217. harness/trajectory_replay/webarena_report_publish.py +230 -0
  218. harness/trajectory_replay/webarena_report_visual.py +693 -0
  219. harness/trajectory_replay/webarena_results.py +644 -0
  220. harness/trajectory_replay/webarena_review.py +198 -0
  221. harness/trajectory_replay/webarena_runtime_source.py +119 -0
  222. harness/web_extract/__init__.py +1 -0
  223. harness/web_extract/analysis.py +683 -0
  224. harness/web_extract/assets/report.css +58 -0
  225. harness/web_extract/assets/report.js +431 -0
  226. harness/web_extract/certification.py +596 -0
  227. harness/web_extract/cli.py +436 -0
  228. harness/web_extract/dataset.py +274 -0
  229. harness/web_extract/example.py +473 -0
  230. harness/web_extract/firecrawl_backend.mjs +178 -0
  231. harness/web_extract/messages.json +1070 -0
  232. harness/web_extract/messages.py +31 -0
  233. harness/web_extract/model_comparison.py +613 -0
  234. harness/web_extract/pipeline.py +2580 -0
  235. harness/web_extract/publication.py +209 -0
  236. harness/web_extract/records.py +574 -0
  237. harness/web_extract/repair.py +657 -0
  238. harness/web_extract/report.py +6167 -0
  239. harness/web_extract/report_suite.py +353 -0
  240. harness/web_extract/report_ui.py +66 -0
  241. harness/web_extract/review.py +1580 -0
  242. harness/web_extract/self_hosted.py +978 -0
  243. harness/web_extract/snapshots.py +1197 -0
  244. harness/web_extract/swde.py +742 -0
  245. harness/web_fetch/__init__.py +5 -0
  246. harness/web_fetch/__main__.py +5 -0
  247. harness/web_fetch/accounts.py +130 -0
  248. harness/web_fetch/adjudication_evidence.py +767 -0
  249. harness/web_fetch/anomaly_review.py +346 -0
  250. harness/web_fetch/artifact_auth.py +135 -0
  251. harness/web_fetch/artifacts.py +521 -0
  252. harness/web_fetch/assembly.py +1324 -0
  253. harness/web_fetch/assets/cohort-diagnostic.css +18 -0
  254. harness/web_fetch/assets/cohort-diagnostic.js +2 -0
  255. harness/web_fetch/assets/methodology/annotation.png +0 -0
  256. harness/web_fetch/assets/methodology/scoring.png +0 -0
  257. harness/web_fetch/assets/methodology/validation.png +0 -0
  258. harness/web_fetch/assets/report-extra.css +64 -0
  259. harness/web_fetch/assets/report-loader.js +167 -0
  260. harness/web_fetch/assets/report-style.css +68 -0
  261. harness/web_fetch/assets/report.js +57 -0
  262. harness/web_fetch/assets/wcxb.css +1 -0
  263. harness/web_fetch/assets/wcxb.js +1 -0
  264. harness/web_fetch/assets/webmainbench-full.css +1 -0
  265. harness/web_fetch/assets/webmainbench-full.js +52 -0
  266. harness/web_fetch/benchmark_runner.py +190 -0
  267. harness/web_fetch/browser_observations.py +980 -0
  268. harness/web_fetch/call_boundary.py +142 -0
  269. harness/web_fetch/certification.py +683 -0
  270. harness/web_fetch/cli.py +394 -0
  271. harness/web_fetch/cohort_exclusions.py +38 -0
  272. harness/web_fetch/comparison_audit.py +479 -0
  273. harness/web_fetch/config.py +231 -0
  274. harness/web_fetch/container_resources.py +220 -0
  275. harness/web_fetch/data/browser-observation-contracts.json +25 -0
  276. harness/web_fetch/data/fixture-dataset.json +42 -0
  277. harness/web_fetch/data/fixture-providers.json +11 -0
  278. harness/web_fetch/data/latency-certifiers/34bdb7aedb25a9e5a0c64e0c549926ed7e71f747f544ec2035f65952c0177347.py.txt +142 -0
  279. harness/web_fetch/data/latency-certifiers/95a2f0a73578346998f0a4ad1256b2cef469a3f2db9b4b30550fc03fc5c259a8.py.txt +142 -0
  280. harness/web_fetch/data/native-projection-history.json +35 -0
  281. harness/web_fetch/data/native-projection-regressions.json +531 -0
  282. harness/web_fetch/data/report-methods.json +6687 -0
  283. harness/web_fetch/data/report-track-context.json +132 -0
  284. harness/web_fetch/data/wcxb-metric-bindings.json +12 -0
  285. harness/web_fetch/data/webmainbench-latency-source.json +18 -0
  286. harness/web_fetch/dataset_assembly.py +274 -0
  287. harness/web_fetch/diagnostics.py +73 -0
  288. harness/web_fetch/document_text.py +55 -0
  289. harness/web_fetch/dynamic_cohort.py +467 -0
  290. harness/web_fetch/evidence.py +753 -0
  291. harness/web_fetch/failure_diagnosis.py +190 -0
  292. harness/web_fetch/headless_runner.py +510 -0
  293. harness/web_fetch/html_to_markdown.mjs +21 -0
  294. harness/web_fetch/local_audit.py +453 -0
  295. harness/web_fetch/local_execution_contract.py +317 -0
  296. harness/web_fetch/local_execution_legacy.py +205 -0
  297. harness/web_fetch/local_revision.py +1192 -0
  298. harness/web_fetch/markdown.py +32 -0
  299. harness/web_fetch/messages.json +2377 -0
  300. harness/web_fetch/methods.py +451 -0
  301. harness/web_fetch/models.py +70 -0
  302. harness/web_fetch/native_projection.py +769 -0
  303. harness/web_fetch/native_routes.py +637 -0
  304. harness/web_fetch/network_dns.py +363 -0
  305. harness/web_fetch/network_preflight.py +119 -0
  306. harness/web_fetch/provider_adoption.py +321 -0
  307. harness/web_fetch/providers/__init__.py +4 -0
  308. harness/web_fetch/providers/base.py +274 -0
  309. harness/web_fetch/providers/cloudflare.py +816 -0
  310. harness/web_fetch/providers/cowagent.py +78 -0
  311. harness/web_fetch/providers/exa.py +177 -0
  312. harness/web_fetch/providers/firecrawl.py +454 -0
  313. harness/web_fetch/providers/fixture.py +49 -0
  314. harness/web_fetch/providers/headless.py +1147 -0
  315. harness/web_fetch/providers/headless_worker.py +512 -0
  316. harness/web_fetch/providers/hosted_fetch.py +250 -0
  317. harness/web_fetch/providers/http.py +87 -0
  318. harness/web_fetch/providers/http_client.py +112 -0
  319. harness/web_fetch/providers/lexmount.py +211 -0
  320. harness/web_fetch/providers/moli.py +889 -0
  321. harness/web_fetch/providers/runtime_identity.py +220 -0
  322. harness/web_fetch/providers/subprocess_jsonl.py +410 -0
  323. harness/web_fetch/providers/tavily.py +116 -0
  324. harness/web_fetch/public_benchmark_sample.py +160 -0
  325. harness/web_fetch/publication.py +26 -0
  326. harness/web_fetch/publication_check.py +379 -0
  327. harness/web_fetch/publication_privacy.py +1136 -0
  328. harness/web_fetch/publication_verify.py +390 -0
  329. harness/web_fetch/published_snapshot.py +379 -0
  330. harness/web_fetch/reference_freeze.py +699 -0
  331. harness/web_fetch/reference_inputs.py +387 -0
  332. harness/web_fetch/reference_lifecycle.py +1811 -0
  333. harness/web_fetch/reference_revision.py +408 -0
  334. harness/web_fetch/refresh_assembly.py +1723 -0
  335. harness/web_fetch/reliability.py +80 -0
  336. harness/web_fetch/render_evidence.py +215 -0
  337. harness/web_fetch/report-translations.json +3456 -0
  338. harness/web_fetch/report.py +4564 -0
  339. harness/web_fetch/report_build.py +1851 -0
  340. harness/web_fetch/report_capabilities.py +149 -0
  341. harness/web_fetch/report_changes.py +224 -0
  342. harness/web_fetch/report_check.py +1004 -0
  343. harness/web_fetch/report_components.py +512 -0
  344. harness/web_fetch/report_diagnostic.py +857 -0
  345. harness/web_fetch/report_explainer.py +1133 -0
  346. harness/web_fetch/report_i18n.py +942 -0
  347. harness/web_fetch/report_interfaces.py +262 -0
  348. harness/web_fetch/report_methods.py +159 -0
  349. harness/web_fetch/report_package.py +1000 -0
  350. harness/web_fetch/report_pricing.py +120 -0
  351. harness/web_fetch/report_publish.py +158 -0
  352. harness/web_fetch/report_quality.py +34 -0
  353. harness/web_fetch/report_reading.py +124 -0
  354. harness/web_fetch/report_resources.py +196 -0
  355. harness/web_fetch/report_saved_composition.py +149 -0
  356. harness/web_fetch/report_saved_matrix.py +2662 -0
  357. harness/web_fetch/report_style.py +9 -0
  358. harness/web_fetch/report_version.py +270 -0
  359. harness/web_fetch/rescoring.py +332 -0
  360. harness/web_fetch/resource_cohort.py +207 -0
  361. harness/web_fetch/result_ledger.py +119 -0
  362. harness/web_fetch/result_review.py +1252 -0
  363. harness/web_fetch/retrieval_hints.py +574 -0
  364. harness/web_fetch/review_execution.py +1638 -0
  365. harness/web_fetch/risk_control.py +524 -0
  366. harness/web_fetch/runner.py +1241 -0
  367. harness/web_fetch/safety.py +140 -0
  368. harness/web_fetch/scoring_revision.py +391 -0
  369. harness/web_fetch/source_capture.py +455 -0
  370. harness/web_fetch/speed_cohort.py +216 -0
  371. harness/web_fetch/storage.py +336 -0
  372. harness/web_fetch/task_outputs.py +346 -0
  373. harness/web_fetch/templates/interpretation-headless.css +326 -0
  374. harness/web_fetch/title_quality.py +93 -0
  375. harness/web_fetch/wcxb_dataset.py +168 -0
  376. harness/web_fetch/wcxb_latency.py +347 -0
  377. harness/web_fetch/wcxb_metrics.py +38 -0
  378. harness/web_fetch/wcxb_publication.py +147 -0
  379. harness/web_fetch/wcxb_report_check.py +753 -0
  380. harness/web_fetch/wcxb_scoring.py +667 -0
  381. harness/web_fetch/webmainbench_assembly.py +684 -0
  382. harness/web_fetch/webmainbench_case_correction.py +267 -0
  383. harness/web_fetch/webmainbench_certification.py +306 -0
  384. harness/web_fetch/webmainbench_correction.py +866 -0
  385. harness/web_fetch/webmainbench_diagnostics.py +1202 -0
  386. harness/web_fetch/webmainbench_evidence.py +205 -0
  387. harness/web_fetch/webmainbench_latency_correction.py +525 -0
  388. harness/web_fetch/webmainbench_latency_diagnostic.py +562 -0
  389. harness/web_fetch/webmainbench_report_check.py +900 -0
  390. harness/web_fetch/webmainbench_runner.py +680 -0
  391. harness/web_fetch/webmainbench_scoring.py +354 -0
  392. harness/web_fetch/workers/__init__.py +1 -0
  393. harness/web_fetch/workers/cowagent_worker.py +70 -0
  394. harness/web_fetch/workers/crawl4ai_worker.py +463 -0
  395. harness/web_fetch/workers/dsh_output.mjs +21 -0
  396. harness/web_fetch/workers/dsh_webfetch_worker.mts +100 -0
  397. harness/web_fetch/workers/langchain_webbase_worker.py +67 -0
  398. harness/web_fetch/workers/native_dns_darwin.c +108 -0
  399. harness/web_fetch/workers/openclaw_webfetch_worker.mts +91 -0
  400. harness/web_fetch/workers/public_dns.mjs +49 -0
  401. harness/web_search/__init__.py +1 -0
  402. harness/web_search/__main__.py +4 -0
  403. harness/web_search/adapters.py +371 -0
  404. harness/web_search/answers.py +326 -0
  405. harness/web_search/assets/comparison.css +5 -0
  406. harness/web_search/assets/comparison.js +45 -0
  407. harness/web_search/assets/delivery-bridge.js +35 -0
  408. harness/web_search/assets/delivery-loader.js +35 -0
  409. harness/web_search/assets/native-evidence-result.js +8 -0
  410. harness/web_search/assets/report.css +69 -0
  411. harness/web_search/assets/report.js +58 -0
  412. harness/web_search/cli.py +44 -0
  413. harness/web_search/comparison.py +843 -0
  414. harness/web_search/core.py +274 -0
  415. harness/web_search/data/native-normalization-regressions.json +354 -0
  416. harness/web_search/delivery.py +163 -0
  417. harness/web_search/diagnostics.py +132 -0
  418. harness/web_search/example.py +147 -0
  419. harness/web_search/messages.json +751 -0
  420. harness/web_search/messages.py +23 -0
  421. harness/web_search/output_levels.py +431 -0
  422. harness/web_search/pages.py +28 -0
  423. harness/web_search/profiles.py +220 -0
  424. harness/web_search/publication.py +208 -0
  425. harness/web_search/relevance.py +828 -0
  426. harness/web_search/report.py +4315 -0
  427. harness/web_search/review.py +1318 -0
  428. harness/web_search/revisions.py +1052 -0
  429. harness/web_search/runner.py +1013 -0
  430. harness/web_search/visual.py +164 -0
  431. harness/webarena_state.py +549 -0
  432. lexbench_webfetch/__init__.py +1 -0
  433. lexbench_webfetch/annotation_revisions.py +269 -0
  434. lexbench_webfetch/article_comments.py +331 -0
  435. lexbench_webfetch/article_dataset.py +285 -0
  436. lexbench_webfetch/article_posts.py +384 -0
  437. lexbench_webfetch/article_qualification.py +192 -0
  438. lexbench_webfetch/article_text.py +259 -0
  439. lexbench_webfetch/article_title.py +186 -0
  440. lexbench_webfetch/benchmark.py +591 -0
  441. lexbench_webfetch/body_coverage.py +582 -0
  442. lexbench_webfetch/cli.py +143 -0
  443. lexbench_webfetch/composite_tasks.py +2026 -0
  444. lexbench_webfetch/content_scopes.py +156 -0
  445. lexbench_webfetch/document_structure.py +326 -0
  446. lexbench_webfetch/dynamic_entry_matching.py +190 -0
  447. lexbench_webfetch/dynamic_lists.py +1658 -0
  448. lexbench_webfetch/dynamic_reference.py +25 -0
  449. lexbench_webfetch/evaluation.py +24 -0
  450. lexbench_webfetch/file_acceptance.py +72 -0
  451. lexbench_webfetch/inputs.py +129 -0
  452. lexbench_webfetch/list_verification.py +90 -0
  453. lexbench_webfetch/metrics.py +1441 -0
  454. lexbench_webfetch/models.py +93 -0
  455. lexbench_webfetch/ordered_tree.py +115 -0
  456. lexbench_webfetch/paths.py +41 -0
  457. lexbench_webfetch/reference_quality.py +1104 -0
  458. lexbench_webfetch/required_components.py +44 -0
  459. lexbench_webfetch/resource_urls.py +35 -0
  460. lexbench_webfetch/response_input.py +135 -0
  461. lexbench_webfetch/risk_control_dataset.py +512 -0
  462. lexbench_webfetch/scoring.py +452 -0
  463. lexbench_webfetch/scoring_regressions.py +215 -0
  464. lexbench_webfetch/stable_tasks.py +2144 -0
  465. lexbench_webfetch/structured_tables.py +793 -0
  466. lexbench_webfetch/versions.py +224 -0
  467. lexbench_websearch/__init__.py +5 -0
  468. lexbench_websearch/answer_rules.py +322 -0
  469. lexbench_websearch/answers.py +383 -0
  470. lexbench_websearch/cli.py +123 -0
  471. lexbench_websearch/contracts.py +58 -0
  472. lexbench_websearch/evidence_quality.py +109 -0
  473. lexbench_websearch/inputs.py +80 -0
  474. lexbench_websearch/outputs.py +161 -0
  475. lexbench_websearch/qualification.py +144 -0
  476. lexbench_websearch/relevance.py +346 -0
  477. lexbench_websearch/rubric.py +34 -0
  478. lexbench_websearch/scoring.py +354 -0
  479. lexbench_websearch/simpleqa.py +467 -0
  480. lexbench_websearch/versions.py +255 -0
  481. webstone-0.8.0.dist-info/METADATA +367 -0
  482. webstone-0.8.0.dist-info/RECORD +488 -0
  483. webstone-0.8.0.dist-info/WHEEL +5 -0
  484. webstone-0.8.0.dist-info/entry_points.txt +4 -0
  485. webstone-0.8.0.dist-info/licenses/LICENSE +202 -0
  486. webstone-0.8.0.dist-info/licenses/NOTICE +6 -0
  487. webstone-0.8.0.dist-info/licenses/THIRD_PARTY.md +22 -0
  488. webstone-0.8.0.dist-info/top_level.txt +3 -0
@@ -0,0 +1,445 @@
1
+ """Bound benchmark work by declared resource isolation, not a global serial loop.
2
+
3
+ Tracks own calls, evidence, retries and verdicts. This owner schedules ready work
4
+ and drains dispatched calls before stopping; it never retries a job.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import argparse
10
+ import hashlib
11
+ import json
12
+ import math
13
+ import multiprocessing
14
+ import signal
15
+ import threading
16
+ import time
17
+ from concurrent.futures import (
18
+ FIRST_COMPLETED,
19
+ BrokenExecutor,
20
+ CancelledError,
21
+ Future,
22
+ ProcessPoolExecutor,
23
+ ThreadPoolExecutor,
24
+ wait,
25
+ )
26
+ from contextlib import contextmanager, nullcontext
27
+ from dataclasses import dataclass
28
+ from pathlib import Path
29
+ from typing import Any
30
+
31
+
32
+ class ExecutionStop:
33
+ """Expose a signal-safe stop flag; notify waiters outside signal handlers."""
34
+
35
+ def __init__(self, event=None):
36
+ self.event = event if event is not None else threading.Event()
37
+ self.requested = False
38
+
39
+ def request_stop(self):
40
+ self.requested = True
41
+
42
+ def is_set(self):
43
+ return self.requested or self.event.is_set()
44
+
45
+ def set(self):
46
+ self.request_stop()
47
+ self.event.set()
48
+
49
+ def wait(self, timeout=None):
50
+ return self.is_set() or self.event.wait(timeout)
51
+
52
+
53
+ @contextmanager
54
+ def _notify_inline_stop(stop):
55
+ """Wake inline callbacks without re-entering an event lock in a signal handler."""
56
+ closed = threading.Event()
57
+
58
+ def notify():
59
+ while not closed.wait(.1):
60
+ if stop.requested:
61
+ stop.event.set()
62
+ return
63
+
64
+ notifier = threading.Thread(target=notify, daemon=True)
65
+ notifier.start()
66
+ try:
67
+ yield
68
+ finally:
69
+ closed.set()
70
+ notifier.join()
71
+
72
+
73
+ def exclusive_policy(resource, reason, *, purpose="quality"):
74
+ """Describe a stateful environment whose full lifecycle has one owner."""
75
+ value = {
76
+ "purpose": purpose,
77
+ "max_in_flight": 1,
78
+ "serial_reason": reason,
79
+ "resources": {resource: {"capacity": 1, "reason": reason}},
80
+ }
81
+ ExecutionPolicy.from_dict(value)
82
+ return value
83
+
84
+
85
+ def assess_execution(plan, *, evidence_root=None):
86
+ """Check a measured preflight and project batch time before collection."""
87
+ if not isinstance(plan, dict):
88
+ raise ValueError("Execution assessment must be an object")
89
+ policy = ExecutionPolicy.from_dict(plan.get("execution_policy"))
90
+ calls = plan.get("planned_calls")
91
+ if type(calls) is not int or calls < 1:
92
+ raise ValueError("planned_calls must be a positive integer")
93
+ sample = plan.get("preflight")
94
+ if not isinstance(sample, dict):
95
+ raise ValueError("Record a bounded measured preflight before collection")
96
+ count, seconds = sample.get("completed_calls"), sample.get("wall_seconds")
97
+ if type(count) is not int or not 1 <= count <= calls:
98
+ raise ValueError("Invalid preflight call population")
99
+ if type(seconds) not in (int, float) or not math.isfinite(seconds) or seconds <= 0:
100
+ raise ValueError("Preflight wall_seconds must be finite and positive")
101
+ if (
102
+ type(sample.get("max_in_flight")) is not int
103
+ or sample["max_in_flight"] != policy.max_in_flight
104
+ ):
105
+ raise ValueError("Preflight concurrency differs from the execution plan")
106
+ identity = sample.get("evidence_sha256")
107
+ if (
108
+ not isinstance(identity, str)
109
+ or len(identity) != 64
110
+ or any(c not in "0123456789abcdef" for c in identity)
111
+ ):
112
+ raise ValueError("Bind the preflight evidence hash")
113
+ if evidence_root is not None:
114
+ root = Path(evidence_root).resolve()
115
+ if not isinstance(sample.get("evidence_path"), str):
116
+ raise ValueError("Bind a preflight evidence_path")
117
+ relative = Path(sample.get("evidence_path", ""))
118
+ source = root / relative
119
+ if (
120
+ relative.is_absolute()
121
+ or ".." in relative.parts
122
+ or not source.is_file()
123
+ or source.is_symlink()
124
+ or source.resolve() != source
125
+ or not source.resolve().is_relative_to(root)
126
+ ):
127
+ raise ValueError("Missing or unsafe preflight evidence")
128
+ if hashlib.sha256(source.read_bytes()).hexdigest() != identity:
129
+ raise ValueError("Preflight evidence changed")
130
+ receipt = json.loads(source.read_text(encoding="utf-8"))
131
+ for key in ("completed_calls", "wall_seconds", "max_in_flight"):
132
+ if type(receipt.get(key)) is not type(sample[key]) or receipt[key] != sample[key]:
133
+ raise ValueError("Preflight summary differs from bound evidence")
134
+ setup = plan.get("setup_seconds", 0)
135
+ budget = plan.get("wall_time_budget_seconds")
136
+ if type(setup) not in (int, float) or not math.isfinite(setup) or setup < 0:
137
+ raise ValueError("Invalid setup time")
138
+ if type(budget) not in (int, float) or not math.isfinite(budget) or budget <= 0:
139
+ raise ValueError("Declare a finite positive wall_time_budget_seconds")
140
+ estimate = setup + calls * seconds / count
141
+ reason = plan.get("over_budget_reason")
142
+ if estimate > budget and (not isinstance(reason, str) or not reason.strip()):
143
+ raise ValueError(
144
+ "Estimated execution exceeds the budget; revise the plan or explain the constraint"
145
+ )
146
+ return {
147
+ "planned_calls": calls,
148
+ "sample_calls": count,
149
+ "estimated_wall_seconds": estimate,
150
+ "wall_time_budget_seconds": budget,
151
+ "purpose": policy.purpose,
152
+ "max_in_flight": policy.max_in_flight,
153
+ "estimate_scope": "Projection from the bound preflight; not a latency ranking",
154
+ }
155
+
156
+
157
+ def validate_capacity(value, label="capacity"):
158
+ if type(value) is not int or not 1 <= value <= 64:
159
+ raise ValueError(f"{label} must be an integer between 1 and 64")
160
+ return value
161
+
162
+
163
+ @dataclass(frozen=True)
164
+ class ExecutionPolicy:
165
+ purpose: str
166
+ max_in_flight: int
167
+ resources: dict[str, int]
168
+ serial_reasons: dict[str, str]
169
+ backend: str = "thread"
170
+
171
+ @classmethod
172
+ def from_dict(cls, value):
173
+ if (
174
+ not isinstance(value, dict)
175
+ or not isinstance(value.get("purpose"), str)
176
+ or value["purpose"] not in {"quality", "latency", "throughput"}
177
+ ):
178
+ raise ValueError("Execution purpose must be quality, latency or throughput")
179
+ maximum = validate_capacity(value.get("max_in_flight"), "max_in_flight")
180
+ reason = value.get("serial_reason", "")
181
+ if maximum == 1 and (not isinstance(reason, str) or not reason.strip()):
182
+ raise ValueError("Serial execution requires a concrete serial_reason")
183
+ resources = value.get("resources")
184
+ if not isinstance(resources, dict) or not resources:
185
+ raise ValueError("Declare the shared resources before execution")
186
+ capacities, reasons = {}, {}
187
+ for name, resource in resources.items():
188
+ if not isinstance(name, str) or not name.strip() or not isinstance(resource, dict):
189
+ raise ValueError("Invalid execution resource")
190
+ capacities[name] = validate_capacity(resource.get("capacity"), "Resource capacity")
191
+ reason = resource.get("reason", "")
192
+ if capacities[name] == 1 and (not isinstance(reason, str) or not reason.strip()):
193
+ raise ValueError("An exclusive resource requires a concrete reason")
194
+ reasons[name] = reason
195
+ if value["purpose"] == "latency" and capacities[name] != 1:
196
+ raise ValueError("Isolated latency requires exclusive measurement resources")
197
+ backend = value.get("backend", "thread")
198
+ if backend not in {"thread", "process"}:
199
+ raise ValueError("Execution backend must be thread or process")
200
+ return cls(value["purpose"], maximum, capacities, reasons, backend)
201
+
202
+
203
+ @dataclass(frozen=True)
204
+ class Job:
205
+ key: str
206
+ resources: tuple[str, ...]
207
+ payload: Any
208
+
209
+
210
+ @contextmanager
211
+ def _retain_call_ownership(on_interrupt=None):
212
+ """Defer Ctrl-C until a future and its durable callback have a known owner."""
213
+ if threading.current_thread() is not threading.main_thread():
214
+ yield
215
+ return
216
+ previous = signal.getsignal(signal.SIGINT)
217
+ interrupted = None
218
+ failure = None
219
+
220
+ def defer(number, frame):
221
+ nonlocal interrupted
222
+ if interrupted is not None:
223
+ signal.signal(signal.SIGINT, previous)
224
+ if callable(previous):
225
+ previous(number, frame)
226
+ elif previous != signal.SIG_IGN:
227
+ raise KeyboardInterrupt()
228
+ return
229
+ interrupted = (number, frame)
230
+ if on_interrupt is not None:
231
+ on_interrupt()
232
+
233
+ signal.signal(signal.SIGINT, defer)
234
+ try:
235
+ yield
236
+ except BaseException as error:
237
+ failure = error
238
+ raise
239
+ finally:
240
+ signal.signal(signal.SIGINT, previous)
241
+ if interrupted is not None and previous != signal.SIG_IGN:
242
+ try:
243
+ if callable(previous):
244
+ previous(*interrupted)
245
+ else:
246
+ raise KeyboardInterrupt()
247
+ except BaseException as interrupt:
248
+ if failure is not None and not isinstance(failure, KeyboardInterrupt):
249
+ raise failure from interrupt
250
+ raise
251
+
252
+
253
+ class _InlineExecutor:
254
+ """Keep a single native call on the main thread, including its signal deadline."""
255
+
256
+ def submit(self, execute, payload):
257
+ future = Future()
258
+ try:
259
+ future.set_result(execute(payload))
260
+ except BaseException as error:
261
+ future.set_exception(error)
262
+ return future
263
+
264
+
265
+ def _initialize_process(initialize, initargs):
266
+ # The parent owns terminal interruption and drains calls before stopping.
267
+ signal.signal(signal.SIGINT, signal.SIG_IGN)
268
+ if initialize is not None:
269
+ initialize(*initargs)
270
+
271
+
272
+ def run_jobs(
273
+ jobs,
274
+ execute,
275
+ commit,
276
+ policy,
277
+ *,
278
+ prepare=None,
279
+ ready_at=None,
280
+ should_stop=lambda: False,
281
+ initialize=None,
282
+ initargs=(),
283
+ stop_event=None,
284
+ ):
285
+ """Dispatch a bounded set; commit every terminal call before propagating errors.
286
+
287
+ Inventories contain small task descriptions. Expensive payload preparation
288
+ happens only after resource admission. Only the parent calls prepare/commit,
289
+ so each track can retain its existing durable evidence and budget boundary.
290
+ """
291
+ pending = list(jobs)
292
+ keys = set()
293
+ for job in pending:
294
+ if (
295
+ not isinstance(job, Job)
296
+ or not isinstance(job.key, str)
297
+ or not job.key
298
+ or job.key in keys
299
+ ):
300
+ raise ValueError("Execution job IDs must be nonempty and globally unique")
301
+ if (
302
+ not job.resources
303
+ or len(set(job.resources)) != len(job.resources)
304
+ or any(r not in policy.resources for r in job.resources)
305
+ ):
306
+ raise ValueError("Every job must bind registered, distinct resources")
307
+ keys.add(job.key)
308
+ active = {name: 0 for name in policy.resources}
309
+ inflight = {}
310
+ failure = None
311
+ cooperative_stop = stop_event is not None
312
+ inline = policy.max_in_flight == 1 and policy.backend == 'thread'
313
+ stop_event = stop_event if isinstance(stop_event, ExecutionStop) else ExecutionStop(stop_event)
314
+ options = {"max_workers": policy.max_in_flight}
315
+ if initialize is not None:
316
+ options.update(initializer=initialize, initargs=initargs)
317
+ executor_class = ThreadPoolExecutor
318
+ if policy.backend == "process":
319
+ executor_class = ProcessPoolExecutor
320
+ options.update(initializer=_initialize_process, initargs=(initialize, initargs))
321
+ # Fresh interpreters avoid copying a parent that may already own threads.
322
+ options["mp_context"] = multiprocessing.get_context("spawn")
323
+
324
+ def stop(error):
325
+ nonlocal failure
326
+ if failure is None:
327
+ failure = error
328
+ stop_event.set()
329
+
330
+ def record_interrupt():
331
+ nonlocal failure
332
+ # A signal can interrupt Event.wait while its non-reentrant lock is held.
333
+ stop_event.request_stop()
334
+ if failure is None:
335
+ failure = KeyboardInterrupt()
336
+
337
+ with (
338
+ _retain_call_ownership(record_interrupt),
339
+ (_notify_inline_stop(stop_event) if inline and cooperative_stop else nullcontext()),
340
+ (nullcontext(_InlineExecutor()) if inline
341
+ else executor_class(**options)) as executor,
342
+ ):
343
+ if (pending and policy.max_in_flight == 1 and policy.backend == 'thread'
344
+ and initialize is not None):
345
+ initialize(*initargs)
346
+ while pending or inflight:
347
+ next_ready = None
348
+ try:
349
+ if failure is None and not stop_event.is_set() and not should_stop():
350
+ for job in list(pending):
351
+ if (failure is not None or stop_event.is_set()
352
+ or len(inflight) >= policy.max_in_flight):
353
+ break
354
+ if any(active[r] >= policy.resources[r] for r in job.resources):
355
+ continue
356
+ if ready_at is not None:
357
+ available = ready_at(job)
358
+ if (not isinstance(available, (int, float))
359
+ or not math.isfinite(available)):
360
+ raise ValueError(
361
+ "Admission deadline must be a finite monotonic time")
362
+ if available > time.monotonic():
363
+ next_ready = (available if next_ready is None
364
+ else min(next_ready, available))
365
+ continue
366
+ payload = prepare(job) if prepare else job.payload
367
+ if failure is not None or stop_event.is_set() or should_stop():
368
+ if prepare is not None:
369
+ commit(job, None, CancelledError("Stopped before dispatch"))
370
+ break
371
+ try:
372
+ future = executor.submit(execute, payload)
373
+ except RuntimeError as error:
374
+ rejected = isinstance(error, BrokenExecutor) or str(error) in {
375
+ "cannot schedule new futures after shutdown",
376
+ "cannot schedule new futures after interpreter shutdown",
377
+ }
378
+ if rejected and prepare is not None:
379
+ try:
380
+ commit(job, None, CancelledError("Submission rejected"))
381
+ finally:
382
+ raise error
383
+ raise
384
+ inflight[future] = job
385
+ for resource in job.resources:
386
+ active[resource] += 1
387
+ pending.remove(job)
388
+ except BaseException as error:
389
+ stop(error)
390
+ if failure is not None:
391
+ stop_event.set()
392
+ if not inflight:
393
+ if (pending and next_ready is not None and failure is None
394
+ and not stop_event.is_set() and not should_stop()):
395
+ stop_event.wait(min(.1, max(0, next_ready - time.monotonic())))
396
+ continue
397
+ break
398
+ try:
399
+ # Notify cooperative workers outside the signal handler even if
400
+ # none of their futures can finish before the stop event is set.
401
+ timeout = .1
402
+ if next_ready is not None:
403
+ timeout = min(timeout, max(0, next_ready - time.monotonic()))
404
+ done, _ = wait(inflight, return_when=FIRST_COMPLETED, timeout=timeout)
405
+ for future in done:
406
+ try:
407
+ job = inflight[future]
408
+ value = error = None
409
+ try:
410
+ value = future.result()
411
+ except BaseException as caught:
412
+ error = caught
413
+ try:
414
+ commit(job, value, error)
415
+ finally:
416
+ inflight.pop(future)
417
+ for resource in job.resources:
418
+ active[resource] -= 1
419
+ except BaseException as caught:
420
+ stop(caught)
421
+ except BaseException as error:
422
+ # An operator interruption stops admission, while dispatched calls
423
+ # still reach the track's durable commit boundary.
424
+ stop(error)
425
+ continue
426
+ if failure is not None:
427
+ raise failure
428
+
429
+
430
+ def main(argv=None):
431
+ parser = argparse.ArgumentParser(description=__doc__)
432
+ parser.add_argument("--plan", type=Path, required=True)
433
+ args = parser.parse_args(argv)
434
+ print(
435
+ json.dumps(
436
+ assess_execution(
437
+ json.loads(args.plan.read_text(encoding="utf-8")), evidence_root=args.plan.parent
438
+ ),
439
+ indent=2,
440
+ )
441
+ )
442
+
443
+
444
+ if __name__ == "__main__":
445
+ main()
@@ -0,0 +1 @@
1
+ """Shared distribution of versioned benchmark definitions."""
@@ -0,0 +1,45 @@
1
+ # Expanded SWDE
2
+
3
+ Short name: **LB-ESWDE**.
4
+
5
+ This research distribution contains {{total_records}} source-qualified pages sampled from [Expanded SWDE](https://www.colinlockard.com/expanded_swde.html). It retains the original benchmark name and attribution. It is a selected evaluation cohort. Scores from this selection do not reproduce the paper's published results.
6
+
7
+ Lockard et al., OpenCeres (2019), extending SWDE by Hao et al. (2011).
8
+
9
+ ## Webstone
10
+
11
+ This dataset is integrated with [Webstone](https://github.com/lexmount/webstone). The [Web Extract guide](https://github.com/lexmount/webstone/blob/main/examples/web_extract/README.md) covers dataset selection, hosted or live inputs, provider selection and certified collection. The upstream benchmark links and original license below remain authoritative for the source data.
12
+
13
+ ```sh
14
+ git clone https://github.com/lexmount/webstone
15
+ cd webstone
16
+ {{tooling_checkout}}uv sync --frozen --extra web-extract-eval
17
+ ```
18
+
19
+ ## Inputs and references
20
+
21
+ `definitions.tar.gz` contains the source-qualified task definition, original HTML, captured resources, references and provenance. `manifest.json` binds the complete archive and every file by SHA-256. Original task inputs and reference values are unchanged from the preceding edition. Webstone uses the shared immutable dataset cache; no extraction service is called during preparation.
22
+
23
+ ```sh
24
+ make pin-dataset DATASET=expanded-swde REV=YOUR_FULL_HF_COMMIT
25
+ uv run --frozen --extra web-extract-eval python -m harness.web_extract.cli \
26
+ --home /absolute/data/expanded-swde prepare --dataset expanded-swde
27
+ ```
28
+
29
+ The selected input mode is `snapshot`. Native extraction services visit a separately deployed, version-bound HTML service. The deployment exposes pages and resources only; reference answers remain outside the hosted page service.
30
+
31
+ ## Scoring
32
+
33
+ The comparison uses value-level micro F1, page-hit macro F1 and exact-field-set rate. Original references are retained alongside source-qualified whitespace/entity and source-case normalization. Declared URL fields use URL equivalence. These are evaluation adaptations; this distribution does not claim the original paper's protocol.
34
+
35
+ ## Rights
36
+
37
+ Expanded SWDE by Lockard et al. (OpenCeres, 2019), extending SWDE by Hao et al. (2011). Original SWDE archive: Microsoft Reciprocal License; webpage content retains original rights. Preserve both source attributions. No blanket MIT or Apache license is granted to website content.
38
+
39
+ Website content retains its original rights. This repository is private. Packaging does not broaden the upstream license or grant rights to the original website content.
40
+
41
+ ## Public HTML snapshots
42
+
43
+ [snapshots.json](snapshots.json) binds every page ID to its SHA-256 and an immutable public deployment. Native extraction APIs receive `fixture_base + '/' + id`; the original HTML files remain in this repository.
44
+
45
+ Original files are authoritative. Publication verifies the declared dataset version and complete input inventory before adding this binding. Changed HTML requires a new deployment and a new HF commit; previous files and deployment URLs remain available. Pin the full HF commit for evaluation. Managed immutable deployments are checked through the whole-dataset manifest before and after calls; external hosts have every page byte-checked when first bound. A mismatch stops collection rather than silently replacing the input. The navigation index does not host dataset HTML.
@@ -0,0 +1,112 @@
1
+ # LexBench-Headless-Browser
2
+
3
+ Short name: **LB-HB**.
4
+
5
+ A versioned data export of [Lexbench-Headless-Browser](https://github.com/lexmount/Lexbench-Headless-Browser/tree/{{upstream_revision}}). It preserves the original CDP compatibility tasks, web platform tests, scenario specifications, browser and driver configuration, and local fixtures.
6
+
7
+ The dataset viewer contains **{{upstream_task_count}} executable task definitions** across {{upstream_subset_count}} subsets. The original source files also retain {{upstream_scenario_count}} driver-neutral scenario specifications, {{upstream_fixture_count}} fixture files and {{upstream_config_count}} configuration files. The upstream manifest declares benchmark version `{{upstream_bench_version}}`; its browser binding configuration names {{upstream_driver_count}} driver interfaces. Source commit: [`{{upstream_revision}}`](https://github.com/lexmount/Lexbench-Headless-Browser/commit/{{upstream_revision}}).
8
+
9
+ ## Populations and task contracts
10
+
11
+ {{dataset_summary}}
12
+
13
+ The `l1.*` and `l2.*` configurations contain the executable tasks selected by the original manifest's `task_glob` entries. Scenario specifications remain in `definitions/tasks/scenarios/` as design sources and provenance. They are excluded from the viewer population and add no extra tasks to the executable count.
14
+
15
+ - **L1 — CDP compatibility.** Protocol commands, framework and tool interfaces, navigation, DOM observation, interaction and other browser operations. Some probes are diagnostic and accept completion without grading returned values; the original tags, optional steps and checks state that scope.
16
+ - **L2 — Web platform behavior.** Browser semantics checked against local fixture pages. The manifest, capability map and driver bindings define which combinations are eligible.
17
+ - **Scenario specifications.** Driver-neutral steps, checks and expected answers. A scenario may have several executable driver translations, so repeated concepts across configurations are intentional.
18
+
19
+ The declared drivers include Playwright, Puppeteer, Chrome Remote Interface, CDP Use, Pydoll, Stagehand, Chrome DevTools MCP, chromedp, Rod, Chromiumoxide, Ferrum, agent-browser and Selenium. The source manifest also includes raw CDP probes. Driver configuration is a compatibility contract; it does not establish that every driver works with every browser or that a particular run passed.
20
+
21
+ ## Files and fidelity
22
+
23
+ `definitions/` keeps the original paths and bytes from the source commit:
24
+
25
+ | Path | Contents |
26
+ | --- | --- |
27
+ | `manifest.json` | Benchmark version, layers, subsets, task globs, browser policy and fixture site contract |
28
+ | `harness_pins.json` | Upstream runtime dependency pins |
29
+ | `tasks/` | Original executable JSON tasks and scenario specifications, including assertions and source wording |
30
+ | `config/` | Browser/driver bindings, capability maps, schemas and CDP waivers |
31
+ | `fixtures/` | Local pages, scripts and assets used by the tasks |
32
+ | `LICENSE`, `THIRD_PARTY_NOTICES.md` | Original license and attribution notices |
33
+
34
+ The additive `definitions/registry.json` binds viewer configurations to their hashes, populations and original source paths. `definitions/viewer/` contains JSON projections for the dataset viewer. Each viewer row's `task` field is the original parsed executable task object; `source_path` locates its unchanged JSON file beneath `definitions/`. The export does not translate instructions, change assertions or replace expected answers. `manifest.json` at the repository root records release hashes and source identity; `definitions/manifest.json` remains the original upstream manifest.
35
+
36
+ Viewer language labels describe the executable task instructions, which are English in this source edition. Fixture content and expected literals may use other languages; their original text is preserved.
37
+
38
+ The archive `definitions.tar.gz` provides the same definition files. Measured attempts, browser responses, scores, generated reports, private captures, executable runner code and browser binaries have separate owners and are excluded from this dataset.
39
+
40
+ ## Load a pinned dataset
41
+
42
+ Webstone records the selected Hugging Face commit in its data lock. Use that immutable revision for the viewer rows:
43
+
44
+ ```python
45
+ import json
46
+ from datasets import load_dataset
47
+ from harness.benchmarks.data import binding
48
+
49
+ selected = binding("headless-browser")
50
+ rows = load_dataset(
51
+ selected["repository"],
52
+ "l1.raw_cdp",
53
+ revision=selected["revision"],
54
+ split="test",
55
+ )
56
+ row = rows[0]
57
+ task = json.loads(row["task"])
58
+ print(row["source_path"], task["task_id"], task["grader"])
59
+ ```
60
+
61
+ Inspect the original scenario specifications under `definitions/tasks/scenarios/`. Dataset access is public and requires no account or token. A Hugging Face data commit and an upstream Git code commit identify different things; retain both when recording a run.
62
+
63
+ ## Execute with the original runner
64
+
65
+ The JSON files use the upstream driver's protocol. Its runner owns fixture routing, scenario expansion, driver adapters and inline grading. Obtain the matching code and follow the [pinned running instructions](https://github.com/lexmount/Lexbench-Headless-Browser/blob/{{upstream_revision}}/docs/RUNNING.md) for dependencies and browser binaries:
66
+
67
+ Create a fresh checkout at the data's recorded code revision, then place the pinned Hugging Face definitions at the original paths. This example copies only the original-source inventory; the viewer projections and registry stay outside the runner's inputs.
68
+
69
+ ```python
70
+ from pathlib import Path
71
+ import hashlib
72
+ import json
73
+ import subprocess
74
+
75
+ from harness.benchmarks.data import definition_path
76
+
77
+ definitions = definition_path("headless-browser", ".")
78
+ registry = json.loads((definitions / "registry.json").read_bytes())
79
+ checkout = Path("/tmp/Lexbench-Headless-Browser") # Must not already exist.
80
+ subprocess.run(["git", "clone", registry["source"]["repository"], str(checkout)], check=True)
81
+ subprocess.run(["git", "-C", str(checkout), "checkout", "--detach", registry["source"]["revision"]], check=True)
82
+ actual = subprocess.check_output(["git", "-C", str(checkout), "rev-parse", "HEAD"], text=True).strip()
83
+ assert actual == registry["source"]["revision"]
84
+ for relative, expected in registry["source_files"].items():
85
+ raw = (definitions / relative).read_bytes()
86
+ assert len(raw) == expected["size"] and hashlib.sha256(raw).hexdigest() == expected["sha256"]
87
+ destination = checkout / relative
88
+ destination.parent.mkdir(parents=True, exist_ok=True)
89
+ destination.write_bytes(raw)
90
+ ```
91
+
92
+ Run the original CLI from that checkout using the pinned guide's setup and scope options. The files it reads now come from the selected Hugging Face snapshot, including the original task JSON, fixtures, configuration, manifest and dependency pins.
93
+
94
+ Check the source manifest and running guide before choosing a scope. Loading this dataset installs neither that runner nor its runtime dependencies. The local fixture server is implemented in [`runner/run.py`](https://github.com/lexmount/Lexbench-Headless-Browser/blob/{{upstream_revision}}/runner/run.py); driver-neutral expansion and platform semantics also depend on the original runner modules and adapters.
95
+
96
+ [Webstone's public-web replay suite](https://huggingface.co/datasets/Lexmount/LexBench-Trajectory-Replay) contains 30 self-authored tasks with its own live website inputs, BrowserTool actions and exact checks. This scenario collection informed their design; none of those complete tasks is included in the Headless executable catalog. Headless execution remains owned by the pinned upstream driver runtime described above. Existing report protocols may select smaller, hash-bound populations such as the historical 372-task protocol; their scores and denominators remain attached to those original runs. The complete source collection published here does not expand an earlier report's measured population.
97
+
98
+ ## License and citation
99
+
100
+ The upstream project uses [Apache-2.0](definitions/LICENSE). Included WPT and Chromium fixtures retain their BSD-3-Clause notices in [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md). Preserve those notices when redistributing the data.
101
+
102
+ ```bibtex
103
+ @misc{lexbench_headless_browser,
104
+ author = {Lexmount},
105
+ title = {LexBench-Headless-Browser},
106
+ year = {2026},
107
+ url = {https://github.com/lexmount/Lexbench-Headless-Browser},
108
+ version = {{{upstream_revision}}}
109
+ }
110
+ ```
111
+
112
+ When using Webstone's separate replay harness, also cite [Webstone](https://github.com/lexmount/webstone).