webstone 0.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (488) hide show
  1. harness/benchmark_execution.py +445 -0
  2. harness/benchmarks/__init__.py +1 -0
  3. harness/benchmarks/cards/expanded-swde.md +45 -0
  4. harness/benchmarks/cards/headless-browser.md +112 -0
  5. harness/benchmarks/cards/languages.json +1018 -0
  6. harness/benchmarks/cards/liveweb-ie.md +49 -0
  7. harness/benchmarks/cards/metadata.json +485 -0
  8. harness/benchmarks/cards/newsxlm.md +47 -0
  9. harness/benchmarks/cards/simpleqa-verified.md +75 -0
  10. harness/benchmarks/cards/swde.json +47 -0
  11. harness/benchmarks/cards/trajectory-replay.md +82 -0
  12. harness/benchmarks/cards/wcxb.md +116 -0
  13. harness/benchmarks/cards/web-extract.md +119 -0
  14. harness/benchmarks/cards/webarena-replay.md +91 -0
  15. harness/benchmarks/cards/webfetch.md +297 -0
  16. harness/benchmarks/cards/webmainbench.md +63 -0
  17. harness/benchmarks/cards/websearch.md +98 -0
  18. harness/benchmarks/cards/zyte-product-extraction.json +42 -0
  19. harness/benchmarks/code-migration.json +4554 -0
  20. harness/benchmarks/data-lock.json +129219 -0
  21. harness/benchmarks/data.py +397 -0
  22. harness/benchmarks/pinning.py +328 -0
  23. harness/benchmarks/publication.py +780 -0
  24. harness/benchmarks/scoring-regressions.json +90025 -0
  25. harness/browser_use/__init__.py +5 -0
  26. harness/browser_use/__main__.py +4 -0
  27. harness/browser_use/assets/evidence-viewer.js +1 -0
  28. harness/browser_use/assets/report.css +63 -0
  29. harness/browser_use/assets/report.js +12 -0
  30. harness/browser_use/assets/webarena-autonomous-evaluation-flow.png +0 -0
  31. harness/browser_use/cli.py +46 -0
  32. harness/browser_use/failure_cause_review.py +160 -0
  33. harness/browser_use/messages.json +699 -0
  34. harness/browser_use/model.py +258 -0
  35. harness/browser_use/publication.py +191 -0
  36. harness/browser_use/report.py +4261 -0
  37. harness/browser_use/report_check.py +982 -0
  38. harness/browser_use/report_i18n.py +175 -0
  39. harness/browser_use/report_package.py +100 -0
  40. harness/browser_use/report_publish.py +194 -0
  41. harness/browser_use/report_visual.py +265 -0
  42. harness/browser_use/review.py +170 -0
  43. harness/browser_use/runner.py +811 -0
  44. harness/browser_use/selection.py +346 -0
  45. harness/container_isolation.py +46 -0
  46. harness/data_home.py +15 -0
  47. harness/distribution.py +46 -0
  48. harness/distribution_data/configs/providers.headless.json +120 -0
  49. harness/distribution_data/evaluation/browser-use-public/adoption-evidence.json +122 -0
  50. harness/distribution_data/evaluation/browser-use-public/benchmark-catalog.json +244 -0
  51. harness/distribution_data/evaluation/browser-use-public/report-changes.json +55 -0
  52. harness/distribution_data/evaluation/webmainbench-full/scoring-regressions.json +43 -0
  53. harness/distribution_data/pyproject.toml +209 -0
  54. harness/distribution_data/uv.lock +5088 -0
  55. harness/env_file.py +125 -0
  56. harness/environments/__init__.py +1 -0
  57. harness/environments/browser.py +150 -0
  58. harness/environments/webarena/__init__.py +1 -0
  59. harness/environments/webarena/backend.py +108 -0
  60. harness/environments/webarena/cli.py +88 -0
  61. harness/environments/webarena/config.py +280 -0
  62. harness/environments/webarena/har.py +835 -0
  63. harness/environments/webarena/manager.py +204 -0
  64. harness/environments/webarena/map_archives.py +60 -0
  65. harness/environments/webarena/map_check.py +103 -0
  66. harness/environments/webarena/map_detail.py +149 -0
  67. harness/environments/webarena/map_setup.py +537 -0
  68. harness/environments/webarena/rainbond.py +1185 -0
  69. harness/environments/webarena/runtime/php-fpm-qemu.ini +12 -0
  70. harness/environments/webarena/runtime.py +736 -0
  71. harness/environments/webarena/topology.json +81 -0
  72. harness/environments/webarena/topology.py +238 -0
  73. harness/fix_log.py +305 -0
  74. harness/lexbench/__init__.py +1 -0
  75. harness/lexbench/__main__.py +4 -0
  76. harness/lexbench/assets/bundle-loader.js +75 -0
  77. harness/lexbench/assets/render_contract.cjs +382 -0
  78. harness/lexbench/assets/ui_check.js +636 -0
  79. harness/lexbench/charts.py +469 -0
  80. harness/lexbench/cli.py +45 -0
  81. harness/lexbench/pack.py +136 -0
  82. harness/lexbench/report.py +2132 -0
  83. harness/lexbench/reproduction.py +234 -0
  84. harness/lexbench/resource_evidence.py +199 -0
  85. harness/lexbench/resource_matrix.py +298 -0
  86. harness/lexbench/resource_profiles.py +315 -0
  87. harness/lexbench/resource_summary.py +277 -0
  88. harness/lexbench/template/README.md +11 -0
  89. harness/lexbench/template/app.js +1607 -0
  90. harness/lexbench/template/index.html +51 -0
  91. harness/lexbench/template/messages.json +744 -0
  92. harness/lexbench/template/style.css +905 -0
  93. harness/lexbench/verification.py +1239 -0
  94. harness/licenses/Lexbench-Apache-2.0.txt +201 -0
  95. harness/licenses/WCXB-CC-BY-4.0.txt +25 -0
  96. harness/numbers.py +14 -0
  97. harness/package_gate.py +84 -0
  98. harness/platform_capabilities.py +37 -0
  99. harness/record_io.py +107 -0
  100. harness/report_languages.py +246 -0
  101. harness/reporting/__init__.py +1 -0
  102. harness/reporting/assets.py +182 -0
  103. harness/reporting/certified_cli.py +24 -0
  104. harness/reporting/links.py +168 -0
  105. harness/reporting/locales.py +30 -0
  106. harness/reporting/mermaid.py +28 -0
  107. harness/reporting/packs.py +336 -0
  108. harness/reporting/presentation.py +176 -0
  109. harness/reporting/pricing.py +29 -0
  110. harness/reporting/privacy.py +95 -0
  111. harness/reporting/publication.py +561 -0
  112. harness/reporting/publication_target.py +92 -0
  113. harness/reporting/publish.py +894 -0
  114. harness/reporting/release_check.py +51 -0
  115. harness/reporting/source_policy.py +64 -0
  116. harness/reporting/templates.py +109 -0
  117. harness/reporting/versions.py +135 -0
  118. harness/reporting/visual.py +215 -0
  119. harness/resource_accounting.py +256 -0
  120. harness/resource_calibration.py +287 -0
  121. harness/results/__init__.py +5 -0
  122. harness/results/__main__.py +3 -0
  123. harness/results/cli.py +123 -0
  124. harness/results/client.py +783 -0
  125. harness/results/versions.py +72 -0
  126. harness/review_runner.py +122 -0
  127. harness/trajectory_replay/__init__.py +5 -0
  128. harness/trajectory_replay/__main__.py +4 -0
  129. harness/trajectory_replay/assets/compact-data-fetch-adapter.js +25 -0
  130. harness/trajectory_replay/assets/compact-detail-loader.js +11 -0
  131. harness/trajectory_replay/assets/compact-evidence-loader.js +11 -0
  132. harness/trajectory_replay/assets/curated-execution-filter.js +38 -0
  133. harness/trajectory_replay/assets/curated-report.css +6 -0
  134. harness/trajectory_replay/assets/evidence-dialog.js +40 -0
  135. harness/trajectory_replay/assets/public-webarena-benchmark-tabs.js +4 -0
  136. harness/trajectory_replay/assets/public-webarena-comparison.js +4 -0
  137. harness/trajectory_replay/assets/public-webarena-execution-filter.js +39 -0
  138. harness/trajectory_replay/assets/public-webarena-report.css +13 -0
  139. harness/trajectory_replay/assets/suite-filter.js +7 -0
  140. harness/trajectory_replay/assets/webarena-article.css +1 -0
  141. harness/trajectory_replay/assets/webarena-catalog.css +12 -0
  142. harness/trajectory_replay/assets/webarena-report.css +52 -0
  143. harness/trajectory_replay/assets/webarena-report.js +280 -0
  144. harness/trajectory_replay/browser_use_seeds.py +375 -0
  145. harness/trajectory_replay/catalog_acceptance.py +717 -0
  146. harness/trajectory_replay/catalog_action_audit.py +190 -0
  147. harness/trajectory_replay/catalog_binding.py +198 -0
  148. harness/trajectory_replay/catalog_comparison.py +423 -0
  149. harness/trajectory_replay/catalog_consolidation.py +330 -0
  150. harness/trajectory_replay/catalog_contracts.py +833 -0
  151. harness/trajectory_replay/catalog_release.py +80 -0
  152. harness/trajectory_replay/catalog_runner.py +832 -0
  153. harness/trajectory_replay/cli.py +309 -0
  154. harness/trajectory_replay/core/__init__.py +1 -0
  155. harness/trajectory_replay/core/diagnostics.py +263 -0
  156. harness/trajectory_replay/core/docker_runner.py +374 -0
  157. harness/trajectory_replay/core/executor.py +72 -0
  158. harness/trajectory_replay/core/metrics.py +365 -0
  159. harness/trajectory_replay/core/report.py +113 -0
  160. harness/trajectory_replay/core/suite.py +190 -0
  161. harness/trajectory_replay/diagnostics.py +685 -0
  162. harness/trajectory_replay/evaluator_transition.py +434 -0
  163. harness/trajectory_replay/executor.py +165 -0
  164. harness/trajectory_replay/expert_seeds.py +188 -0
  165. harness/trajectory_replay/failure_matrix.py +280 -0
  166. harness/trajectory_replay/interpretation.py +186 -0
  167. harness/trajectory_replay/messages.json +1002 -0
  168. harness/trajectory_replay/messages.py +11 -0
  169. harness/trajectory_replay/metrics.py +35 -0
  170. harness/trajectory_replay/participants.py +97 -0
  171. harness/trajectory_replay/playwright_tool.py +1331 -0
  172. harness/trajectory_replay/public_webarena/__init__.py +5 -0
  173. harness/trajectory_replay/public_webarena/cli.py +204 -0
  174. harness/trajectory_replay/public_webarena/config.py +79 -0
  175. harness/trajectory_replay/public_webarena/diagnostics.py +1163 -0
  176. harness/trajectory_replay/public_webarena/evaluators.py +155 -0
  177. harness/trajectory_replay/public_webarena/executor.py +695 -0
  178. harness/trajectory_replay/public_webarena/har.py +393 -0
  179. harness/trajectory_replay/public_webarena/manifest_repair.py +393 -0
  180. harness/trajectory_replay/public_webarena/metrics.py +47 -0
  181. harness/trajectory_replay/public_webarena/publication.py +492 -0
  182. harness/trajectory_replay/public_webarena/qualification.py +2765 -0
  183. harness/trajectory_replay/public_webarena/release.py +171 -0
  184. harness/trajectory_replay/public_webarena/report.py +1903 -0
  185. harness/trajectory_replay/public_webarena/response_builder.py +327 -0
  186. harness/trajectory_replay/public_webarena/result_audit.py +185 -0
  187. harness/trajectory_replay/public_webarena/runner.py +908 -0
  188. harness/trajectory_replay/public_webarena/suite.py +410 -0
  189. harness/trajectory_replay/public_webarena/targets.py +93 -0
  190. harness/trajectory_replay/public_webarena/webarena.py +735 -0
  191. harness/trajectory_replay/public_webarena/webarena_environment.py +735 -0
  192. harness/trajectory_replay/remote_execution.py +1581 -0
  193. harness/trajectory_replay/remote_runtime.py +1071 -0
  194. harness/trajectory_replay/report.py +1236 -0
  195. harness/trajectory_replay/report_compaction.py +217 -0
  196. harness/trajectory_replay/report_inputs.py +170 -0
  197. harness/trajectory_replay/report_languages.py +14 -0
  198. harness/trajectory_replay/resources/fixtures/app.html +11 -0
  199. harness/trajectory_replay/resources/fixtures/catalog.html +9 -0
  200. harness/trajectory_replay/resources/fixtures/core.html +18 -0
  201. harness/trajectory_replay/resources/fixtures/dynamic.html +11 -0
  202. harness/trajectory_replay/resources/fixtures/forms.html +11 -0
  203. harness/trajectory_replay/resources/fixtures/interaction-space.html +142 -0
  204. harness/trajectory_replay/resources/fixtures/page2.html +3 -0
  205. harness/trajectory_replay/resources/fixtures/page3.html +2 -0
  206. harness/trajectory_replay/retrieval_acceptance.py +1080 -0
  207. harness/trajectory_replay/runner.py +304 -0
  208. harness/trajectory_replay/runtime_identity.py +131 -0
  209. harness/trajectory_replay/suite.py +83 -0
  210. harness/trajectory_replay/webarena_corrections.py +1088 -0
  211. harness/trajectory_replay/webarena_dataset.py +684 -0
  212. harness/trajectory_replay/webarena_plan.py +658 -0
  213. harness/trajectory_replay/webarena_replay.py +1657 -0
  214. harness/trajectory_replay/webarena_report.py +2290 -0
  215. harness/trajectory_replay/webarena_report_check.py +944 -0
  216. harness/trajectory_replay/webarena_report_package.py +155 -0
  217. harness/trajectory_replay/webarena_report_publish.py +230 -0
  218. harness/trajectory_replay/webarena_report_visual.py +693 -0
  219. harness/trajectory_replay/webarena_results.py +644 -0
  220. harness/trajectory_replay/webarena_review.py +198 -0
  221. harness/trajectory_replay/webarena_runtime_source.py +119 -0
  222. harness/web_extract/__init__.py +1 -0
  223. harness/web_extract/analysis.py +683 -0
  224. harness/web_extract/assets/report.css +58 -0
  225. harness/web_extract/assets/report.js +431 -0
  226. harness/web_extract/certification.py +596 -0
  227. harness/web_extract/cli.py +436 -0
  228. harness/web_extract/dataset.py +274 -0
  229. harness/web_extract/example.py +473 -0
  230. harness/web_extract/firecrawl_backend.mjs +178 -0
  231. harness/web_extract/messages.json +1070 -0
  232. harness/web_extract/messages.py +31 -0
  233. harness/web_extract/model_comparison.py +613 -0
  234. harness/web_extract/pipeline.py +2580 -0
  235. harness/web_extract/publication.py +209 -0
  236. harness/web_extract/records.py +574 -0
  237. harness/web_extract/repair.py +657 -0
  238. harness/web_extract/report.py +6167 -0
  239. harness/web_extract/report_suite.py +353 -0
  240. harness/web_extract/report_ui.py +66 -0
  241. harness/web_extract/review.py +1580 -0
  242. harness/web_extract/self_hosted.py +978 -0
  243. harness/web_extract/snapshots.py +1197 -0
  244. harness/web_extract/swde.py +742 -0
  245. harness/web_fetch/__init__.py +5 -0
  246. harness/web_fetch/__main__.py +5 -0
  247. harness/web_fetch/accounts.py +130 -0
  248. harness/web_fetch/adjudication_evidence.py +767 -0
  249. harness/web_fetch/anomaly_review.py +346 -0
  250. harness/web_fetch/artifact_auth.py +135 -0
  251. harness/web_fetch/artifacts.py +521 -0
  252. harness/web_fetch/assembly.py +1324 -0
  253. harness/web_fetch/assets/cohort-diagnostic.css +18 -0
  254. harness/web_fetch/assets/cohort-diagnostic.js +2 -0
  255. harness/web_fetch/assets/methodology/annotation.png +0 -0
  256. harness/web_fetch/assets/methodology/scoring.png +0 -0
  257. harness/web_fetch/assets/methodology/validation.png +0 -0
  258. harness/web_fetch/assets/report-extra.css +64 -0
  259. harness/web_fetch/assets/report-loader.js +167 -0
  260. harness/web_fetch/assets/report-style.css +68 -0
  261. harness/web_fetch/assets/report.js +57 -0
  262. harness/web_fetch/assets/wcxb.css +1 -0
  263. harness/web_fetch/assets/wcxb.js +1 -0
  264. harness/web_fetch/assets/webmainbench-full.css +1 -0
  265. harness/web_fetch/assets/webmainbench-full.js +52 -0
  266. harness/web_fetch/benchmark_runner.py +190 -0
  267. harness/web_fetch/browser_observations.py +980 -0
  268. harness/web_fetch/call_boundary.py +142 -0
  269. harness/web_fetch/certification.py +683 -0
  270. harness/web_fetch/cli.py +394 -0
  271. harness/web_fetch/cohort_exclusions.py +38 -0
  272. harness/web_fetch/comparison_audit.py +479 -0
  273. harness/web_fetch/config.py +231 -0
  274. harness/web_fetch/container_resources.py +220 -0
  275. harness/web_fetch/data/browser-observation-contracts.json +25 -0
  276. harness/web_fetch/data/fixture-dataset.json +42 -0
  277. harness/web_fetch/data/fixture-providers.json +11 -0
  278. harness/web_fetch/data/latency-certifiers/34bdb7aedb25a9e5a0c64e0c549926ed7e71f747f544ec2035f65952c0177347.py.txt +142 -0
  279. harness/web_fetch/data/latency-certifiers/95a2f0a73578346998f0a4ad1256b2cef469a3f2db9b4b30550fc03fc5c259a8.py.txt +142 -0
  280. harness/web_fetch/data/native-projection-history.json +35 -0
  281. harness/web_fetch/data/native-projection-regressions.json +531 -0
  282. harness/web_fetch/data/report-methods.json +6687 -0
  283. harness/web_fetch/data/report-track-context.json +132 -0
  284. harness/web_fetch/data/wcxb-metric-bindings.json +12 -0
  285. harness/web_fetch/data/webmainbench-latency-source.json +18 -0
  286. harness/web_fetch/dataset_assembly.py +274 -0
  287. harness/web_fetch/diagnostics.py +73 -0
  288. harness/web_fetch/document_text.py +55 -0
  289. harness/web_fetch/dynamic_cohort.py +467 -0
  290. harness/web_fetch/evidence.py +753 -0
  291. harness/web_fetch/failure_diagnosis.py +190 -0
  292. harness/web_fetch/headless_runner.py +510 -0
  293. harness/web_fetch/html_to_markdown.mjs +21 -0
  294. harness/web_fetch/local_audit.py +453 -0
  295. harness/web_fetch/local_execution_contract.py +317 -0
  296. harness/web_fetch/local_execution_legacy.py +205 -0
  297. harness/web_fetch/local_revision.py +1192 -0
  298. harness/web_fetch/markdown.py +32 -0
  299. harness/web_fetch/messages.json +2377 -0
  300. harness/web_fetch/methods.py +451 -0
  301. harness/web_fetch/models.py +70 -0
  302. harness/web_fetch/native_projection.py +769 -0
  303. harness/web_fetch/native_routes.py +637 -0
  304. harness/web_fetch/network_dns.py +363 -0
  305. harness/web_fetch/network_preflight.py +119 -0
  306. harness/web_fetch/provider_adoption.py +321 -0
  307. harness/web_fetch/providers/__init__.py +4 -0
  308. harness/web_fetch/providers/base.py +274 -0
  309. harness/web_fetch/providers/cloudflare.py +816 -0
  310. harness/web_fetch/providers/cowagent.py +78 -0
  311. harness/web_fetch/providers/exa.py +177 -0
  312. harness/web_fetch/providers/firecrawl.py +454 -0
  313. harness/web_fetch/providers/fixture.py +49 -0
  314. harness/web_fetch/providers/headless.py +1147 -0
  315. harness/web_fetch/providers/headless_worker.py +512 -0
  316. harness/web_fetch/providers/hosted_fetch.py +250 -0
  317. harness/web_fetch/providers/http.py +87 -0
  318. harness/web_fetch/providers/http_client.py +112 -0
  319. harness/web_fetch/providers/lexmount.py +211 -0
  320. harness/web_fetch/providers/moli.py +889 -0
  321. harness/web_fetch/providers/runtime_identity.py +220 -0
  322. harness/web_fetch/providers/subprocess_jsonl.py +410 -0
  323. harness/web_fetch/providers/tavily.py +116 -0
  324. harness/web_fetch/public_benchmark_sample.py +160 -0
  325. harness/web_fetch/publication.py +26 -0
  326. harness/web_fetch/publication_check.py +379 -0
  327. harness/web_fetch/publication_privacy.py +1136 -0
  328. harness/web_fetch/publication_verify.py +390 -0
  329. harness/web_fetch/published_snapshot.py +379 -0
  330. harness/web_fetch/reference_freeze.py +699 -0
  331. harness/web_fetch/reference_inputs.py +387 -0
  332. harness/web_fetch/reference_lifecycle.py +1811 -0
  333. harness/web_fetch/reference_revision.py +408 -0
  334. harness/web_fetch/refresh_assembly.py +1723 -0
  335. harness/web_fetch/reliability.py +80 -0
  336. harness/web_fetch/render_evidence.py +215 -0
  337. harness/web_fetch/report-translations.json +3456 -0
  338. harness/web_fetch/report.py +4564 -0
  339. harness/web_fetch/report_build.py +1851 -0
  340. harness/web_fetch/report_capabilities.py +149 -0
  341. harness/web_fetch/report_changes.py +224 -0
  342. harness/web_fetch/report_check.py +1004 -0
  343. harness/web_fetch/report_components.py +512 -0
  344. harness/web_fetch/report_diagnostic.py +857 -0
  345. harness/web_fetch/report_explainer.py +1133 -0
  346. harness/web_fetch/report_i18n.py +942 -0
  347. harness/web_fetch/report_interfaces.py +262 -0
  348. harness/web_fetch/report_methods.py +159 -0
  349. harness/web_fetch/report_package.py +1000 -0
  350. harness/web_fetch/report_pricing.py +120 -0
  351. harness/web_fetch/report_publish.py +158 -0
  352. harness/web_fetch/report_quality.py +34 -0
  353. harness/web_fetch/report_reading.py +124 -0
  354. harness/web_fetch/report_resources.py +196 -0
  355. harness/web_fetch/report_saved_composition.py +149 -0
  356. harness/web_fetch/report_saved_matrix.py +2662 -0
  357. harness/web_fetch/report_style.py +9 -0
  358. harness/web_fetch/report_version.py +270 -0
  359. harness/web_fetch/rescoring.py +332 -0
  360. harness/web_fetch/resource_cohort.py +207 -0
  361. harness/web_fetch/result_ledger.py +119 -0
  362. harness/web_fetch/result_review.py +1252 -0
  363. harness/web_fetch/retrieval_hints.py +574 -0
  364. harness/web_fetch/review_execution.py +1638 -0
  365. harness/web_fetch/risk_control.py +524 -0
  366. harness/web_fetch/runner.py +1241 -0
  367. harness/web_fetch/safety.py +140 -0
  368. harness/web_fetch/scoring_revision.py +391 -0
  369. harness/web_fetch/source_capture.py +455 -0
  370. harness/web_fetch/speed_cohort.py +216 -0
  371. harness/web_fetch/storage.py +336 -0
  372. harness/web_fetch/task_outputs.py +346 -0
  373. harness/web_fetch/templates/interpretation-headless.css +326 -0
  374. harness/web_fetch/title_quality.py +93 -0
  375. harness/web_fetch/wcxb_dataset.py +168 -0
  376. harness/web_fetch/wcxb_latency.py +347 -0
  377. harness/web_fetch/wcxb_metrics.py +38 -0
  378. harness/web_fetch/wcxb_publication.py +147 -0
  379. harness/web_fetch/wcxb_report_check.py +753 -0
  380. harness/web_fetch/wcxb_scoring.py +667 -0
  381. harness/web_fetch/webmainbench_assembly.py +684 -0
  382. harness/web_fetch/webmainbench_case_correction.py +267 -0
  383. harness/web_fetch/webmainbench_certification.py +306 -0
  384. harness/web_fetch/webmainbench_correction.py +866 -0
  385. harness/web_fetch/webmainbench_diagnostics.py +1202 -0
  386. harness/web_fetch/webmainbench_evidence.py +205 -0
  387. harness/web_fetch/webmainbench_latency_correction.py +525 -0
  388. harness/web_fetch/webmainbench_latency_diagnostic.py +562 -0
  389. harness/web_fetch/webmainbench_report_check.py +900 -0
  390. harness/web_fetch/webmainbench_runner.py +680 -0
  391. harness/web_fetch/webmainbench_scoring.py +354 -0
  392. harness/web_fetch/workers/__init__.py +1 -0
  393. harness/web_fetch/workers/cowagent_worker.py +70 -0
  394. harness/web_fetch/workers/crawl4ai_worker.py +463 -0
  395. harness/web_fetch/workers/dsh_output.mjs +21 -0
  396. harness/web_fetch/workers/dsh_webfetch_worker.mts +100 -0
  397. harness/web_fetch/workers/langchain_webbase_worker.py +67 -0
  398. harness/web_fetch/workers/native_dns_darwin.c +108 -0
  399. harness/web_fetch/workers/openclaw_webfetch_worker.mts +91 -0
  400. harness/web_fetch/workers/public_dns.mjs +49 -0
  401. harness/web_search/__init__.py +1 -0
  402. harness/web_search/__main__.py +4 -0
  403. harness/web_search/adapters.py +371 -0
  404. harness/web_search/answers.py +326 -0
  405. harness/web_search/assets/comparison.css +5 -0
  406. harness/web_search/assets/comparison.js +45 -0
  407. harness/web_search/assets/delivery-bridge.js +35 -0
  408. harness/web_search/assets/delivery-loader.js +35 -0
  409. harness/web_search/assets/native-evidence-result.js +8 -0
  410. harness/web_search/assets/report.css +69 -0
  411. harness/web_search/assets/report.js +58 -0
  412. harness/web_search/cli.py +44 -0
  413. harness/web_search/comparison.py +843 -0
  414. harness/web_search/core.py +274 -0
  415. harness/web_search/data/native-normalization-regressions.json +354 -0
  416. harness/web_search/delivery.py +163 -0
  417. harness/web_search/diagnostics.py +132 -0
  418. harness/web_search/example.py +147 -0
  419. harness/web_search/messages.json +751 -0
  420. harness/web_search/messages.py +23 -0
  421. harness/web_search/output_levels.py +431 -0
  422. harness/web_search/pages.py +28 -0
  423. harness/web_search/profiles.py +220 -0
  424. harness/web_search/publication.py +208 -0
  425. harness/web_search/relevance.py +828 -0
  426. harness/web_search/report.py +4315 -0
  427. harness/web_search/review.py +1318 -0
  428. harness/web_search/revisions.py +1052 -0
  429. harness/web_search/runner.py +1013 -0
  430. harness/web_search/visual.py +164 -0
  431. harness/webarena_state.py +549 -0
  432. lexbench_webfetch/__init__.py +1 -0
  433. lexbench_webfetch/annotation_revisions.py +269 -0
  434. lexbench_webfetch/article_comments.py +331 -0
  435. lexbench_webfetch/article_dataset.py +285 -0
  436. lexbench_webfetch/article_posts.py +384 -0
  437. lexbench_webfetch/article_qualification.py +192 -0
  438. lexbench_webfetch/article_text.py +259 -0
  439. lexbench_webfetch/article_title.py +186 -0
  440. lexbench_webfetch/benchmark.py +591 -0
  441. lexbench_webfetch/body_coverage.py +582 -0
  442. lexbench_webfetch/cli.py +143 -0
  443. lexbench_webfetch/composite_tasks.py +2026 -0
  444. lexbench_webfetch/content_scopes.py +156 -0
  445. lexbench_webfetch/document_structure.py +326 -0
  446. lexbench_webfetch/dynamic_entry_matching.py +190 -0
  447. lexbench_webfetch/dynamic_lists.py +1658 -0
  448. lexbench_webfetch/dynamic_reference.py +25 -0
  449. lexbench_webfetch/evaluation.py +24 -0
  450. lexbench_webfetch/file_acceptance.py +72 -0
  451. lexbench_webfetch/inputs.py +129 -0
  452. lexbench_webfetch/list_verification.py +90 -0
  453. lexbench_webfetch/metrics.py +1441 -0
  454. lexbench_webfetch/models.py +93 -0
  455. lexbench_webfetch/ordered_tree.py +115 -0
  456. lexbench_webfetch/paths.py +41 -0
  457. lexbench_webfetch/reference_quality.py +1104 -0
  458. lexbench_webfetch/required_components.py +44 -0
  459. lexbench_webfetch/resource_urls.py +35 -0
  460. lexbench_webfetch/response_input.py +135 -0
  461. lexbench_webfetch/risk_control_dataset.py +512 -0
  462. lexbench_webfetch/scoring.py +452 -0
  463. lexbench_webfetch/scoring_regressions.py +215 -0
  464. lexbench_webfetch/stable_tasks.py +2144 -0
  465. lexbench_webfetch/structured_tables.py +793 -0
  466. lexbench_webfetch/versions.py +224 -0
  467. lexbench_websearch/__init__.py +5 -0
  468. lexbench_websearch/answer_rules.py +322 -0
  469. lexbench_websearch/answers.py +383 -0
  470. lexbench_websearch/cli.py +123 -0
  471. lexbench_websearch/contracts.py +58 -0
  472. lexbench_websearch/evidence_quality.py +109 -0
  473. lexbench_websearch/inputs.py +80 -0
  474. lexbench_websearch/outputs.py +161 -0
  475. lexbench_websearch/qualification.py +144 -0
  476. lexbench_websearch/relevance.py +346 -0
  477. lexbench_websearch/rubric.py +34 -0
  478. lexbench_websearch/scoring.py +354 -0
  479. lexbench_websearch/simpleqa.py +467 -0
  480. lexbench_websearch/versions.py +255 -0
  481. webstone-0.8.0.dist-info/METADATA +367 -0
  482. webstone-0.8.0.dist-info/RECORD +488 -0
  483. webstone-0.8.0.dist-info/WHEEL +5 -0
  484. webstone-0.8.0.dist-info/entry_points.txt +4 -0
  485. webstone-0.8.0.dist-info/licenses/LICENSE +202 -0
  486. webstone-0.8.0.dist-info/licenses/NOTICE +6 -0
  487. webstone-0.8.0.dist-info/licenses/THIRD_PARTY.md +22 -0
  488. webstone-0.8.0.dist-info/top_level.txt +3 -0
@@ -0,0 +1,353 @@
1
+ """Compose independent record benchmarks through the maintained report owner."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+ from html import escape
7
+ from pathlib import Path
8
+
9
+ from bs4 import BeautifulSoup
10
+
11
+ from harness.web_extract import report as owner
12
+ from harness.web_extract.pipeline import digest, read_json, require_external_output, save_json
13
+
14
+ MODE = "field-extraction-suite"
15
+
16
+
17
+ def cohort_registry(entries):
18
+ """Public scope authority: ordered cohorts and their exact child inputs."""
19
+ return [
20
+ {key: entry.get(key, "") for key in (
21
+ "id", "label", "summary_html", "content_sha256", "profile_sha256"
22
+ )}
23
+ for entry in entries
24
+ ]
25
+
26
+
27
+ def suite_inputs(profile):
28
+ """Resolve private child inputs from the suite's hash-bound content."""
29
+ authority = read_json(profile)
30
+ binding = authority.get("research_report", {}).get("content", {})
31
+ if not binding.get("path"):
32
+ raise ValueError("Suite authority requires its bound content path")
33
+ content = profile.parent / binding["path"]
34
+ if content.is_symlink() or digest(content.read_bytes()) != binding.get("sha256"):
35
+ raise ValueError("Suite content differs from its bound input profile")
36
+ entries = read_json(content)["cohorts"]
37
+ if cohort_registry(entries) != authority.get("research_report", {}).get("cohorts"):
38
+ raise ValueError("Suite registry differs from its profile-authorized content")
39
+ for entry in entries:
40
+ for key in ("content", "profile"):
41
+ path = Path(entry[key])
42
+ require_external_output(path)
43
+ if path.is_symlink() or digest(path.read_bytes()) != entry.get(key + "_sha256"):
44
+ raise ValueError("Suite child input differs from its declared binding")
45
+ return content, entries
46
+
47
+
48
+ def section(output, entry, language="zh"):
49
+ child = output / entry["id"]
50
+ manifest = read_json(child / "report-manifest.json")
51
+ metrics = read_json(child / "data/metrics.json")
52
+ methods = manifest["planned_methods"]
53
+ contracts = owner.selected_execution_contracts(
54
+ read_json(child / "public-evidence/sources.json").values()
55
+ )
56
+ latency_compared = (set(manifest["comparison_methods"])
57
+ if owner.latency_comparable(contracts, methods) else set())
58
+ def t(key, **values):
59
+ return owner.message(key, language, **values)
60
+
61
+ def score(method, family, field):
62
+ if method not in manifest["comparison_methods"]:
63
+ return None
64
+ scores = metrics[method]["scores"]
65
+ return scores[family][field] if scores is not None else None
66
+
67
+ quality, operation = [], []
68
+ for method in methods:
69
+ current = metrics[method]
70
+ latency = current["latency"]
71
+ quality.append(
72
+ [
73
+ escape(manifest["method_labels"][method])
74
+ + (
75
+ "<br><small>" + escape(manifest["skipped_methods"][method]) + "</small>"
76
+ if method in manifest["skipped_methods"]
77
+ else ""
78
+ ),
79
+ f"{current['executed']} / {current['planned']}",
80
+ owner.record_metric(score(method, "value_micro", "f1")),
81
+ owner.record_metric(score(method, "page_hit_macro", "f1")),
82
+ owner.record_metric(score(method, "exact_field_sets", "rate"), True),
83
+ f'<a href="{entry["id"]}/{language}/interpretation.html#{method}">'
84
+ + t("suite_results") + "</a>",
85
+ ]
86
+ )
87
+ operation.append(
88
+ [
89
+ escape(manifest["method_labels"][method]),
90
+ f"{latency['timed']} / {current['planned']}",
91
+ *[owner.record_metric(latency[key]) for key in ("mean", "p50", "p95")],
92
+ escape(owner.price_quote(manifest["prices"][method], language)),
93
+ f'<a href="{entry["id"]}/{language}/accounts.html">{t("suite_pricing")}</a>',
94
+ ]
95
+ )
96
+ quality_table = owner.record_comparison(
97
+ t("suite_quality_headers"),
98
+ quality,
99
+ {
100
+ column: (key, "max", [score(m, family, field) for m in methods])
101
+ for column, key, family, field in [
102
+ (2, "micro-f1", "value_micro", "f1"),
103
+ (3, "macro-f1", "page_hit_macro", "f1"),
104
+ (4, "exact-rate", "exact_field_sets", "rate"),
105
+ ]
106
+ },
107
+ )
108
+ time_table = owner.record_comparison(
109
+ t("suite_operation_headers"),
110
+ operation,
111
+ {
112
+ column: (key, "min" if latency_compared else "neutral", [
113
+ metrics[m]["latency"][key] if m in latency_compared else None for m in methods
114
+ ])
115
+ for column, key in [(2, "mean"), (3, "p50"), (4, "p95")]
116
+ },
117
+ )
118
+ return (
119
+ f'<section data-cohort="{entry["id"]}" id="{entry["id"]}"><h2>{escape(entry["label"])} · '
120
+ + t("suite_cases", count=manifest["case_count"]) + "</h2>"
121
+ + entry.get("summary_html", "")
122
+ + quality_table
123
+ + "<h3>" + t("latency_and_pricing") + "</h3>"
124
+ + time_table
125
+ + f'<p><a href="{entry["id"]}/{language}/dataset.html">{t("suite_dataset")}</a> · '
126
+ f'<a href="{entry["id"]}/{language}/protocol.html">{t("suite_examples")}</a> · '
127
+ f'<a href="{entry["id"]}/{language}/results.html">{t("suite_case_results")}</a>'
128
+ "</p></section>"
129
+ )
130
+
131
+
132
+ def build(output, content, *, profile, language=None):
133
+ require_external_output(output)
134
+ language = owner.resolve_report_language(output, language, supported=("zh",))
135
+ model, authority = read_json(content), read_json(profile)
136
+ if authority.get("research_report", {}).get("content", {}).get("sha256") != digest(
137
+ content.read_bytes()
138
+ ):
139
+ raise ValueError("Suite content differs from its instance profile")
140
+ entries = model["cohorts"]
141
+ names = [entry["id"] for entry in entries]
142
+ if (
143
+ not names
144
+ or len(set(names)) != len(names)
145
+ or any(
146
+ name in {"zh", "en", "public-evidence", "assets"}
147
+ or not re.fullmatch(r"[a-z0-9]+(?:-[a-z0-9]+)*", name)
148
+ for name in names
149
+ )
150
+ ):
151
+ raise ValueError("Suite cohorts need distinct safe directory names")
152
+ for entry in entries:
153
+ child_content = read_json(Path(entry["content"]))
154
+ if child_content.get("site_prefix") != entry["id"]:
155
+ raise ValueError("Each cohort must declare its suite directory")
156
+ for key in ("content", "profile"):
157
+ if entry.get(key + "_sha256") != digest(Path(entry[key]).read_bytes()):
158
+ raise ValueError("Suite child input differs from its declared binding")
159
+ if authority.get("research_report", {}).get("cohorts") != cohort_registry(entries):
160
+ raise ValueError("Suite registry differs from its profile-authorized content")
161
+ if owner.public_risk(profile.read_bytes()):
162
+ raise ValueError("Suite authority must contain only publication-safe configuration")
163
+ if output.exists() and any(output.iterdir()):
164
+ raise ValueError("Use an empty suite report directory")
165
+ output.mkdir(parents=True, exist_ok=True)
166
+ (output / "index.html").write_text(
167
+ '<!doctype html><html><head><meta charset="utf-8">'
168
+ '<meta http-equiv="refresh" content="0;url=zh/index.html"></head>'
169
+ '<body><a href="zh/index.html">Web Extract</a></body></html>',
170
+ encoding="utf-8",
171
+ )
172
+ bindings = []
173
+ for entry in entries:
174
+ owner.build(
175
+ output / entry["id"],
176
+ Path(entry["content"]),
177
+ home=Path(entry["home"]),
178
+ profile=Path(entry["profile"]),
179
+ language=language,
180
+ )
181
+ bindings.append(
182
+ {key: entry[key] for key in ("id", "label")}
183
+ | {
184
+ "summary_html": entry.get("summary_html", ""),
185
+ "manifest_sha256": digest(
186
+ (output / entry["id"] / "report-manifest.json").read_bytes()
187
+ ),
188
+ }
189
+ )
190
+ for directory in ("zh", "en"):
191
+ (output / directory).mkdir()
192
+ (output / "style.css").write_text(owner.CSS, encoding="utf-8")
193
+ (output / "app.js").write_text(owner.JS, encoding="utf-8")
194
+ owner.copy_report_assets(output, content, model)
195
+ body = model.get("intro_html", "") + "".join(
196
+ section(output, entry, language) for entry in bindings
197
+ )
198
+ body += owner.report_metric_guide(language, records=True) + model.get("interpretation_html", "")
199
+ (output / "zh/index.html").write_text(
200
+ owner.page(
201
+ language, "index", model["title"], body, languages=(language,), state=MODE,
202
+ notice=owner.message("suite_notice", language),
203
+ navigation=[{"route": f'{e["id"]}/{language}/index', "label": e["label"]}
204
+ for e in bindings],
205
+ ),
206
+ encoding="utf-8",
207
+ )
208
+ (output / "en/index.html").write_text(
209
+ owner.message("english_not_generated_notice", "en"), encoding="utf-8"
210
+ )
211
+ input_profile = owner.copy_input_profile(output, profile, preserve_bytes=True)
212
+ save_json(output / "suite.json", {"cohorts": bindings})
213
+ save_json(
214
+ output / "report-manifest.json",
215
+ {
216
+ "mode": MODE,
217
+ "template": owner.template_identity("web-extract"),
218
+ "template_version": owner.VERSION,
219
+ "webstone": owner.bind_build_version(output),
220
+ "renderer_sha256": digest(Path(owner.__file__).read_bytes()),
221
+ "composition_sha256": digest(Path(__file__).read_bytes()),
222
+ **owner.report_dependencies(),
223
+ "input_profile": input_profile,
224
+ "profile_sha256": digest(profile.read_bytes()),
225
+ "content_sha256": digest(content.read_bytes()),
226
+ "local_example": authority.get("local_example", False),
227
+ "dataset_kind": "local-example"
228
+ if authority.get("local_example", False)
229
+ else "published-upstream",
230
+ "languages": ["zh"],
231
+ "primary_language": "zh",
232
+ "cohorts": bindings,
233
+ "files": {
234
+ p.relative_to(output).as_posix(): digest(p.read_bytes())
235
+ for p in sorted(output.rglob("*"))
236
+ if p.is_file()
237
+ },
238
+ },
239
+ exclusive=True,
240
+ )
241
+ return _check_rendered_suite(output)
242
+
243
+
244
+ def _check_rendered_suite(output, *, release=False, visual_receipt=None):
245
+ output = owner.open_report(output)
246
+ manifest = read_json(output / "report-manifest.json")
247
+ owner.check_report_identity(output, manifest, "web-extract")
248
+ owner.check_input_profile(output, manifest)
249
+ if (
250
+ manifest.get("mode") != MODE
251
+ or manifest.get("renderer_sha256") != digest(Path(owner.__file__).read_bytes())
252
+ or manifest.get("composition_sha256") != digest(Path(__file__).read_bytes())
253
+ ):
254
+ raise ValueError("Suite renderer changed; rebuild required")
255
+ actual = {p.relative_to(output).as_posix() for p in output.rglob("*") if p.is_file()}
256
+ if actual != set(manifest["files"]) | {"report-manifest.json"}:
257
+ raise ValueError("Suite inventory changed")
258
+ for name, sha in manifest["files"].items():
259
+ path = output / name
260
+ if (
261
+ path.is_symlink()
262
+ or digest(path.read_bytes()) != sha
263
+ or owner.public_risk(path.read_bytes())
264
+ ):
265
+ raise ValueError("Suite file changed or has a privacy risk: " + name)
266
+ entries = read_json(output / "suite.json")["cohorts"]
267
+ if entries != manifest["cohorts"]:
268
+ raise ValueError("Suite cohort bindings changed")
269
+ authority = read_json(output / manifest["input_profile"]["path"])
270
+ if (
271
+ digest((output / manifest["input_profile"]["path"]).read_bytes())
272
+ != manifest.get("profile_sha256")
273
+ or authority.get("research_report", {}).get("content", {}).get("sha256")
274
+ != manifest.get("content_sha256")
275
+ ):
276
+ raise ValueError("Suite content differs from its bound input profile")
277
+ declared = authority.get("research_report", {}).get("cohorts")
278
+ if [{key: e.get(key, "") for key in ("id", "label", "summary_html")} for e in entries] != [
279
+ {key: e.get(key, "") for key in ("id", "label", "summary_html")} for e in declared or []
280
+ ]:
281
+ raise ValueError("Suite registry differs from its profile-authorized content")
282
+ names = [entry["id"] for entry in entries]
283
+ if (
284
+ not names
285
+ or len(names) != len(set(names))
286
+ or any(
287
+ name in {"zh", "en", "public-evidence", "assets"}
288
+ or not re.fullmatch(r"[a-z0-9]+(?:-[a-z0-9]+)*", name)
289
+ for name in names
290
+ )
291
+ ):
292
+ raise ValueError("Suite cohort directories changed")
293
+ visual = read_json(visual_receipt) if release and visual_receipt else {}
294
+ for entry, source in zip(entries, declared, strict=True):
295
+ child = output / entry["id"]
296
+ child_manifest = read_json(child / "report-manifest.json")
297
+ if any(child_manifest.get(key + "_sha256") != source.get(key + "_sha256")
298
+ for key in ("content", "profile")):
299
+ raise ValueError("Suite child input binding changed")
300
+ if digest((child / "report-manifest.json").read_bytes()) != entry["manifest_sha256"]:
301
+ raise ValueError("Suite cohort manifest changed")
302
+ receipt = visual.get("cohorts", {}).get(entry["id"], {})
303
+ if release and (
304
+ not receipt.get("path")
305
+ or digest(Path(receipt["path"]).read_bytes()) != receipt.get("sha256")
306
+ ):
307
+ raise ValueError("Suite requires each cohort's bound visual receipt")
308
+ owner._check_record_cohort(
309
+ child, release=release, visual_receipt=Path(receipt["path"]) if release else None,
310
+ )
311
+ from harness.reporting.presentation import check_metric_guide, check_presentation
312
+
313
+ soup = check_presentation((output / "zh/index.html").read_text(encoding="utf-8"))
314
+ expected = BeautifulSoup(owner.page(
315
+ "zh", "index", "", "".join(section(output, entry) for entry in entries),
316
+ languages=("zh",), state=MODE,
317
+ navigation=[{"route": f'{e["id"]}/zh/index', "label": e["label"]} for e in entries],
318
+ ), "html.parser")
319
+ if [str(node) for node in soup.select("[data-cohort]")] != [
320
+ str(node) for node in expected.select("[data-cohort]")
321
+ ]:
322
+ raise ValueError("Suite comparison differs from its independent cohort results")
323
+ check_metric_guide(
324
+ str(soup),
325
+ owner.metric_definitions("zh", records=True),
326
+ title=owner.message("guide_title", "zh"),
327
+ headings=owner.message("guide_headings", "zh"),
328
+ )
329
+ if [(a.get("href"), a.get_text()) for a in soup.select("nav a")] != [
330
+ (entry["id"] + "/zh/index.html", entry["label"]) for entry in entries
331
+ ]:
332
+ raise ValueError("Suite navigation differs from its benchmark registry")
333
+ notice = BeautifulSoup((output / "en/index.html").read_text(encoding="utf-8"), "html.parser")
334
+ if {p.name for p in (output / "en").glob("*.html")} != {"index.html"} or not notice.select_one(
335
+ '[data-language-notice] a[href="zh/index.html"]'
336
+ ):
337
+ raise ValueError("Suite English notice is missing or stale")
338
+ links = owner.LinkIndex(errors="strict")
339
+ for path in [output / "zh/index.html", output / "en/index.html"]:
340
+ for value in links.page(path).values:
341
+ parsed = owner.urlsplit(value)
342
+ if parsed.scheme or parsed.netloc:
343
+ continue
344
+ target = output / owner.unquote(parsed.path) if parsed.path else path
345
+ if not target.resolve().is_relative_to(output) or not target.is_file():
346
+ raise ValueError("Broken suite navigation: " + value)
347
+ if parsed.fragment and owner.unquote(parsed.fragment) not in links.page(target).ids:
348
+ raise ValueError("Broken suite anchor: " + value)
349
+ if release:
350
+ if manifest.get("local_example"):
351
+ raise ValueError("Synthetic suites cannot be published as measured results")
352
+ owner.check_visual_receipt(output, manifest, visual_receipt, routes=("index",))
353
+ return manifest
@@ -0,0 +1,66 @@
1
+ """Static report controls for field, method, source and diagnosis drilldown."""
2
+
3
+ import json
4
+ from pathlib import Path
5
+
6
+ from harness.web_extract.messages import message
7
+
8
+ ASSETS = Path(__file__).with_name("assets")
9
+ JS = (ASSETS / "report.js").read_text(encoding="utf-8")
10
+
11
+ CLIENT_MESSAGE_KEYS = (
12
+ "records_qualified_reference",
13
+ "showing_results",
14
+ "audited_reference",
15
+ "stages_client_transport",
16
+ "compare_all_methods_a24f79dd",
17
+ "complete_request_response_and_repair_history",
18
+ "current_call_latency",
19
+ "exact_error",
20
+ "failure_repair",
21
+ "field",
22
+ "field_labels_availability",
23
+ "field_labels_OutOfStock",
24
+ "field_labels_price",
25
+ "field_labels_sku",
26
+ "full_result",
27
+ "load_failed_close_and_retry_or_use_the_raw_file",
28
+ "loading_complete_evidence",
29
+ "no_accepted_value",
30
+ "no_recognizable_product_fields_returned",
31
+ "null",
32
+ "original_call_and_score_retained",
33
+ "outcome_243a87d2",
34
+ "outcomes_correct",
35
+ "outcomes_missing",
36
+ "outcomes_unexpected",
37
+ "outcomes_wrong",
38
+ "outcomes_partial",
39
+ "outcomes_invalid",
40
+ "outcomes_unavailable",
41
+ "output",
42
+ "stages_provider_schema",
43
+ "reference",
44
+ "reviewed_cause",
45
+ "see_field_judgments_and_complete_evidence_below",
46
+ "source_and_annotations_cf191383",
47
+ "source_evidence_and_audited_reference",
48
+ "stage_fb689e6a",
49
+ "stages_fields",
50
+ "stages_model_capacity",
51
+ "stages_navigation",
52
+ "stages_provider_fetch",
53
+ "stages_provider_response",
54
+ "structured_fields_returned_inspect_field_differences",
55
+ "treatment",
56
+ )
57
+ JS = JS.replace(
58
+ "__REPORT_MESSAGES__",
59
+ json.dumps(
60
+ {
61
+ language: {key: message(key, language, shown="{shown}") for key in CLIENT_MESSAGE_KEYS}
62
+ for language in ("en", "zh")
63
+ },
64
+ ensure_ascii=False,
65
+ ),
66
+ )