webstone 0.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- harness/benchmark_execution.py +445 -0
- harness/benchmarks/__init__.py +1 -0
- harness/benchmarks/cards/expanded-swde.md +45 -0
- harness/benchmarks/cards/headless-browser.md +112 -0
- harness/benchmarks/cards/languages.json +1018 -0
- harness/benchmarks/cards/liveweb-ie.md +49 -0
- harness/benchmarks/cards/metadata.json +485 -0
- harness/benchmarks/cards/newsxlm.md +47 -0
- harness/benchmarks/cards/simpleqa-verified.md +75 -0
- harness/benchmarks/cards/swde.json +47 -0
- harness/benchmarks/cards/trajectory-replay.md +82 -0
- harness/benchmarks/cards/wcxb.md +116 -0
- harness/benchmarks/cards/web-extract.md +119 -0
- harness/benchmarks/cards/webarena-replay.md +91 -0
- harness/benchmarks/cards/webfetch.md +297 -0
- harness/benchmarks/cards/webmainbench.md +63 -0
- harness/benchmarks/cards/websearch.md +98 -0
- harness/benchmarks/cards/zyte-product-extraction.json +42 -0
- harness/benchmarks/code-migration.json +4554 -0
- harness/benchmarks/data-lock.json +129219 -0
- harness/benchmarks/data.py +397 -0
- harness/benchmarks/pinning.py +328 -0
- harness/benchmarks/publication.py +780 -0
- harness/benchmarks/scoring-regressions.json +90025 -0
- harness/browser_use/__init__.py +5 -0
- harness/browser_use/__main__.py +4 -0
- harness/browser_use/assets/evidence-viewer.js +1 -0
- harness/browser_use/assets/report.css +63 -0
- harness/browser_use/assets/report.js +12 -0
- harness/browser_use/assets/webarena-autonomous-evaluation-flow.png +0 -0
- harness/browser_use/cli.py +46 -0
- harness/browser_use/failure_cause_review.py +160 -0
- harness/browser_use/messages.json +699 -0
- harness/browser_use/model.py +258 -0
- harness/browser_use/publication.py +191 -0
- harness/browser_use/report.py +4261 -0
- harness/browser_use/report_check.py +982 -0
- harness/browser_use/report_i18n.py +175 -0
- harness/browser_use/report_package.py +100 -0
- harness/browser_use/report_publish.py +194 -0
- harness/browser_use/report_visual.py +265 -0
- harness/browser_use/review.py +170 -0
- harness/browser_use/runner.py +811 -0
- harness/browser_use/selection.py +346 -0
- harness/container_isolation.py +46 -0
- harness/data_home.py +15 -0
- harness/distribution.py +46 -0
- harness/distribution_data/configs/providers.headless.json +120 -0
- harness/distribution_data/evaluation/browser-use-public/adoption-evidence.json +122 -0
- harness/distribution_data/evaluation/browser-use-public/benchmark-catalog.json +244 -0
- harness/distribution_data/evaluation/browser-use-public/report-changes.json +55 -0
- harness/distribution_data/evaluation/webmainbench-full/scoring-regressions.json +43 -0
- harness/distribution_data/pyproject.toml +209 -0
- harness/distribution_data/uv.lock +5088 -0
- harness/env_file.py +125 -0
- harness/environments/__init__.py +1 -0
- harness/environments/browser.py +150 -0
- harness/environments/webarena/__init__.py +1 -0
- harness/environments/webarena/backend.py +108 -0
- harness/environments/webarena/cli.py +88 -0
- harness/environments/webarena/config.py +280 -0
- harness/environments/webarena/har.py +835 -0
- harness/environments/webarena/manager.py +204 -0
- harness/environments/webarena/map_archives.py +60 -0
- harness/environments/webarena/map_check.py +103 -0
- harness/environments/webarena/map_detail.py +149 -0
- harness/environments/webarena/map_setup.py +537 -0
- harness/environments/webarena/rainbond.py +1185 -0
- harness/environments/webarena/runtime/php-fpm-qemu.ini +12 -0
- harness/environments/webarena/runtime.py +736 -0
- harness/environments/webarena/topology.json +81 -0
- harness/environments/webarena/topology.py +238 -0
- harness/fix_log.py +305 -0
- harness/lexbench/__init__.py +1 -0
- harness/lexbench/__main__.py +4 -0
- harness/lexbench/assets/bundle-loader.js +75 -0
- harness/lexbench/assets/render_contract.cjs +382 -0
- harness/lexbench/assets/ui_check.js +636 -0
- harness/lexbench/charts.py +469 -0
- harness/lexbench/cli.py +45 -0
- harness/lexbench/pack.py +136 -0
- harness/lexbench/report.py +2132 -0
- harness/lexbench/reproduction.py +234 -0
- harness/lexbench/resource_evidence.py +199 -0
- harness/lexbench/resource_matrix.py +298 -0
- harness/lexbench/resource_profiles.py +315 -0
- harness/lexbench/resource_summary.py +277 -0
- harness/lexbench/template/README.md +11 -0
- harness/lexbench/template/app.js +1607 -0
- harness/lexbench/template/index.html +51 -0
- harness/lexbench/template/messages.json +744 -0
- harness/lexbench/template/style.css +905 -0
- harness/lexbench/verification.py +1239 -0
- harness/licenses/Lexbench-Apache-2.0.txt +201 -0
- harness/licenses/WCXB-CC-BY-4.0.txt +25 -0
- harness/numbers.py +14 -0
- harness/package_gate.py +84 -0
- harness/platform_capabilities.py +37 -0
- harness/record_io.py +107 -0
- harness/report_languages.py +246 -0
- harness/reporting/__init__.py +1 -0
- harness/reporting/assets.py +182 -0
- harness/reporting/certified_cli.py +24 -0
- harness/reporting/links.py +168 -0
- harness/reporting/locales.py +30 -0
- harness/reporting/mermaid.py +28 -0
- harness/reporting/packs.py +336 -0
- harness/reporting/presentation.py +176 -0
- harness/reporting/pricing.py +29 -0
- harness/reporting/privacy.py +95 -0
- harness/reporting/publication.py +561 -0
- harness/reporting/publication_target.py +92 -0
- harness/reporting/publish.py +894 -0
- harness/reporting/release_check.py +51 -0
- harness/reporting/source_policy.py +64 -0
- harness/reporting/templates.py +109 -0
- harness/reporting/versions.py +135 -0
- harness/reporting/visual.py +215 -0
- harness/resource_accounting.py +256 -0
- harness/resource_calibration.py +287 -0
- harness/results/__init__.py +5 -0
- harness/results/__main__.py +3 -0
- harness/results/cli.py +123 -0
- harness/results/client.py +783 -0
- harness/results/versions.py +72 -0
- harness/review_runner.py +122 -0
- harness/trajectory_replay/__init__.py +5 -0
- harness/trajectory_replay/__main__.py +4 -0
- harness/trajectory_replay/assets/compact-data-fetch-adapter.js +25 -0
- harness/trajectory_replay/assets/compact-detail-loader.js +11 -0
- harness/trajectory_replay/assets/compact-evidence-loader.js +11 -0
- harness/trajectory_replay/assets/curated-execution-filter.js +38 -0
- harness/trajectory_replay/assets/curated-report.css +6 -0
- harness/trajectory_replay/assets/evidence-dialog.js +40 -0
- harness/trajectory_replay/assets/public-webarena-benchmark-tabs.js +4 -0
- harness/trajectory_replay/assets/public-webarena-comparison.js +4 -0
- harness/trajectory_replay/assets/public-webarena-execution-filter.js +39 -0
- harness/trajectory_replay/assets/public-webarena-report.css +13 -0
- harness/trajectory_replay/assets/suite-filter.js +7 -0
- harness/trajectory_replay/assets/webarena-article.css +1 -0
- harness/trajectory_replay/assets/webarena-catalog.css +12 -0
- harness/trajectory_replay/assets/webarena-report.css +52 -0
- harness/trajectory_replay/assets/webarena-report.js +280 -0
- harness/trajectory_replay/browser_use_seeds.py +375 -0
- harness/trajectory_replay/catalog_acceptance.py +717 -0
- harness/trajectory_replay/catalog_action_audit.py +190 -0
- harness/trajectory_replay/catalog_binding.py +198 -0
- harness/trajectory_replay/catalog_comparison.py +423 -0
- harness/trajectory_replay/catalog_consolidation.py +330 -0
- harness/trajectory_replay/catalog_contracts.py +833 -0
- harness/trajectory_replay/catalog_release.py +80 -0
- harness/trajectory_replay/catalog_runner.py +832 -0
- harness/trajectory_replay/cli.py +309 -0
- harness/trajectory_replay/core/__init__.py +1 -0
- harness/trajectory_replay/core/diagnostics.py +263 -0
- harness/trajectory_replay/core/docker_runner.py +374 -0
- harness/trajectory_replay/core/executor.py +72 -0
- harness/trajectory_replay/core/metrics.py +365 -0
- harness/trajectory_replay/core/report.py +113 -0
- harness/trajectory_replay/core/suite.py +190 -0
- harness/trajectory_replay/diagnostics.py +685 -0
- harness/trajectory_replay/evaluator_transition.py +434 -0
- harness/trajectory_replay/executor.py +165 -0
- harness/trajectory_replay/expert_seeds.py +188 -0
- harness/trajectory_replay/failure_matrix.py +280 -0
- harness/trajectory_replay/interpretation.py +186 -0
- harness/trajectory_replay/messages.json +1002 -0
- harness/trajectory_replay/messages.py +11 -0
- harness/trajectory_replay/metrics.py +35 -0
- harness/trajectory_replay/participants.py +97 -0
- harness/trajectory_replay/playwright_tool.py +1331 -0
- harness/trajectory_replay/public_webarena/__init__.py +5 -0
- harness/trajectory_replay/public_webarena/cli.py +204 -0
- harness/trajectory_replay/public_webarena/config.py +79 -0
- harness/trajectory_replay/public_webarena/diagnostics.py +1163 -0
- harness/trajectory_replay/public_webarena/evaluators.py +155 -0
- harness/trajectory_replay/public_webarena/executor.py +695 -0
- harness/trajectory_replay/public_webarena/har.py +393 -0
- harness/trajectory_replay/public_webarena/manifest_repair.py +393 -0
- harness/trajectory_replay/public_webarena/metrics.py +47 -0
- harness/trajectory_replay/public_webarena/publication.py +492 -0
- harness/trajectory_replay/public_webarena/qualification.py +2765 -0
- harness/trajectory_replay/public_webarena/release.py +171 -0
- harness/trajectory_replay/public_webarena/report.py +1903 -0
- harness/trajectory_replay/public_webarena/response_builder.py +327 -0
- harness/trajectory_replay/public_webarena/result_audit.py +185 -0
- harness/trajectory_replay/public_webarena/runner.py +908 -0
- harness/trajectory_replay/public_webarena/suite.py +410 -0
- harness/trajectory_replay/public_webarena/targets.py +93 -0
- harness/trajectory_replay/public_webarena/webarena.py +735 -0
- harness/trajectory_replay/public_webarena/webarena_environment.py +735 -0
- harness/trajectory_replay/remote_execution.py +1581 -0
- harness/trajectory_replay/remote_runtime.py +1071 -0
- harness/trajectory_replay/report.py +1236 -0
- harness/trajectory_replay/report_compaction.py +217 -0
- harness/trajectory_replay/report_inputs.py +170 -0
- harness/trajectory_replay/report_languages.py +14 -0
- harness/trajectory_replay/resources/fixtures/app.html +11 -0
- harness/trajectory_replay/resources/fixtures/catalog.html +9 -0
- harness/trajectory_replay/resources/fixtures/core.html +18 -0
- harness/trajectory_replay/resources/fixtures/dynamic.html +11 -0
- harness/trajectory_replay/resources/fixtures/forms.html +11 -0
- harness/trajectory_replay/resources/fixtures/interaction-space.html +142 -0
- harness/trajectory_replay/resources/fixtures/page2.html +3 -0
- harness/trajectory_replay/resources/fixtures/page3.html +2 -0
- harness/trajectory_replay/retrieval_acceptance.py +1080 -0
- harness/trajectory_replay/runner.py +304 -0
- harness/trajectory_replay/runtime_identity.py +131 -0
- harness/trajectory_replay/suite.py +83 -0
- harness/trajectory_replay/webarena_corrections.py +1088 -0
- harness/trajectory_replay/webarena_dataset.py +684 -0
- harness/trajectory_replay/webarena_plan.py +658 -0
- harness/trajectory_replay/webarena_replay.py +1657 -0
- harness/trajectory_replay/webarena_report.py +2290 -0
- harness/trajectory_replay/webarena_report_check.py +944 -0
- harness/trajectory_replay/webarena_report_package.py +155 -0
- harness/trajectory_replay/webarena_report_publish.py +230 -0
- harness/trajectory_replay/webarena_report_visual.py +693 -0
- harness/trajectory_replay/webarena_results.py +644 -0
- harness/trajectory_replay/webarena_review.py +198 -0
- harness/trajectory_replay/webarena_runtime_source.py +119 -0
- harness/web_extract/__init__.py +1 -0
- harness/web_extract/analysis.py +683 -0
- harness/web_extract/assets/report.css +58 -0
- harness/web_extract/assets/report.js +431 -0
- harness/web_extract/certification.py +596 -0
- harness/web_extract/cli.py +436 -0
- harness/web_extract/dataset.py +274 -0
- harness/web_extract/example.py +473 -0
- harness/web_extract/firecrawl_backend.mjs +178 -0
- harness/web_extract/messages.json +1070 -0
- harness/web_extract/messages.py +31 -0
- harness/web_extract/model_comparison.py +613 -0
- harness/web_extract/pipeline.py +2580 -0
- harness/web_extract/publication.py +209 -0
- harness/web_extract/records.py +574 -0
- harness/web_extract/repair.py +657 -0
- harness/web_extract/report.py +6167 -0
- harness/web_extract/report_suite.py +353 -0
- harness/web_extract/report_ui.py +66 -0
- harness/web_extract/review.py +1580 -0
- harness/web_extract/self_hosted.py +978 -0
- harness/web_extract/snapshots.py +1197 -0
- harness/web_extract/swde.py +742 -0
- harness/web_fetch/__init__.py +5 -0
- harness/web_fetch/__main__.py +5 -0
- harness/web_fetch/accounts.py +130 -0
- harness/web_fetch/adjudication_evidence.py +767 -0
- harness/web_fetch/anomaly_review.py +346 -0
- harness/web_fetch/artifact_auth.py +135 -0
- harness/web_fetch/artifacts.py +521 -0
- harness/web_fetch/assembly.py +1324 -0
- harness/web_fetch/assets/cohort-diagnostic.css +18 -0
- harness/web_fetch/assets/cohort-diagnostic.js +2 -0
- harness/web_fetch/assets/methodology/annotation.png +0 -0
- harness/web_fetch/assets/methodology/scoring.png +0 -0
- harness/web_fetch/assets/methodology/validation.png +0 -0
- harness/web_fetch/assets/report-extra.css +64 -0
- harness/web_fetch/assets/report-loader.js +167 -0
- harness/web_fetch/assets/report-style.css +68 -0
- harness/web_fetch/assets/report.js +57 -0
- harness/web_fetch/assets/wcxb.css +1 -0
- harness/web_fetch/assets/wcxb.js +1 -0
- harness/web_fetch/assets/webmainbench-full.css +1 -0
- harness/web_fetch/assets/webmainbench-full.js +52 -0
- harness/web_fetch/benchmark_runner.py +190 -0
- harness/web_fetch/browser_observations.py +980 -0
- harness/web_fetch/call_boundary.py +142 -0
- harness/web_fetch/certification.py +683 -0
- harness/web_fetch/cli.py +394 -0
- harness/web_fetch/cohort_exclusions.py +38 -0
- harness/web_fetch/comparison_audit.py +479 -0
- harness/web_fetch/config.py +231 -0
- harness/web_fetch/container_resources.py +220 -0
- harness/web_fetch/data/browser-observation-contracts.json +25 -0
- harness/web_fetch/data/fixture-dataset.json +42 -0
- harness/web_fetch/data/fixture-providers.json +11 -0
- harness/web_fetch/data/latency-certifiers/34bdb7aedb25a9e5a0c64e0c549926ed7e71f747f544ec2035f65952c0177347.py.txt +142 -0
- harness/web_fetch/data/latency-certifiers/95a2f0a73578346998f0a4ad1256b2cef469a3f2db9b4b30550fc03fc5c259a8.py.txt +142 -0
- harness/web_fetch/data/native-projection-history.json +35 -0
- harness/web_fetch/data/native-projection-regressions.json +531 -0
- harness/web_fetch/data/report-methods.json +6687 -0
- harness/web_fetch/data/report-track-context.json +132 -0
- harness/web_fetch/data/wcxb-metric-bindings.json +12 -0
- harness/web_fetch/data/webmainbench-latency-source.json +18 -0
- harness/web_fetch/dataset_assembly.py +274 -0
- harness/web_fetch/diagnostics.py +73 -0
- harness/web_fetch/document_text.py +55 -0
- harness/web_fetch/dynamic_cohort.py +467 -0
- harness/web_fetch/evidence.py +753 -0
- harness/web_fetch/failure_diagnosis.py +190 -0
- harness/web_fetch/headless_runner.py +510 -0
- harness/web_fetch/html_to_markdown.mjs +21 -0
- harness/web_fetch/local_audit.py +453 -0
- harness/web_fetch/local_execution_contract.py +317 -0
- harness/web_fetch/local_execution_legacy.py +205 -0
- harness/web_fetch/local_revision.py +1192 -0
- harness/web_fetch/markdown.py +32 -0
- harness/web_fetch/messages.json +2377 -0
- harness/web_fetch/methods.py +451 -0
- harness/web_fetch/models.py +70 -0
- harness/web_fetch/native_projection.py +769 -0
- harness/web_fetch/native_routes.py +637 -0
- harness/web_fetch/network_dns.py +363 -0
- harness/web_fetch/network_preflight.py +119 -0
- harness/web_fetch/provider_adoption.py +321 -0
- harness/web_fetch/providers/__init__.py +4 -0
- harness/web_fetch/providers/base.py +274 -0
- harness/web_fetch/providers/cloudflare.py +816 -0
- harness/web_fetch/providers/cowagent.py +78 -0
- harness/web_fetch/providers/exa.py +177 -0
- harness/web_fetch/providers/firecrawl.py +454 -0
- harness/web_fetch/providers/fixture.py +49 -0
- harness/web_fetch/providers/headless.py +1147 -0
- harness/web_fetch/providers/headless_worker.py +512 -0
- harness/web_fetch/providers/hosted_fetch.py +250 -0
- harness/web_fetch/providers/http.py +87 -0
- harness/web_fetch/providers/http_client.py +112 -0
- harness/web_fetch/providers/lexmount.py +211 -0
- harness/web_fetch/providers/moli.py +889 -0
- harness/web_fetch/providers/runtime_identity.py +220 -0
- harness/web_fetch/providers/subprocess_jsonl.py +410 -0
- harness/web_fetch/providers/tavily.py +116 -0
- harness/web_fetch/public_benchmark_sample.py +160 -0
- harness/web_fetch/publication.py +26 -0
- harness/web_fetch/publication_check.py +379 -0
- harness/web_fetch/publication_privacy.py +1136 -0
- harness/web_fetch/publication_verify.py +390 -0
- harness/web_fetch/published_snapshot.py +379 -0
- harness/web_fetch/reference_freeze.py +699 -0
- harness/web_fetch/reference_inputs.py +387 -0
- harness/web_fetch/reference_lifecycle.py +1811 -0
- harness/web_fetch/reference_revision.py +408 -0
- harness/web_fetch/refresh_assembly.py +1723 -0
- harness/web_fetch/reliability.py +80 -0
- harness/web_fetch/render_evidence.py +215 -0
- harness/web_fetch/report-translations.json +3456 -0
- harness/web_fetch/report.py +4564 -0
- harness/web_fetch/report_build.py +1851 -0
- harness/web_fetch/report_capabilities.py +149 -0
- harness/web_fetch/report_changes.py +224 -0
- harness/web_fetch/report_check.py +1004 -0
- harness/web_fetch/report_components.py +512 -0
- harness/web_fetch/report_diagnostic.py +857 -0
- harness/web_fetch/report_explainer.py +1133 -0
- harness/web_fetch/report_i18n.py +942 -0
- harness/web_fetch/report_interfaces.py +262 -0
- harness/web_fetch/report_methods.py +159 -0
- harness/web_fetch/report_package.py +1000 -0
- harness/web_fetch/report_pricing.py +120 -0
- harness/web_fetch/report_publish.py +158 -0
- harness/web_fetch/report_quality.py +34 -0
- harness/web_fetch/report_reading.py +124 -0
- harness/web_fetch/report_resources.py +196 -0
- harness/web_fetch/report_saved_composition.py +149 -0
- harness/web_fetch/report_saved_matrix.py +2662 -0
- harness/web_fetch/report_style.py +9 -0
- harness/web_fetch/report_version.py +270 -0
- harness/web_fetch/rescoring.py +332 -0
- harness/web_fetch/resource_cohort.py +207 -0
- harness/web_fetch/result_ledger.py +119 -0
- harness/web_fetch/result_review.py +1252 -0
- harness/web_fetch/retrieval_hints.py +574 -0
- harness/web_fetch/review_execution.py +1638 -0
- harness/web_fetch/risk_control.py +524 -0
- harness/web_fetch/runner.py +1241 -0
- harness/web_fetch/safety.py +140 -0
- harness/web_fetch/scoring_revision.py +391 -0
- harness/web_fetch/source_capture.py +455 -0
- harness/web_fetch/speed_cohort.py +216 -0
- harness/web_fetch/storage.py +336 -0
- harness/web_fetch/task_outputs.py +346 -0
- harness/web_fetch/templates/interpretation-headless.css +326 -0
- harness/web_fetch/title_quality.py +93 -0
- harness/web_fetch/wcxb_dataset.py +168 -0
- harness/web_fetch/wcxb_latency.py +347 -0
- harness/web_fetch/wcxb_metrics.py +38 -0
- harness/web_fetch/wcxb_publication.py +147 -0
- harness/web_fetch/wcxb_report_check.py +753 -0
- harness/web_fetch/wcxb_scoring.py +667 -0
- harness/web_fetch/webmainbench_assembly.py +684 -0
- harness/web_fetch/webmainbench_case_correction.py +267 -0
- harness/web_fetch/webmainbench_certification.py +306 -0
- harness/web_fetch/webmainbench_correction.py +866 -0
- harness/web_fetch/webmainbench_diagnostics.py +1202 -0
- harness/web_fetch/webmainbench_evidence.py +205 -0
- harness/web_fetch/webmainbench_latency_correction.py +525 -0
- harness/web_fetch/webmainbench_latency_diagnostic.py +562 -0
- harness/web_fetch/webmainbench_report_check.py +900 -0
- harness/web_fetch/webmainbench_runner.py +680 -0
- harness/web_fetch/webmainbench_scoring.py +354 -0
- harness/web_fetch/workers/__init__.py +1 -0
- harness/web_fetch/workers/cowagent_worker.py +70 -0
- harness/web_fetch/workers/crawl4ai_worker.py +463 -0
- harness/web_fetch/workers/dsh_output.mjs +21 -0
- harness/web_fetch/workers/dsh_webfetch_worker.mts +100 -0
- harness/web_fetch/workers/langchain_webbase_worker.py +67 -0
- harness/web_fetch/workers/native_dns_darwin.c +108 -0
- harness/web_fetch/workers/openclaw_webfetch_worker.mts +91 -0
- harness/web_fetch/workers/public_dns.mjs +49 -0
- harness/web_search/__init__.py +1 -0
- harness/web_search/__main__.py +4 -0
- harness/web_search/adapters.py +371 -0
- harness/web_search/answers.py +326 -0
- harness/web_search/assets/comparison.css +5 -0
- harness/web_search/assets/comparison.js +45 -0
- harness/web_search/assets/delivery-bridge.js +35 -0
- harness/web_search/assets/delivery-loader.js +35 -0
- harness/web_search/assets/native-evidence-result.js +8 -0
- harness/web_search/assets/report.css +69 -0
- harness/web_search/assets/report.js +58 -0
- harness/web_search/cli.py +44 -0
- harness/web_search/comparison.py +843 -0
- harness/web_search/core.py +274 -0
- harness/web_search/data/native-normalization-regressions.json +354 -0
- harness/web_search/delivery.py +163 -0
- harness/web_search/diagnostics.py +132 -0
- harness/web_search/example.py +147 -0
- harness/web_search/messages.json +751 -0
- harness/web_search/messages.py +23 -0
- harness/web_search/output_levels.py +431 -0
- harness/web_search/pages.py +28 -0
- harness/web_search/profiles.py +220 -0
- harness/web_search/publication.py +208 -0
- harness/web_search/relevance.py +828 -0
- harness/web_search/report.py +4315 -0
- harness/web_search/review.py +1318 -0
- harness/web_search/revisions.py +1052 -0
- harness/web_search/runner.py +1013 -0
- harness/web_search/visual.py +164 -0
- harness/webarena_state.py +549 -0
- lexbench_webfetch/__init__.py +1 -0
- lexbench_webfetch/annotation_revisions.py +269 -0
- lexbench_webfetch/article_comments.py +331 -0
- lexbench_webfetch/article_dataset.py +285 -0
- lexbench_webfetch/article_posts.py +384 -0
- lexbench_webfetch/article_qualification.py +192 -0
- lexbench_webfetch/article_text.py +259 -0
- lexbench_webfetch/article_title.py +186 -0
- lexbench_webfetch/benchmark.py +591 -0
- lexbench_webfetch/body_coverage.py +582 -0
- lexbench_webfetch/cli.py +143 -0
- lexbench_webfetch/composite_tasks.py +2026 -0
- lexbench_webfetch/content_scopes.py +156 -0
- lexbench_webfetch/document_structure.py +326 -0
- lexbench_webfetch/dynamic_entry_matching.py +190 -0
- lexbench_webfetch/dynamic_lists.py +1658 -0
- lexbench_webfetch/dynamic_reference.py +25 -0
- lexbench_webfetch/evaluation.py +24 -0
- lexbench_webfetch/file_acceptance.py +72 -0
- lexbench_webfetch/inputs.py +129 -0
- lexbench_webfetch/list_verification.py +90 -0
- lexbench_webfetch/metrics.py +1441 -0
- lexbench_webfetch/models.py +93 -0
- lexbench_webfetch/ordered_tree.py +115 -0
- lexbench_webfetch/paths.py +41 -0
- lexbench_webfetch/reference_quality.py +1104 -0
- lexbench_webfetch/required_components.py +44 -0
- lexbench_webfetch/resource_urls.py +35 -0
- lexbench_webfetch/response_input.py +135 -0
- lexbench_webfetch/risk_control_dataset.py +512 -0
- lexbench_webfetch/scoring.py +452 -0
- lexbench_webfetch/scoring_regressions.py +215 -0
- lexbench_webfetch/stable_tasks.py +2144 -0
- lexbench_webfetch/structured_tables.py +793 -0
- lexbench_webfetch/versions.py +224 -0
- lexbench_websearch/__init__.py +5 -0
- lexbench_websearch/answer_rules.py +322 -0
- lexbench_websearch/answers.py +383 -0
- lexbench_websearch/cli.py +123 -0
- lexbench_websearch/contracts.py +58 -0
- lexbench_websearch/evidence_quality.py +109 -0
- lexbench_websearch/inputs.py +80 -0
- lexbench_websearch/outputs.py +161 -0
- lexbench_websearch/qualification.py +144 -0
- lexbench_websearch/relevance.py +346 -0
- lexbench_websearch/rubric.py +34 -0
- lexbench_websearch/scoring.py +354 -0
- lexbench_websearch/simpleqa.py +467 -0
- lexbench_websearch/versions.py +255 -0
- webstone-0.8.0.dist-info/METADATA +367 -0
- webstone-0.8.0.dist-info/RECORD +488 -0
- webstone-0.8.0.dist-info/WHEEL +5 -0
- webstone-0.8.0.dist-info/entry_points.txt +4 -0
- webstone-0.8.0.dist-info/licenses/LICENSE +202 -0
- webstone-0.8.0.dist-info/licenses/NOTICE +6 -0
- webstone-0.8.0.dist-info/licenses/THIRD_PARTY.md +22 -0
- webstone-0.8.0.dist-info/top_level.txt +3 -0
|
@@ -0,0 +1,1004 @@
|
|
|
1
|
+
"""Data and generated-output acceptance checks for the current report."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import hashlib
|
|
5
|
+
import json
|
|
6
|
+
import re
|
|
7
|
+
from collections import Counter
|
|
8
|
+
from html.parser import HTMLParser
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from urllib.parse import parse_qsl, unquote, urlencode, urljoin, urlsplit
|
|
11
|
+
|
|
12
|
+
from bs4 import BeautifulSoup
|
|
13
|
+
from lexbench_webfetch.metrics import task_measured, task_passed
|
|
14
|
+
from lexbench_webfetch.paths import contained_path, safe_identifier
|
|
15
|
+
from lexbench_webfetch.stable_tasks import validate_acceptance_contract
|
|
16
|
+
|
|
17
|
+
from harness.record_io import file_sha256, load_jsonl, read_json
|
|
18
|
+
from harness.report_languages import check_language_contract, generated_languages, visible_numbers
|
|
19
|
+
from harness.reporting.presentation import check_presentation
|
|
20
|
+
from harness.reporting.release_check import language_editions, open_report, read_page
|
|
21
|
+
from harness.reporting.templates import resolve_template
|
|
22
|
+
from harness.web_fetch.adjudication_evidence import (
|
|
23
|
+
require_adjudication_evidence,
|
|
24
|
+
require_current_review_state,
|
|
25
|
+
)
|
|
26
|
+
from harness.web_fetch.local_audit import (
|
|
27
|
+
require_complete_local_inventory,
|
|
28
|
+
require_local_measurement,
|
|
29
|
+
)
|
|
30
|
+
from harness.web_fetch.local_execution_contract import local_measurement_cases
|
|
31
|
+
from harness.web_fetch.methods import native_browser_engine
|
|
32
|
+
from harness.web_fetch.publication_privacy import public_render_views, read_evidence_binding
|
|
33
|
+
from harness.web_fetch.reference_lifecycle import validate_resource_source_evidence
|
|
34
|
+
from harness.web_fetch.reliability import require_reliable
|
|
35
|
+
from harness.web_fetch.report import local_pass_rate_ranking
|
|
36
|
+
from harness.web_fetch.report_build import validate_local_presentation
|
|
37
|
+
from harness.web_fetch.report_explainer import check_current_browser_article
|
|
38
|
+
from harness.web_fetch.report_i18n import report_text, t
|
|
39
|
+
from harness.web_fetch.report_interfaces import SPECS
|
|
40
|
+
from harness.web_fetch.report_methods import PROFILES
|
|
41
|
+
from harness.web_fetch.report_reading import render_archived_reading
|
|
42
|
+
from harness.web_fetch.report_resources import check_resource_presentation
|
|
43
|
+
from harness.web_fetch.report_version import check_release
|
|
44
|
+
from harness.web_fetch.result_ledger import validate_ledger
|
|
45
|
+
from harness.web_fetch.storage import require_canonical_inputs, require_current_report_profile
|
|
46
|
+
from harness.web_fetch.wcxb_report_check import check as check_wcxb
|
|
47
|
+
from harness.web_fetch.webmainbench_report_check import check_sample_report
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def require_public_evidence_page(content, relative, views):
|
|
51
|
+
"""Require inspectable public downloads and honest original/public hashes."""
|
|
52
|
+
content = re.sub(r'<(pre|script|style)\b[^>]*>.*?</\1>', '', content, flags=re.I | re.S)
|
|
53
|
+
soup = BeautifulSoup(content, 'html.parser')
|
|
54
|
+
visible = soup.get_text(' ', strip=True)
|
|
55
|
+
if '<span data-report-en=' in visible:
|
|
56
|
+
raise ValueError('Public evidence link displays escaped interface markup')
|
|
57
|
+
base = 'https://report.invalid/'+relative
|
|
58
|
+
if soup.find('base', href=True):
|
|
59
|
+
base = urljoin(base, soup.find('base')['href'])
|
|
60
|
+
targets = set()
|
|
61
|
+
for anchor in soup.select('a[href]'):
|
|
62
|
+
href = anchor['href']
|
|
63
|
+
parsed = urlsplit(href)
|
|
64
|
+
if parsed.scheme or parsed.netloc or href.startswith('/'):
|
|
65
|
+
continue
|
|
66
|
+
targets.add(unquote(urlsplit(urljoin(base, href)).path).lstrip('/'))
|
|
67
|
+
answer = views['scored_answer']
|
|
68
|
+
if answer['original_sha256'] != answer['published_sha256']:
|
|
69
|
+
if (answer['original_sha256'] not in visible or answer['published_sha256'] not in visible
|
|
70
|
+
or answer['path'] not in targets or answer['receipt'] not in targets):
|
|
71
|
+
raise ValueError('Public answer lacks visible original/public identities or download links')
|
|
72
|
+
native = views.get('native_response')
|
|
73
|
+
if native and (native['path'] not in targets or native['receipt'] not in targets):
|
|
74
|
+
raise ValueError('Complete native public response or its identity receipt is not linked')
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def check_public_evidence_pages(root, records):
|
|
78
|
+
views = public_render_views(root, records)
|
|
79
|
+
editions = language_editions(root)
|
|
80
|
+
for row in records:
|
|
81
|
+
selected = views[row['attempt_id']]
|
|
82
|
+
for _, prefix in editions:
|
|
83
|
+
relative = prefix+'attempts/'+row['attempt_id']+'.html'
|
|
84
|
+
path = root/relative
|
|
85
|
+
if not path.is_file():
|
|
86
|
+
raise ValueError('Public evidence result page is missing')
|
|
87
|
+
require_public_evidence_page(path.read_text(encoding="utf-8"), relative, selected)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class Tables(HTMLParser):
|
|
91
|
+
def __init__(self):
|
|
92
|
+
super().__init__()
|
|
93
|
+
self.in_th = False
|
|
94
|
+
self.first = False
|
|
95
|
+
self.headers = []
|
|
96
|
+
self.unindexed_headers = []
|
|
97
|
+
self.unindexed = False
|
|
98
|
+
|
|
99
|
+
def handle_starttag(self, tag, attrs):
|
|
100
|
+
if tag == "table":
|
|
101
|
+
self.first = True
|
|
102
|
+
self.unindexed = dict(attrs).get('data-row-index') == 'false'
|
|
103
|
+
if tag == "th" and self.first:
|
|
104
|
+
self.in_th = True
|
|
105
|
+
self.first = False
|
|
106
|
+
self.headers.append("")
|
|
107
|
+
self.unindexed_headers.append(self.unindexed)
|
|
108
|
+
|
|
109
|
+
def handle_data(self, text):
|
|
110
|
+
if self.in_th:
|
|
111
|
+
self.headers[-1] += text
|
|
112
|
+
|
|
113
|
+
def handle_endtag(self, tag):
|
|
114
|
+
if tag == "th":
|
|
115
|
+
self.in_th = False
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def require_table_index_headers(parser, *, english=False, diagnostic=False):
|
|
119
|
+
expected = report_text('en' if english else 'zh')('no', 'No.')
|
|
120
|
+
for header, unindexed in zip(parser.headers, parser.unindexed_headers, strict=True):
|
|
121
|
+
if diagnostic and unindexed and header in {t('method', 'Method'), t('method_4', 'Method ↕'), 'Method', 'Method ↕'}:
|
|
122
|
+
continue
|
|
123
|
+
if header != expected:
|
|
124
|
+
raise ValueError('Unexpected table index header: ' + header)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def require_home_comparison_layout(home, track_count, *, diagnostic=False, sample_ranking=False):
|
|
128
|
+
overall = ('任务通过情况' if diagnostic else
|
|
129
|
+
'本轮任务通过率排名' if sample_ranking else '综合通过率总榜')
|
|
130
|
+
details = '按任务类别查看诊断结果' if diagnostic else f'{track_count} 类分榜'
|
|
131
|
+
markers = [f'{track_count} 类任务概览', overall, details]
|
|
132
|
+
headings = [re.sub(r'<[^>]+>', '', value) for value in re.findall(r'<h2[^>]*>(.*?)</h2>', home, re.S)]
|
|
133
|
+
if any(marker not in headings for marker in markers) or not (
|
|
134
|
+
headings.index(markers[1]) < headings.index(markers[0]) < headings.index(markers[2])):
|
|
135
|
+
raise ValueError('Task overview and comparison must precede detailed results')
|
|
136
|
+
if t('judgement_coverage', 'Judgement coverage') not in home or (t('best_in_family', 'Best in family:') in home) == diagnostic:
|
|
137
|
+
raise ValueError('Comparison winner disclosure differs from evidence state')
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
class VisibleText(HTMLParser):
|
|
141
|
+
def __init__(self):
|
|
142
|
+
super().__init__()
|
|
143
|
+
self.skip = 0
|
|
144
|
+
self.parts = []
|
|
145
|
+
|
|
146
|
+
def handle_starttag(self, tag, attrs):
|
|
147
|
+
if tag in {"pre", "script", "style"}:
|
|
148
|
+
self.skip += 1
|
|
149
|
+
|
|
150
|
+
def handle_endtag(self, tag):
|
|
151
|
+
if tag in {"pre", "script", "style"}:
|
|
152
|
+
self.skip = max(0, self.skip - 1)
|
|
153
|
+
|
|
154
|
+
def handle_data(self, text):
|
|
155
|
+
if not self.skip:
|
|
156
|
+
self.parts.append(text)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
class EnglishNarrative(HTMLParser):
|
|
160
|
+
def __init__(self, allowed):
|
|
161
|
+
super().__init__()
|
|
162
|
+
self.skip = 0
|
|
163
|
+
self.allowed = sorted(allowed, key=len, reverse=True)
|
|
164
|
+
self.untranslated = []
|
|
165
|
+
|
|
166
|
+
def handle_starttag(self, tag, attrs):
|
|
167
|
+
if tag in {"pre", "code", "script", "style"}:
|
|
168
|
+
self.skip += 1
|
|
169
|
+
|
|
170
|
+
def handle_endtag(self, tag):
|
|
171
|
+
if tag in {"pre", "code", "script", "style"}:
|
|
172
|
+
self.skip = max(0, self.skip - 1)
|
|
173
|
+
|
|
174
|
+
def handle_data(self, text):
|
|
175
|
+
value = text.strip()
|
|
176
|
+
remainder = value
|
|
177
|
+
for allowed in self.allowed:
|
|
178
|
+
remainder = remainder.replace(allowed, "")
|
|
179
|
+
if not self.skip and re.search("[\\u4e00-\\u9fff]", remainder):
|
|
180
|
+
self.untranslated.append(value)
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
class PageShape(HTMLParser):
|
|
184
|
+
def __init__(self, page=None):
|
|
185
|
+
super().__init__()
|
|
186
|
+
self.tags = Counter()
|
|
187
|
+
self.ids = set()
|
|
188
|
+
self.data = []
|
|
189
|
+
self.links = []
|
|
190
|
+
self.skip = 0
|
|
191
|
+
self.numbers = []
|
|
192
|
+
self.page = page
|
|
193
|
+
self.images = []
|
|
194
|
+
self.english_images = []
|
|
195
|
+
|
|
196
|
+
def handle_starttag(self, tag, attrs):
|
|
197
|
+
values = dict(attrs)
|
|
198
|
+
# This script selects the primary language at the shared entry point;
|
|
199
|
+
# it is not report content and intentionally exists only on that page.
|
|
200
|
+
if tag == "script" and "data-report-language-entry" in values:
|
|
201
|
+
self.skip += 1
|
|
202
|
+
return
|
|
203
|
+
self.tags[tag] += 1
|
|
204
|
+
if values.get("id"):
|
|
205
|
+
self.ids.add(values["id"])
|
|
206
|
+
self.data.extend(
|
|
207
|
+
(
|
|
208
|
+
(key, value)
|
|
209
|
+
for key, value in attrs
|
|
210
|
+
if key.startswith("data-") and key != "data-i18n-attrs"
|
|
211
|
+
and not (key == "data-en-src" and self.page
|
|
212
|
+
and self.page.parts[-2:] == ("methodology", "index.html"))
|
|
213
|
+
)
|
|
214
|
+
)
|
|
215
|
+
if values.get("href") and (not values.get("hreflang")):
|
|
216
|
+
self.links.append(values["href"])
|
|
217
|
+
if tag == "img" and values.get("src"):
|
|
218
|
+
self.images.append(values["src"])
|
|
219
|
+
self.english_images.append(values.get("data-en-src", values["src"]))
|
|
220
|
+
if tag in {"pre", "code", "script", "style"}:
|
|
221
|
+
self.skip += 1
|
|
222
|
+
|
|
223
|
+
def handle_endtag(self, tag):
|
|
224
|
+
if tag in {"pre", "code", "script", "style"}:
|
|
225
|
+
self.skip = max(0, self.skip - 1)
|
|
226
|
+
|
|
227
|
+
def handle_data(self, text):
|
|
228
|
+
if not self.skip:
|
|
229
|
+
self.numbers.extend(visible_numbers(text))
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def check_localized_image_sources(zh, en, canonical):
|
|
233
|
+
"""Require each declared English companion, never silently use the old image."""
|
|
234
|
+
expected = Counter(canonical(zh.page, value) for value in zh.english_images)
|
|
235
|
+
actual = Counter(canonical(en.page, value) for value in en.images)
|
|
236
|
+
if expected != actual:
|
|
237
|
+
raise ValueError('Localized image sources differ from the declared companions')
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def check_saved_result_summary(root, dataset, records):
|
|
241
|
+
cases = {case['id']: case for case in dataset['cases']}
|
|
242
|
+
summary = read_json(root / 'summary.json')
|
|
243
|
+
groups = {}
|
|
244
|
+
for row in records:
|
|
245
|
+
key = (cases[row['case_id']].get('track', 'web'), row['provider'])
|
|
246
|
+
groups.setdefault(key, []).append(row)
|
|
247
|
+
for (track, provider), rows in groups.items():
|
|
248
|
+
saved = summary.get(track, {}).get(provider, {})
|
|
249
|
+
expected = {'attempts': len(rows),
|
|
250
|
+
'task_pass': sum(task_passed(row['ok'], row['metrics']) for row in rows),
|
|
251
|
+
'task_evaluated_attempts': sum(task_measured(row['metrics']) for row in rows),
|
|
252
|
+
'task_excluded_attempts': sum(bool(row['metrics'].get('evaluation_excluded')) for row in rows)}
|
|
253
|
+
if any(saved.get(key) != value for key, value in expected.items()):
|
|
254
|
+
raise ValueError('Saved result summary differs from current evidence: ' + provider + '/' + track)
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def require_replacement_execution(root, binding, physical, logical):
|
|
258
|
+
"""Keep superseded task calls as history without counting them as current rows."""
|
|
259
|
+
receipt = read_json(read_evidence_binding(root, binding))
|
|
260
|
+
previous = load_jsonl(read_evidence_binding(root, receipt['baseline_results']))
|
|
261
|
+
calls = load_jsonl(read_evidence_binding(root, receipt['cohort_results']))
|
|
262
|
+
expected_physical = Counter(row['attempt_id'] for row in previous + calls)
|
|
263
|
+
current = [row for row in previous if row['case_id'] != receipt['removed_case_id']]
|
|
264
|
+
expected_logical = Counter(row['attempt_id'] for row in current + calls)
|
|
265
|
+
if physical != expected_physical or logical != expected_logical:
|
|
266
|
+
raise ValueError('Task replacement physical/logical call inventory differs')
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
class VisibleLocalScope(HTMLParser):
|
|
270
|
+
"""Disclosure attributes count only when attached to visible explanatory text."""
|
|
271
|
+
def __init__(self):
|
|
272
|
+
super().__init__()
|
|
273
|
+
self.stack = []
|
|
274
|
+
self.visible = set()
|
|
275
|
+
self.is_report = False
|
|
276
|
+
|
|
277
|
+
def handle_starttag(self, tag, attrs):
|
|
278
|
+
attrs = dict(attrs)
|
|
279
|
+
if tag == 'meta' and attrs.get('name') == 'webstone-report' and attrs.get('content') == '1':
|
|
280
|
+
self.is_report = True
|
|
281
|
+
inherited = self.stack[-1][1:] if self.stack else (False, set())
|
|
282
|
+
style = re.sub(r'\s+', '', attrs.get('style', '')).lower()
|
|
283
|
+
hidden = (inherited[0] or tag in {'script', 'style', 'template'}
|
|
284
|
+
or 'hidden' in attrs or attrs.get('aria-hidden') == 'true'
|
|
285
|
+
or 'display:none' in style or 'visibility:hidden' in style)
|
|
286
|
+
kinds = set(inherited[1])
|
|
287
|
+
if attrs.get('data-evidence-state') in {'local_measurement', 'saved_diagnostic', 'saved_results'}: kinds.add('evidence')
|
|
288
|
+
if attrs.get('data-ranking-scope') == 'saved-task-pass-rate': kinds.add('ranking')
|
|
289
|
+
if tag not in {'area','base','br','col','embed','hr','img','input','link','meta','param','source','track','wbr'}:
|
|
290
|
+
self.stack.append((tag, hidden, kinds))
|
|
291
|
+
|
|
292
|
+
def handle_endtag(self, tag):
|
|
293
|
+
for index in range(len(self.stack) - 1, -1, -1):
|
|
294
|
+
if self.stack[index][0] == tag:
|
|
295
|
+
del self.stack[index:]
|
|
296
|
+
break
|
|
297
|
+
|
|
298
|
+
def handle_data(self, data):
|
|
299
|
+
if data.strip() and self.stack and not self.stack[-1][1]:
|
|
300
|
+
self.visible.update(self.stack[-1][2])
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def check_local_diagnostic_presentation(root: Path, *, ranked=False):
|
|
304
|
+
"""Track evidence state and scope any explicitly requested sample ranking."""
|
|
305
|
+
from harness.report_languages import english_notice
|
|
306
|
+
for path in root.rglob('*.html'):
|
|
307
|
+
relative = path.relative_to(root)
|
|
308
|
+
parts = relative.parts[1:] if relative.parts[0] == 'en' else relative.parts
|
|
309
|
+
if parts[0] in {'evidence', 'artifacts', 'sources', 'references', 'local-evidence'}:
|
|
310
|
+
continue
|
|
311
|
+
page = path.read_text(encoding='utf-8')
|
|
312
|
+
disclosure = VisibleLocalScope()
|
|
313
|
+
disclosure.feed(page)
|
|
314
|
+
if relative.as_posix() == 'en/index.html' and page == english_notice():
|
|
315
|
+
continue
|
|
316
|
+
if not disclosure.is_report or 'evidence' not in disclosure.visible:
|
|
317
|
+
raise ValueError('Local diagnostic disclosure missing: ' + str(path.relative_to(root)))
|
|
318
|
+
if ranked and 'ranking' not in disclosure.visible:
|
|
319
|
+
raise ValueError('Sample ranking scope missing: ' + str(path.relative_to(root)))
|
|
320
|
+
for table in re.findall(r'<table\b[^>]*\bid="ranking-[^"]+".*?</table>', page, re.S):
|
|
321
|
+
if not ranked and (re.search(r'<th>\s*(?:序号|Rank|No\.)\s*</th>', table) or 'comparison-winner' in table):
|
|
322
|
+
raise ValueError('Unrequested diagnostic ranking: ' + str(path.relative_to(root)))
|
|
323
|
+
if re.search(r'<th>\s*(?:序号|Rank|No\.)\s*</th>', table) and 'ranking' not in disclosure.visible:
|
|
324
|
+
raise ValueError('Sample ranking scope missing: ' + str(path.relative_to(root)))
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def check_report_language_parity(root: Path, d, *, canonical_root: Path | None = None):
|
|
328
|
+
"""Apply the same language checks to every score view."""
|
|
329
|
+
canonical_root = root if canonical_root is None else canonical_root
|
|
330
|
+
has_english = "en" in generated_languages(root)
|
|
331
|
+
zh_pages = {
|
|
332
|
+
p.relative_to(root) for p in root.rglob("*.html") if p.relative_to(root).parts[0] not in {"en"}
|
|
333
|
+
}
|
|
334
|
+
en_pages = {Path(*p.relative_to(root).parts[1:]) for p in (root / "en").rglob("*.html")}
|
|
335
|
+
if has_english:
|
|
336
|
+
if not zh_pages == en_pages:
|
|
337
|
+
raise ValueError(("Chinese/English route inventories differ", zh_pages ^ en_pages))
|
|
338
|
+
|
|
339
|
+
def canonical_link(page, value):
|
|
340
|
+
parsed = urlsplit(value)
|
|
341
|
+
if parsed.scheme or parsed.netloc:
|
|
342
|
+
return value
|
|
343
|
+
target = ((page.parent / parsed.path) if parsed.path else page).resolve().relative_to(canonical_root)
|
|
344
|
+
if (
|
|
345
|
+
target.parts
|
|
346
|
+
and target.parts[0] == "en"
|
|
347
|
+
and (target.suffix.lower() in {".html", ".htm"})
|
|
348
|
+
):
|
|
349
|
+
target = Path(*target.parts[1:])
|
|
350
|
+
query = parsed.query
|
|
351
|
+
if target == Path('index.html'):
|
|
352
|
+
query = urlencode([(key, item) for key, item in parse_qsl(query, keep_blank_values=True)
|
|
353
|
+
if (key, item) != ("report-language", "zh")])
|
|
354
|
+
return (
|
|
355
|
+
target.as_posix()
|
|
356
|
+
+ ("?" + query if query else "")
|
|
357
|
+
+ ("#" + parsed.fragment if parsed.fragment else "")
|
|
358
|
+
)
|
|
359
|
+
|
|
360
|
+
bilingual_issues = []
|
|
361
|
+
if has_english:
|
|
362
|
+
for relative in sorted(zh_pages):
|
|
363
|
+
zh_path = root / relative
|
|
364
|
+
en_path = root / "en" / relative
|
|
365
|
+
zh = PageShape(zh_path)
|
|
366
|
+
zh.feed(zh_path.read_text(encoding="utf-8"))
|
|
367
|
+
en = PageShape(en_path)
|
|
368
|
+
en.feed(en_path.read_text(encoding="utf-8"))
|
|
369
|
+
if zh.tags != en.tags:
|
|
370
|
+
bilingual_issues.append((relative, "semantic tag counts differ"))
|
|
371
|
+
if zh.ids != en.ids:
|
|
372
|
+
bilingual_issues.append((relative, "element IDs differ"))
|
|
373
|
+
if Counter(zh.data) != Counter(en.data):
|
|
374
|
+
bilingual_issues.append((relative, "data attributes differ"))
|
|
375
|
+
zh_numbers = Counter(zh.numbers)
|
|
376
|
+
en_numbers = Counter(en.numbers)
|
|
377
|
+
if zh_numbers != en_numbers:
|
|
378
|
+
bilingual_issues.append(
|
|
379
|
+
(
|
|
380
|
+
relative,
|
|
381
|
+
"visible numeric values differ",
|
|
382
|
+
zh_numbers - en_numbers,
|
|
383
|
+
en_numbers - zh_numbers,
|
|
384
|
+
)
|
|
385
|
+
)
|
|
386
|
+
zh_links = Counter(canonical_link(zh_path, value) for value in zh.links)
|
|
387
|
+
en_links = Counter(canonical_link(en_path, value) for value in en.links)
|
|
388
|
+
if zh_links != en_links:
|
|
389
|
+
bilingual_issues.append((relative, "evidence or navigation links differ"))
|
|
390
|
+
if relative == Path("methodology/index.html"):
|
|
391
|
+
check_localized_image_sources(zh, en, canonical_link)
|
|
392
|
+
if relative in {Path("guide/index.html"), Path("guide/article.html")}:
|
|
393
|
+
if Counter(canonical_link(zh_path, value) for value in zh.images) != Counter(
|
|
394
|
+
canonical_link(en_path, value) for value in en.images
|
|
395
|
+
):
|
|
396
|
+
bilingual_issues.append((relative, "article image sources differ"))
|
|
397
|
+
if not not bilingual_issues:
|
|
398
|
+
raise ValueError(
|
|
399
|
+
("Chinese/English page parity failed", bilingual_issues[:200], len(bilingual_issues))
|
|
400
|
+
)
|
|
401
|
+
english_primary = [
|
|
402
|
+
Path(name)
|
|
403
|
+
for name in (
|
|
404
|
+
"index.html",
|
|
405
|
+
"methodology/index.html",
|
|
406
|
+
"review/index.html",
|
|
407
|
+
"guide/index.html",
|
|
408
|
+
"adoption/index.html",
|
|
409
|
+
"reproduce/index.html",
|
|
410
|
+
"changes/index.html",
|
|
411
|
+
"accounts/index.html",
|
|
412
|
+
)
|
|
413
|
+
]
|
|
414
|
+
allowed_english_source_text = {"中文"} | {
|
|
415
|
+
case["name"] for case in d["cases"] if case.get("name")
|
|
416
|
+
}
|
|
417
|
+
archived_manifest = root / "archived-reading.json"
|
|
418
|
+
if archived_manifest.is_file():
|
|
419
|
+
# Validate the archived source before allowing its original case names.
|
|
420
|
+
render_archived_reading(root, "en" if has_english else "zh")
|
|
421
|
+
allowed_english_source_text.update(read_json(archived_manifest).get("preserved_source_names", []))
|
|
422
|
+
if has_english:
|
|
423
|
+
for relative in english_primary:
|
|
424
|
+
if relative not in en_pages:
|
|
425
|
+
continue
|
|
426
|
+
parser = EnglishNarrative(allowed_english_source_text)
|
|
427
|
+
parser.feed((root / "en" / relative).read_text(encoding="utf-8"))
|
|
428
|
+
if not not parser.untranslated:
|
|
429
|
+
raise ValueError(
|
|
430
|
+
(relative, "untranslated English report narrative", parser.untranslated[:20])
|
|
431
|
+
)
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
|
|
435
|
+
def check(root: Path, *, authority_root: Path, profile: Path | None = None, presentation_binding: Path | None = None, source_dataset: Path | None = None):
|
|
436
|
+
root = open_report(root)
|
|
437
|
+
repository = authority_root.resolve()
|
|
438
|
+
profile_path = None
|
|
439
|
+
if profile:
|
|
440
|
+
profile_path = (
|
|
441
|
+
(repository / profile).resolve() if not profile.is_absolute() else profile.resolve()
|
|
442
|
+
)
|
|
443
|
+
if (root / 'dataset.json').is_file() and (root / 'results.jsonl').is_file():
|
|
444
|
+
check_release(root, read_json(root / 'dataset.json'),
|
|
445
|
+
load_jsonl(root / 'results.jsonl'), profile=profile_path)
|
|
446
|
+
if (root / 'report-manifest.json').is_file() and read_json(root / 'report-manifest.json').get('evidence_state') == 'published_snapshot_score_audit':
|
|
447
|
+
from harness.web_fetch.published_snapshot import check_snapshot_audit
|
|
448
|
+
if profile_path and file_sha256(profile_path) != read_json(root / 'report-manifest.json').get('profile_sha256'):
|
|
449
|
+
raise ValueError('Snapshot audit profile changed')
|
|
450
|
+
return check_snapshot_audit(root)
|
|
451
|
+
if (root / "wcxb-score-summary.json").is_file():
|
|
452
|
+
|
|
453
|
+
check_wcxb(root, authority_root=repository, profile_path=profile_path)
|
|
454
|
+
return None
|
|
455
|
+
if profile_path:
|
|
456
|
+
require_current_report_profile(root, profile_path)
|
|
457
|
+
canonical = (
|
|
458
|
+
profile_path.parent
|
|
459
|
+
if (profile_path.parent / "dataset.json").is_file()
|
|
460
|
+
else repository / "evaluation"
|
|
461
|
+
)
|
|
462
|
+
if not read_json(profile_path).get('frozen_inputs'):
|
|
463
|
+
require_canonical_inputs(root, canonical)
|
|
464
|
+
d = read_json(root / "dataset.json")
|
|
465
|
+
manifest_path = root / 'report-manifest.json'
|
|
466
|
+
if manifest_path.is_file():
|
|
467
|
+
manifest = read_json(manifest_path)
|
|
468
|
+
expected_template = resolve_template('webfetch', root=repository)
|
|
469
|
+
if manifest.get('template') != expected_template:
|
|
470
|
+
raise ValueError('Report manifest differs from the maintained template')
|
|
471
|
+
for name in ('dataset', 'results'):
|
|
472
|
+
path = root / ('dataset.json' if name == 'dataset' else 'results.jsonl')
|
|
473
|
+
if manifest.get(name + '_sha256') != hashlib.sha256(path.read_bytes()).hexdigest():
|
|
474
|
+
raise ValueError('Report manifest evidence changed: ' + name)
|
|
475
|
+
if manifest.get('generated_languages') != list(generated_languages(root)):
|
|
476
|
+
raise ValueError('Report manifest languages differ from generated routes')
|
|
477
|
+
rs = load_jsonl(root / "results.jsonl")
|
|
478
|
+
if source_dataset:
|
|
479
|
+
if not profile_path:
|
|
480
|
+
raise ValueError('Frozen source dataset requires an explicit report profile')
|
|
481
|
+
profile_data = read_json(profile_path)
|
|
482
|
+
if source_dataset.is_symlink():
|
|
483
|
+
raise ValueError('Frozen source dataset rejects symlinks')
|
|
484
|
+
source_raw = source_dataset.read_bytes()
|
|
485
|
+
if hashlib.sha256(source_raw).hexdigest() != profile_data.get('frozen_inputs', {}).get('dataset_sha256'):
|
|
486
|
+
raise ValueError('Frozen source dataset differs from current profile')
|
|
487
|
+
if json.loads(source_raw)['cases'] != d['cases']:
|
|
488
|
+
raise ValueError('Report cases differ from frozen source dataset')
|
|
489
|
+
check_language_contract(root)
|
|
490
|
+
has_english = "en" in generated_languages(root)
|
|
491
|
+
if d.get("comparison", {}).get("kind") == "webmainbench_frozen_html_to_markdown":
|
|
492
|
+
|
|
493
|
+
check_sample_report(root, d, rs, authority_root=repository, profile_path=profile_path)
|
|
494
|
+
return (
|
|
495
|
+
"PASS: WebMainBench preselected-200 matrix, sample, scores, retries, "
|
|
496
|
+
"declared-language routes and reference assets verified"
|
|
497
|
+
)
|
|
498
|
+
public_payload_views = False
|
|
499
|
+
current_snapshot_authority = False
|
|
500
|
+
names = {r["provider"] for r in rs}
|
|
501
|
+
local_measurement = d.get("comparison", {}).get("kind") == "uniform_native_local_measurement"
|
|
502
|
+
if presentation_binding and not local_measurement:
|
|
503
|
+
raise ValueError("Presentation binding requires a frozen uniform-native diagnostic snapshot")
|
|
504
|
+
sample_ranking = local_measurement and local_pass_rate_ranking(d)
|
|
505
|
+
if local_measurement:
|
|
506
|
+
check_local_diagnostic_presentation(root, ranked=sample_ranking)
|
|
507
|
+
if (root / 'result-review.json').is_file():
|
|
508
|
+
review=read_json(root / 'result-review.json')
|
|
509
|
+
require_current_review_state(rs,review)
|
|
510
|
+
require_adjudication_evidence(root,d,rs,review)
|
|
511
|
+
if (root / 'local-evidence/result-ledger.json').is_file():
|
|
512
|
+
ledger = validate_ledger(read_json(root / 'local-evidence/result-ledger.json'), rs, source=root)
|
|
513
|
+
if ledger['states'][-1]['source_results_sha256'] != hashlib.sha256((root / 'results.jsonl').read_bytes()).hexdigest():
|
|
514
|
+
raise ValueError('Result ledger source hash differs from current results')
|
|
515
|
+
if presentation_binding:
|
|
516
|
+
if profile:
|
|
517
|
+
raise ValueError("Presentation binding cannot certify a release profile")
|
|
518
|
+
checked_binding = validate_local_presentation(root, presentation_binding)[0]
|
|
519
|
+
public_payload_views = bool(checked_binding.get("publication_evidence_views"))
|
|
520
|
+
if checked_binding.get('offline_revision'):
|
|
521
|
+
current_receipt = read_json(read_evidence_binding(root, checked_binding['offline_revision']))
|
|
522
|
+
current_snapshot_authority = (current_receipt.get('kind') == 'current_local_snapshot_authority'
|
|
523
|
+
and current_receipt.get('schema_version') == 1)
|
|
524
|
+
else:
|
|
525
|
+
require_local_measurement(root, d, rs)
|
|
526
|
+
inventory = {track: {"case_ids": [case["id"] for case in d["cases"] if case.get("track", "web") == track], "providers": providers} for track, providers in d["coverage_plan"].items()}
|
|
527
|
+
local_contract = read_json(root / "local-run-contract.json")
|
|
528
|
+
canonical_cases = ({case["id"]: case for case in d["cases"]}
|
|
529
|
+
if presentation_binding and (current_snapshot_authority
|
|
530
|
+
or d.get('comparison', {}).get('task_replacement'))
|
|
531
|
+
else {case["id"]: case for case in local_measurement_cases(local_contract, root)})
|
|
532
|
+
canonical_dataset = (root / "dataset.json" if local_contract.get("dataset_sources")
|
|
533
|
+
else repository / local_contract["dataset_path"])
|
|
534
|
+
elif source_dataset or (profile_path and (read_json(profile_path).get('frozen_inputs') or (profile_path.parent / 'dataset.json').is_file())):
|
|
535
|
+
inventory = {
|
|
536
|
+
track: {
|
|
537
|
+
"case_ids": [
|
|
538
|
+
case["id"] for case in d["cases"] if case.get("track", "web") == track
|
|
539
|
+
],
|
|
540
|
+
"providers": providers,
|
|
541
|
+
}
|
|
542
|
+
for track, providers in d.get("coverage_plan", {}).items()
|
|
543
|
+
}
|
|
544
|
+
canonical_dataset = source_dataset or (root / 'dataset.json' if read_json(profile_path).get('frozen_inputs') else profile_path.parent / "dataset.json")
|
|
545
|
+
else:
|
|
546
|
+
inventory = read_json(repository / "configs/evaluation-inventory.json")
|
|
547
|
+
canonical_dataset = repository / "evaluation/dataset.json"
|
|
548
|
+
if d.get("comparison", {}).get("kind") in {"headless_html", "headless_markdown"}:
|
|
549
|
+
headless = read_json(repository / "configs/providers.headless.json")
|
|
550
|
+
if profile_path is None:
|
|
551
|
+
raise ValueError("Headless report checks require --profile")
|
|
552
|
+
profile = read_json(profile_path)
|
|
553
|
+
if profile.get("comparison_kind") not in {"headless_html", "headless_markdown"}:
|
|
554
|
+
raise ValueError(
|
|
555
|
+
"Legacy headless report cannot be checked against a different "
|
|
556
|
+
"current report profile"
|
|
557
|
+
)
|
|
558
|
+
current_providers = profile["providers"]
|
|
559
|
+
if not set(current_providers) <= set(headless["providers"]):
|
|
560
|
+
raise ValueError("Current report references an unknown provider configuration")
|
|
561
|
+
inventory = {
|
|
562
|
+
track: {**group, "providers": current_providers} for track, group in inventory.items()
|
|
563
|
+
}
|
|
564
|
+
for group in inventory.values():
|
|
565
|
+
providers = group["providers"]
|
|
566
|
+
if not providers or len(providers) != len(set(providers)):
|
|
567
|
+
raise ValueError("Inventory providers must be nonempty and unique")
|
|
568
|
+
for provider in providers:
|
|
569
|
+
safe_identifier(provider)
|
|
570
|
+
if not len({c["id"] for c in d["cases"]}) == len(d["cases"]):
|
|
571
|
+
raise ValueError("Duplicate dataset case IDs")
|
|
572
|
+
inventory_ids = [cid for group in inventory.values() for cid in group["case_ids"]]
|
|
573
|
+
if not len(set(inventory_ids)) == len(inventory_ids):
|
|
574
|
+
raise ValueError("Duplicate inventory case IDs")
|
|
575
|
+
if not {(c["id"], c.get("track", "web")) for c in d["cases"]} == {
|
|
576
|
+
(cid, t) for t, v in inventory.items() for cid in v["case_ids"]
|
|
577
|
+
}:
|
|
578
|
+
raise ValueError("Dataset differs from reviewed independent inventory")
|
|
579
|
+
plan = d.get("coverage_plan", {})
|
|
580
|
+
if any(not providers or len(providers) != len(set(providers)) for providers in plan.values()):
|
|
581
|
+
raise ValueError("Coverage providers must be nonempty and unique")
|
|
582
|
+
if not {t: set(p) for t, p in plan.items()} == {
|
|
583
|
+
t: set(v["providers"]) for t, v in inventory.items()
|
|
584
|
+
}:
|
|
585
|
+
raise ValueError("Provider coverage differs from reviewed inventory")
|
|
586
|
+
expected_pairs = {
|
|
587
|
+
(p, c["id"]) for c in d["cases"] for p in plan.get(c.get("track", "web"), names)
|
|
588
|
+
}
|
|
589
|
+
expected = len(expected_pairs)
|
|
590
|
+
if not len(rs) == expected:
|
|
591
|
+
raise ValueError("Report acceptance check failed")
|
|
592
|
+
if not {(r["provider"], r["case_id"]) for r in rs} == expected_pairs:
|
|
593
|
+
raise ValueError("Report acceptance check failed")
|
|
594
|
+
if not len({r["attempt_id"] for r in rs}) == expected:
|
|
595
|
+
raise ValueError("Report acceptance check failed")
|
|
596
|
+
if not len({(r["provider"], r["case_id"]) for r in rs}) == expected:
|
|
597
|
+
raise ValueError("Report acceptance check failed")
|
|
598
|
+
if public_payload_views:
|
|
599
|
+
check_public_evidence_pages(root, rs)
|
|
600
|
+
case_by_id = {c["id"]: c for c in d["cases"]}
|
|
601
|
+
for case in d["cases"]:
|
|
602
|
+
safe_identifier(case["id"])
|
|
603
|
+
validate_resource_source_evidence(case, canonical_dataset)
|
|
604
|
+
if local_measurement:
|
|
605
|
+
reliability = require_complete_local_inventory(d, rs)
|
|
606
|
+
else:
|
|
607
|
+
reliability = require_reliable(d, rs)
|
|
608
|
+
saved_reliability = read_json(root / "reliability.json")
|
|
609
|
+
if not saved_reliability == reliability:
|
|
610
|
+
raise ValueError("Saved reliability status differs from report evidence")
|
|
611
|
+
if local_measurement and d.get('comparison', {}).get('result_ledger'):
|
|
612
|
+
check_saved_result_summary(root, d, rs)
|
|
613
|
+
for r in rs:
|
|
614
|
+
safe_identifier(r["attempt_id"])
|
|
615
|
+
safe_identifier(r["provider"])
|
|
616
|
+
safe_identifier(r["case_id"])
|
|
617
|
+
if public_payload_views:
|
|
618
|
+
read_evidence_binding(root, {'path': r['artifact_path'],
|
|
619
|
+
'sha256': r['metrics']['content_sha256']})
|
|
620
|
+
else:
|
|
621
|
+
p = contained_path(root, r['artifact_path'], reject_symlinks=True)
|
|
622
|
+
if hashlib.sha256(p.read_bytes()).hexdigest() != r['metrics']['content_sha256']:
|
|
623
|
+
raise ValueError(p)
|
|
624
|
+
if not r["ok"]:
|
|
625
|
+
if not not r["metrics"]["content_pass"]:
|
|
626
|
+
raise ValueError("Report acceptance check failed")
|
|
627
|
+
if r["metrics"].get("word_f1") is not None:
|
|
628
|
+
if not case_by_id[r["case_id"]].get("ground_truth_main_content"):
|
|
629
|
+
raise ValueError("Report acceptance check failed")
|
|
630
|
+
for c in d["cases"]:
|
|
631
|
+
if c.get("track", "web") in {"web", "dynamic", "document"} and (
|
|
632
|
+
not c.get("ground_truth_main_content")
|
|
633
|
+
):
|
|
634
|
+
stable_checks = bool(
|
|
635
|
+
c.get("required_facts")
|
|
636
|
+
or c.get("required_patterns")
|
|
637
|
+
or c.get("acceptance_contract")
|
|
638
|
+
)
|
|
639
|
+
if local_measurement:
|
|
640
|
+
if c.get("acceptance_contract"):
|
|
641
|
+
validate_acceptance_contract(c["acceptance_contract"])
|
|
642
|
+
elif not (stable_checks or c.get("task_kind") == "file"):
|
|
643
|
+
raise ValueError(c["id"])
|
|
644
|
+
elif not stable_checks:
|
|
645
|
+
raise ValueError(c["id"])
|
|
646
|
+
if c.get("ground_truth_main_content"):
|
|
647
|
+
expected = c["ground_truth_main_content"].encode()
|
|
648
|
+
if local_measurement:
|
|
649
|
+
valid_reference = expected == canonical_cases[c["id"]]["ground_truth_main_content"].encode()
|
|
650
|
+
elif source_dataset:
|
|
651
|
+
digest = hashlib.sha256(expected).hexdigest()
|
|
652
|
+
valid_reference = all(r['metrics'].get('scored_reference_sha256') == digest for r in rs if r['case_id'] == c['id'])
|
|
653
|
+
else:
|
|
654
|
+
valid_reference = hashlib.sha256(expected).hexdigest() == c["source_verification"]["reference_sha256"]
|
|
655
|
+
if not valid_reference:
|
|
656
|
+
raise ValueError(c["id"])
|
|
657
|
+
if not (root / "references" / f"{c['id']}.txt").read_bytes() == expected:
|
|
658
|
+
raise ValueError(c["id"])
|
|
659
|
+
if not {r["provider"] for r in rs if r["case_id"] == c["id"]} == set(
|
|
660
|
+
plan.get(c.get("track", "web"), names)
|
|
661
|
+
):
|
|
662
|
+
raise ValueError(c["id"])
|
|
663
|
+
check_report_language_parity(root, d)
|
|
664
|
+
pages = ((page, root, rs, case_by_id, local_measurement) for page in root.rglob('*.html'))
|
|
665
|
+
for p, page_root, page_records, page_cases, page_diagnostic in pages:
|
|
666
|
+
text = p.read_text(encoding="utf-8")
|
|
667
|
+
check_presentation(text)
|
|
668
|
+
reader = VisibleText()
|
|
669
|
+
reader.feed(text)
|
|
670
|
+
visible = " ".join(reader.parts)
|
|
671
|
+
for banned in [
|
|
672
|
+
"人工复核",
|
|
673
|
+
"人工检查",
|
|
674
|
+
"短参考答案词级",
|
|
675
|
+
"组 · 查看",
|
|
676
|
+
"已确认有效",
|
|
677
|
+
"标准答案漂移候选队列",
|
|
678
|
+
"需辅助",
|
|
679
|
+
"需用户辅助",
|
|
680
|
+
"请自行登录",
|
|
681
|
+
"补采记录",
|
|
682
|
+
"本轮解析修正",
|
|
683
|
+
"内容已合并",
|
|
684
|
+
"build_v8_report",
|
|
685
|
+
"report_v8.py",
|
|
686
|
+
"任务修订",
|
|
687
|
+
"评分 8.2",
|
|
688
|
+
"已执行/总案例",
|
|
689
|
+
"已判分/已调用",
|
|
690
|
+
]:
|
|
691
|
+
if not banned not in visible:
|
|
692
|
+
raise ValueError((p, banned))
|
|
693
|
+
if not not re.search("\\bHIL\\b", visible):
|
|
694
|
+
raise ValueError((p, "HIL"))
|
|
695
|
+
parser = Tables()
|
|
696
|
+
parser.feed(text)
|
|
697
|
+
relative_parts = p.relative_to(page_root).parts
|
|
698
|
+
if "browser-use" not in relative_parts:
|
|
699
|
+
require_table_index_headers(parser, english=relative_parts[0] == 'en',
|
|
700
|
+
diagnostic=page_diagnostic)
|
|
701
|
+
if p.parent.name == "attempts":
|
|
702
|
+
record = next(r for r in page_records if r["attempt_id"] == p.stem)
|
|
703
|
+
case = page_cases[record["case_id"]]
|
|
704
|
+
heading = (
|
|
705
|
+
t('target_file', 'Target file')
|
|
706
|
+
if case.get("task_kind") == "file"
|
|
707
|
+
else t('reference_and_acceptance_contract', 'Reference and acceptance contract')
|
|
708
|
+
if case.get("task_kind") == "structure"
|
|
709
|
+
else t('acceptance_rules_no_full_text_reference', 'Acceptance rules (no full-text reference)')
|
|
710
|
+
if case.get("task_kind", "body") == "body" and not (case.get("ground_truth_main_content") or case.get("reference_answer"))
|
|
711
|
+
else t('complete_reference_answer', 'Complete reference answer')
|
|
712
|
+
)
|
|
713
|
+
actual_heading = (
|
|
714
|
+
{
|
|
715
|
+
t('target_file', 'Target file'): "Target file",
|
|
716
|
+
t('reference_and_acceptance_contract', 'Reference and acceptance contract'): "Reference and acceptance contract",
|
|
717
|
+
t('complete_reference_answer', 'Complete reference answer'): "Complete reference answer",
|
|
718
|
+
t('acceptance_rules_no_full_text_reference', 'Acceptance rules (no full-text reference)'): "Acceptance rules (no full-text reference)",
|
|
719
|
+
}[heading]
|
|
720
|
+
if relative_parts[0] == "en"
|
|
721
|
+
else heading
|
|
722
|
+
)
|
|
723
|
+
if not text.count("<h3>" + actual_heading + "</h3>") == 1:
|
|
724
|
+
raise ValueError(p)
|
|
725
|
+
if (
|
|
726
|
+
not text.count(
|
|
727
|
+
"<h3>"
|
|
728
|
+
+ ("Actual output" if relative_parts[0] == "en" else t('actual_output', 'Actual output'))
|
|
729
|
+
+ "</h3>"
|
|
730
|
+
)
|
|
731
|
+
== 1
|
|
732
|
+
):
|
|
733
|
+
raise ValueError(p)
|
|
734
|
+
for delivered in (
|
|
735
|
+
record.get("provider_metadata", {}).get("file_evaluation", {}).get("files", [])
|
|
736
|
+
):
|
|
737
|
+
if not ("../" + delivered["artifact_path"] in text and delivered["sha256"] in text):
|
|
738
|
+
raise ValueError(p)
|
|
739
|
+
execution = read_json(root / "execution.json")["batches"]
|
|
740
|
+
physical_attempts = Counter(i for b in execution for i in b["attempt_ids"])
|
|
741
|
+
logical_attempts = Counter(r["attempt_id"] for r in rs)
|
|
742
|
+
if current_snapshot_authority:
|
|
743
|
+
if physical_attempts != logical_attempts:
|
|
744
|
+
raise ValueError('Current snapshot physical/logical call inventory differs')
|
|
745
|
+
elif d.get('comparison', {}).get('task_replacement'):
|
|
746
|
+
require_replacement_execution(root, d['comparison']['task_replacement'],
|
|
747
|
+
physical_attempts, logical_attempts)
|
|
748
|
+
elif d.get("comparison", {}).get("superseded_formal_provider_calls"):
|
|
749
|
+
if not set(physical_attempts) == set(logical_attempts):
|
|
750
|
+
raise ValueError("Report acceptance check failed")
|
|
751
|
+
if (
|
|
752
|
+
not sum(physical_attempts.values()) - sum(logical_attempts.values())
|
|
753
|
+
== d["comparison"]["superseded_formal_provider_calls"]
|
|
754
|
+
):
|
|
755
|
+
raise ValueError("Report acceptance check failed")
|
|
756
|
+
elif not physical_attempts == logical_attempts:
|
|
757
|
+
raise ValueError("Report acceptance check failed")
|
|
758
|
+
home = (root / "index.html").read_text(encoding="utf-8")
|
|
759
|
+
track_count = len({case.get("track", "web") for case in d["cases"]})
|
|
760
|
+
require_home_comparison_layout(home, track_count, diagnostic=local_measurement and not sample_ranking, sample_ranking=sample_ranking)
|
|
761
|
+
if (root / "resource-cohort/resource-comparison.json").is_file():
|
|
762
|
+
resource_summary = read_json(root / "resource-cohort/resource-comparison.json")
|
|
763
|
+
guide = (root / "guide/index.html").read_text(encoding="utf-8")
|
|
764
|
+
for page_name, text in [("home", home), ("guide", guide)]:
|
|
765
|
+
check_resource_presentation(text, resource_summary)
|
|
766
|
+
if "data-resource-winner=" not in text:
|
|
767
|
+
raise ValueError((page_name, "missing resource winner summary"))
|
|
768
|
+
if not all(label in text for label in (t('cpu_and_compute', 'CPU and compute'), t('memory', 'Memory'), t('network', 'Network'))):
|
|
769
|
+
raise ValueError((page_name, "missing grouped resource tables"))
|
|
770
|
+
if has_english:
|
|
771
|
+
for page_name, text in [
|
|
772
|
+
("English home", (root / "en/index.html").read_text(encoding="utf-8")),
|
|
773
|
+
("English guide", (root / "en/guide/index.html").read_text(encoding="utf-8")),
|
|
774
|
+
]:
|
|
775
|
+
if "data-resource-winner=" not in text:
|
|
776
|
+
raise ValueError((page_name, "missing resource winner summary"))
|
|
777
|
+
if not all(label in text for label in ("CPU and compute", "Memory", "Network")):
|
|
778
|
+
raise ValueError((page_name, "missing grouped resource tables"))
|
|
779
|
+
check_resource_presentation(text, resource_summary)
|
|
780
|
+
methodology = (root / "methodology/index.html").read_text(encoding="utf-8")
|
|
781
|
+
for phrase in [
|
|
782
|
+
t('included_in_latency', 'Included in latency'),
|
|
783
|
+
t('excluded_from_latency', 'Excluded from latency'),
|
|
784
|
+
"供应商上报的服务端耗时只作附加证据",
|
|
785
|
+
"形成可送入统一评分器的最终答案",
|
|
786
|
+
]:
|
|
787
|
+
if phrase not in methodology:
|
|
788
|
+
raise ValueError(phrase)
|
|
789
|
+
return (
|
|
790
|
+
f"PASS: {len(d['cases'])} tasks / {len(rs)} records / {len(names)} providers; "
|
|
791
|
+
f"{len(list(root.rglob('*.html')))} logical report routes; "
|
|
792
|
+
"reference integrity, all numbered tables, no workflow leftovers."
|
|
793
|
+
)
|
|
794
|
+
|
|
795
|
+
|
|
796
|
+
def main(argv=None, *, authority_root: Path | None = None):
|
|
797
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
798
|
+
parser.add_argument("root", type=Path)
|
|
799
|
+
parser.add_argument("--profile", type=Path)
|
|
800
|
+
parser.add_argument('--source-dataset', type=Path)
|
|
801
|
+
parser.add_argument("--authority-root", type=Path, default=authority_root or Path.cwd())
|
|
802
|
+
parser.add_argument("--presentation-binding", type=Path)
|
|
803
|
+
parser.add_argument("--visual-receipt",type=Path,help="Full snapshot-audit browser checks; save receipt outside the report")
|
|
804
|
+
args = parser.parse_args(argv)
|
|
805
|
+
if args.visual_receipt:
|
|
806
|
+
from harness.web_fetch.published_snapshot import check_snapshot_visual
|
|
807
|
+
print(check_snapshot_visual(args.root,args.visual_receipt))
|
|
808
|
+
result = check(args.root, authority_root=args.authority_root, profile=args.profile, presentation_binding=args.presentation_binding, source_dataset=args.source_dataset)
|
|
809
|
+
if result is not None:
|
|
810
|
+
print(result)
|
|
811
|
+
|
|
812
|
+
|
|
813
|
+
def check_revision(out: Path, *, authority_root: Path, profile_path: Path | None = None):
|
|
814
|
+
out = open_report(out)
|
|
815
|
+
repository = authority_root.resolve()
|
|
816
|
+
if (out / "wcxb-score-summary.json").is_file():
|
|
817
|
+
|
|
818
|
+
check_wcxb(out, authority_root=repository, profile_path=profile_path)
|
|
819
|
+
return "PASS: WCXB release scope and report acceptance checks verified"
|
|
820
|
+
dataset = read_json(out / "dataset.json")
|
|
821
|
+
|
|
822
|
+
if dataset.get("comparison", {}).get("kind") == "webmainbench_frozen_html_to_markdown":
|
|
823
|
+
|
|
824
|
+
check_sample_report(out, authority_root=repository, profile_path=profile_path)
|
|
825
|
+
return (
|
|
826
|
+
"PASS: WebMainBench preselected-200 release scope and report acceptance checks verified"
|
|
827
|
+
)
|
|
828
|
+
binding = dataset.get('comparison', {}).get('report_interpretation')
|
|
829
|
+
if binding:
|
|
830
|
+
check_current_browser_article(out, {case['id']:case for case in dataset['cases']},
|
|
831
|
+
load_jsonl(out/'results.jsonl'), binding)
|
|
832
|
+
provenance_path = out / "implementation-provenance.json"
|
|
833
|
+
if provenance_path.is_file():
|
|
834
|
+
provenance = read_json(provenance_path)
|
|
835
|
+
records = load_jsonl(out / "results.jsonl")
|
|
836
|
+
home = read_page(out / "index.html")
|
|
837
|
+
for item in provenance.get("implementations", []):
|
|
838
|
+
name = item.get("provider")
|
|
839
|
+
if native_browser_engine(name) != "moli" or not item.get("version"):
|
|
840
|
+
continue
|
|
841
|
+
hashes = {r.get("provider_metadata", {}).get("binary_sha256")
|
|
842
|
+
for r in records if r["provider"] == name}
|
|
843
|
+
if hashes != {item.get("binary_sha256")}:
|
|
844
|
+
continue
|
|
845
|
+
expected = "Moli " + item["version"]
|
|
846
|
+
links = home.select('a[href="providers/' + name + '.html"]')
|
|
847
|
+
if not links or any(anchor.get_text() != expected for anchor in links):
|
|
848
|
+
raise ValueError("Measured method version label: " + name)
|
|
849
|
+
check_language_contract(out)
|
|
850
|
+
has_english = "en" in generated_languages(out)
|
|
851
|
+
for path in out.rglob("*.html"):
|
|
852
|
+
if not has_english and path.relative_to(out).parts[0] == "en":
|
|
853
|
+
continue
|
|
854
|
+
soup = read_page(path)
|
|
855
|
+
relative_parts = path.relative_to(out).parts
|
|
856
|
+
is_english = relative_parts[0] == "en"
|
|
857
|
+
is_browser_use = "browser-use" in relative_parts
|
|
858
|
+
if path in {out / "index.html", out / "en/index.html"}:
|
|
859
|
+
overview = soup.select_one(".home-overview, #overview")
|
|
860
|
+
if not (overview and (overview.name != "details" or not overview.has_attr("open"))):
|
|
861
|
+
raise ValueError(path)
|
|
862
|
+
panels = soup.select('#home-rankings > [role="tabpanel"]')
|
|
863
|
+
if not sum(not p.has_attr("hidden") for p in panels) == 1:
|
|
864
|
+
raise ValueError(path)
|
|
865
|
+
for panel in panels:
|
|
866
|
+
if not len(panel.select(".track-guide")) == 1:
|
|
867
|
+
raise ValueError(path)
|
|
868
|
+
if not len(panel.select(".track-flow li")) >= 4:
|
|
869
|
+
raise ValueError(path)
|
|
870
|
+
for selector in (".track-meaning", ".track-flow", ".pass-contract"):
|
|
871
|
+
if panel.select_one(selector).find_parent("details") is not None:
|
|
872
|
+
raise ValueError(path)
|
|
873
|
+
for selector in (".track-more", ".ranking-metric-help"):
|
|
874
|
+
if not not panel.select_one(selector).has_attr("open"):
|
|
875
|
+
raise ValueError(path)
|
|
876
|
+
for pre in soup.select("pre"):
|
|
877
|
+
pre.decompose()
|
|
878
|
+
if not ("条件通过率" not in soup.get_text() and "条件达标率" not in soup.get_text()):
|
|
879
|
+
raise ValueError(path)
|
|
880
|
+
if not is_browser_use:
|
|
881
|
+
for table in soup.select("table"):
|
|
882
|
+
width = len(table.select("thead th"))
|
|
883
|
+
if not all(
|
|
884
|
+
len(row.find_all("td", recursive=False)) == width
|
|
885
|
+
for row in table.select("tbody > tr")
|
|
886
|
+
):
|
|
887
|
+
raise ValueError((path, width))
|
|
888
|
+
for table in soup.select("table.ranking"):
|
|
889
|
+
headers = table.select("thead th")
|
|
890
|
+
body_families = {"web", "dynamic", "challenge", "risk_control", "document"}
|
|
891
|
+
family = table.get("id", "").removeprefix("ranking-")
|
|
892
|
+
if family in body_families and any(
|
|
893
|
+
c.get("track", "web") == family and c.get("task_kind", "body") in {"body", "structure"}
|
|
894
|
+
for c in dataset["cases"]
|
|
895
|
+
):
|
|
896
|
+
labels = [h.get_text() for h in headers]
|
|
897
|
+
required = (
|
|
898
|
+
["Recall", "Precision", "F1"] if is_english else [t('recall', 'Recall'), t('precision', 'Precision'), "F1"]
|
|
899
|
+
)
|
|
900
|
+
forbidden = (
|
|
901
|
+
["Resource hit rate", "File retrieval rate"]
|
|
902
|
+
if is_english else [t('resource_match_rate', 'Resource match rate'), t('file_delivery_rate', 'File delivery rate')]
|
|
903
|
+
)
|
|
904
|
+
if not all(any(label in h for h in labels) for label in required) or any(
|
|
905
|
+
label in h for label in forbidden for h in labels
|
|
906
|
+
):
|
|
907
|
+
raise ValueError((path, "Body ranking metric columns", family))
|
|
908
|
+
sort = int(table["data-col"])
|
|
909
|
+
expected = (
|
|
910
|
+
(["Matched repositories"] if is_english else [t('matched_repositories_2', 'Matched repositories')])
|
|
911
|
+
if path.parent.name == "adoption"
|
|
912
|
+
else ["Task pass rate", "Body pass rate", "Overall task pass rate"]
|
|
913
|
+
if is_english
|
|
914
|
+
else [t('task_pass_rate', 'Task pass rate'), t('body_text_pass_rate', 'Body-text pass rate'), t('overall_task_pass_rate', 'Overall task pass rate')]
|
|
915
|
+
)
|
|
916
|
+
if not any(label in headers[sort].get_text() for label in expected):
|
|
917
|
+
raise ValueError((path, sort))
|
|
918
|
+
legend = table.parent.find_previous_sibling(
|
|
919
|
+
"div", class_="metric-help"
|
|
920
|
+
) or table.parent.find_previous_sibling("details", class_="ranking-metric-help")
|
|
921
|
+
if not (legend and legend.select("dt")):
|
|
922
|
+
raise ValueError(path)
|
|
923
|
+
track = soup.select_one('select[data-filter="track"]')
|
|
924
|
+
if track:
|
|
925
|
+
if not track.select("option")[0]["value"] == "":
|
|
926
|
+
raise ValueError(path)
|
|
927
|
+
if path.parent.name in {"providers", "analysis"}:
|
|
928
|
+
if not not soup.select(
|
|
929
|
+
'select[data-filter="transport"],select[data-filter="content"],select[data-filter="resources"]'
|
|
930
|
+
):
|
|
931
|
+
raise ValueError(path)
|
|
932
|
+
expected_results = (
|
|
933
|
+
["Pass", "Fail", "All"] if is_english else [t('pass', 'Pass'), t('fail', 'Fail'), t('all', 'All')]
|
|
934
|
+
)
|
|
935
|
+
if (
|
|
936
|
+
not [x.get_text() for x in soup.select('select[data-filter="result"] option')]
|
|
937
|
+
== expected_results
|
|
938
|
+
):
|
|
939
|
+
raise ValueError(path)
|
|
940
|
+
if (
|
|
941
|
+
not soup.select_one('select[data-filter="result"] option[selected]')["value"]
|
|
942
|
+
== "fail"
|
|
943
|
+
):
|
|
944
|
+
raise ValueError(path)
|
|
945
|
+
expected_failure = report_text('en' if is_english else 'zh')('failure_class', 'Failure class')
|
|
946
|
+
if expected_failure not in [h.get_text() for h in soup.select("#filtered-table th")]:
|
|
947
|
+
raise ValueError(path)
|
|
948
|
+
records = load_jsonl(out / 'results.jsonl')
|
|
949
|
+
synthetic = dataset.get('comparison', {}).get('kind') == 'synthetic_browser_report'
|
|
950
|
+
if records and all(row.get('provider_type') == 'fixture' for row in records) and (
|
|
951
|
+
synthetic or all(not native_browser_engine(row['provider']) for row in records)
|
|
952
|
+
):
|
|
953
|
+
methods = read_page(out / 'adoption/index.html')
|
|
954
|
+
for name in {row['provider'] for row in records}:
|
|
955
|
+
if methods.find(id='interface-' + name) is None:
|
|
956
|
+
raise ValueError('Synthetic adapter interface is missing: ' + name)
|
|
957
|
+
return 'PASS: synthetic report presentation, declared adapters and bound interpretation verified'
|
|
958
|
+
counts = read_json(out / "adoption/counts.json")["methods"]
|
|
959
|
+
names = {row["provider"] for row in load_jsonl(out / "results.jsonl")}
|
|
960
|
+
|
|
961
|
+
if not {item["provider"] for item in counts} == set(PROFILES) | names:
|
|
962
|
+
raise ValueError("Report presentation check failed")
|
|
963
|
+
if len(counts) != len({item["provider"] for item in counts}):
|
|
964
|
+
raise ValueError("Duplicate adoption provider counts")
|
|
965
|
+
interfaces = read_page(out / "adoption/index.html").select(
|
|
966
|
+
'.interface-panel[id^="interface-"]'
|
|
967
|
+
)
|
|
968
|
+
interface_names = set(SPECS) | {name for name in names if native_browser_engine(name)}
|
|
969
|
+
|
|
970
|
+
if not (
|
|
971
|
+
len(interfaces) == len(interface_names)
|
|
972
|
+
and sum(p.has_attr("open") for p in interfaces) == bool(interfaces)
|
|
973
|
+
):
|
|
974
|
+
raise ValueError("Report presentation check failed")
|
|
975
|
+
if {panel.get("id") for panel in interfaces} != {"interface-" + name for name in interface_names}:
|
|
976
|
+
raise ValueError("Interface inventory differs from registry")
|
|
977
|
+
for panel in interfaces:
|
|
978
|
+
if not len(panel.select("pre")) == 2:
|
|
979
|
+
raise ValueError("Report presentation check failed")
|
|
980
|
+
for item in counts:
|
|
981
|
+
if item["repository_count"] is None:
|
|
982
|
+
continue
|
|
983
|
+
if not item["repository_count"] == len({x["repo"].lower() for x in item["repositories"]}):
|
|
984
|
+
raise ValueError("Report presentation check failed")
|
|
985
|
+
if not item["file_count"] == len(
|
|
986
|
+
{(r["repo"].lower(), m["path"]) for r in item["repositories"] for m in r["matches"]}
|
|
987
|
+
):
|
|
988
|
+
raise ValueError("Report presentation check failed")
|
|
989
|
+
return (
|
|
990
|
+
"PASS: all tables align, obsolete metrics absent, active sort columns correct, "
|
|
991
|
+
"three-state results, all-type defaults, adoption counts traceable"
|
|
992
|
+
)
|
|
993
|
+
|
|
994
|
+
|
|
995
|
+
def revision_main(argv=None, *, authority_root: Path | None = None):
|
|
996
|
+
parser = argparse.ArgumentParser(description="Report presentation and adoption checks")
|
|
997
|
+
parser.add_argument("root", type=Path, nargs="?")
|
|
998
|
+
parser.add_argument("--profile", type=Path)
|
|
999
|
+
parser.add_argument("--authority-root", type=Path, default=authority_root or Path.cwd())
|
|
1000
|
+
args = parser.parse_args(argv)
|
|
1001
|
+
result = check_revision(
|
|
1002
|
+
args.root or args.authority_root / "reports/webfetch", authority_root=args.authority_root, profile_path=args.profile
|
|
1003
|
+
)
|
|
1004
|
+
print(result)
|