webstone 0.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- harness/benchmark_execution.py +445 -0
- harness/benchmarks/__init__.py +1 -0
- harness/benchmarks/cards/expanded-swde.md +45 -0
- harness/benchmarks/cards/headless-browser.md +112 -0
- harness/benchmarks/cards/languages.json +1018 -0
- harness/benchmarks/cards/liveweb-ie.md +49 -0
- harness/benchmarks/cards/metadata.json +485 -0
- harness/benchmarks/cards/newsxlm.md +47 -0
- harness/benchmarks/cards/simpleqa-verified.md +75 -0
- harness/benchmarks/cards/swde.json +47 -0
- harness/benchmarks/cards/trajectory-replay.md +82 -0
- harness/benchmarks/cards/wcxb.md +116 -0
- harness/benchmarks/cards/web-extract.md +119 -0
- harness/benchmarks/cards/webarena-replay.md +91 -0
- harness/benchmarks/cards/webfetch.md +297 -0
- harness/benchmarks/cards/webmainbench.md +63 -0
- harness/benchmarks/cards/websearch.md +98 -0
- harness/benchmarks/cards/zyte-product-extraction.json +42 -0
- harness/benchmarks/code-migration.json +4554 -0
- harness/benchmarks/data-lock.json +129219 -0
- harness/benchmarks/data.py +397 -0
- harness/benchmarks/pinning.py +328 -0
- harness/benchmarks/publication.py +780 -0
- harness/benchmarks/scoring-regressions.json +90025 -0
- harness/browser_use/__init__.py +5 -0
- harness/browser_use/__main__.py +4 -0
- harness/browser_use/assets/evidence-viewer.js +1 -0
- harness/browser_use/assets/report.css +63 -0
- harness/browser_use/assets/report.js +12 -0
- harness/browser_use/assets/webarena-autonomous-evaluation-flow.png +0 -0
- harness/browser_use/cli.py +46 -0
- harness/browser_use/failure_cause_review.py +160 -0
- harness/browser_use/messages.json +699 -0
- harness/browser_use/model.py +258 -0
- harness/browser_use/publication.py +191 -0
- harness/browser_use/report.py +4261 -0
- harness/browser_use/report_check.py +982 -0
- harness/browser_use/report_i18n.py +175 -0
- harness/browser_use/report_package.py +100 -0
- harness/browser_use/report_publish.py +194 -0
- harness/browser_use/report_visual.py +265 -0
- harness/browser_use/review.py +170 -0
- harness/browser_use/runner.py +811 -0
- harness/browser_use/selection.py +346 -0
- harness/container_isolation.py +46 -0
- harness/data_home.py +15 -0
- harness/distribution.py +46 -0
- harness/distribution_data/configs/providers.headless.json +120 -0
- harness/distribution_data/evaluation/browser-use-public/adoption-evidence.json +122 -0
- harness/distribution_data/evaluation/browser-use-public/benchmark-catalog.json +244 -0
- harness/distribution_data/evaluation/browser-use-public/report-changes.json +55 -0
- harness/distribution_data/evaluation/webmainbench-full/scoring-regressions.json +43 -0
- harness/distribution_data/pyproject.toml +209 -0
- harness/distribution_data/uv.lock +5088 -0
- harness/env_file.py +125 -0
- harness/environments/__init__.py +1 -0
- harness/environments/browser.py +150 -0
- harness/environments/webarena/__init__.py +1 -0
- harness/environments/webarena/backend.py +108 -0
- harness/environments/webarena/cli.py +88 -0
- harness/environments/webarena/config.py +280 -0
- harness/environments/webarena/har.py +835 -0
- harness/environments/webarena/manager.py +204 -0
- harness/environments/webarena/map_archives.py +60 -0
- harness/environments/webarena/map_check.py +103 -0
- harness/environments/webarena/map_detail.py +149 -0
- harness/environments/webarena/map_setup.py +537 -0
- harness/environments/webarena/rainbond.py +1185 -0
- harness/environments/webarena/runtime/php-fpm-qemu.ini +12 -0
- harness/environments/webarena/runtime.py +736 -0
- harness/environments/webarena/topology.json +81 -0
- harness/environments/webarena/topology.py +238 -0
- harness/fix_log.py +305 -0
- harness/lexbench/__init__.py +1 -0
- harness/lexbench/__main__.py +4 -0
- harness/lexbench/assets/bundle-loader.js +75 -0
- harness/lexbench/assets/render_contract.cjs +382 -0
- harness/lexbench/assets/ui_check.js +636 -0
- harness/lexbench/charts.py +469 -0
- harness/lexbench/cli.py +45 -0
- harness/lexbench/pack.py +136 -0
- harness/lexbench/report.py +2132 -0
- harness/lexbench/reproduction.py +234 -0
- harness/lexbench/resource_evidence.py +199 -0
- harness/lexbench/resource_matrix.py +298 -0
- harness/lexbench/resource_profiles.py +315 -0
- harness/lexbench/resource_summary.py +277 -0
- harness/lexbench/template/README.md +11 -0
- harness/lexbench/template/app.js +1607 -0
- harness/lexbench/template/index.html +51 -0
- harness/lexbench/template/messages.json +744 -0
- harness/lexbench/template/style.css +905 -0
- harness/lexbench/verification.py +1239 -0
- harness/licenses/Lexbench-Apache-2.0.txt +201 -0
- harness/licenses/WCXB-CC-BY-4.0.txt +25 -0
- harness/numbers.py +14 -0
- harness/package_gate.py +84 -0
- harness/platform_capabilities.py +37 -0
- harness/record_io.py +107 -0
- harness/report_languages.py +246 -0
- harness/reporting/__init__.py +1 -0
- harness/reporting/assets.py +182 -0
- harness/reporting/certified_cli.py +24 -0
- harness/reporting/links.py +168 -0
- harness/reporting/locales.py +30 -0
- harness/reporting/mermaid.py +28 -0
- harness/reporting/packs.py +336 -0
- harness/reporting/presentation.py +176 -0
- harness/reporting/pricing.py +29 -0
- harness/reporting/privacy.py +95 -0
- harness/reporting/publication.py +561 -0
- harness/reporting/publication_target.py +92 -0
- harness/reporting/publish.py +894 -0
- harness/reporting/release_check.py +51 -0
- harness/reporting/source_policy.py +64 -0
- harness/reporting/templates.py +109 -0
- harness/reporting/versions.py +135 -0
- harness/reporting/visual.py +215 -0
- harness/resource_accounting.py +256 -0
- harness/resource_calibration.py +287 -0
- harness/results/__init__.py +5 -0
- harness/results/__main__.py +3 -0
- harness/results/cli.py +123 -0
- harness/results/client.py +783 -0
- harness/results/versions.py +72 -0
- harness/review_runner.py +122 -0
- harness/trajectory_replay/__init__.py +5 -0
- harness/trajectory_replay/__main__.py +4 -0
- harness/trajectory_replay/assets/compact-data-fetch-adapter.js +25 -0
- harness/trajectory_replay/assets/compact-detail-loader.js +11 -0
- harness/trajectory_replay/assets/compact-evidence-loader.js +11 -0
- harness/trajectory_replay/assets/curated-execution-filter.js +38 -0
- harness/trajectory_replay/assets/curated-report.css +6 -0
- harness/trajectory_replay/assets/evidence-dialog.js +40 -0
- harness/trajectory_replay/assets/public-webarena-benchmark-tabs.js +4 -0
- harness/trajectory_replay/assets/public-webarena-comparison.js +4 -0
- harness/trajectory_replay/assets/public-webarena-execution-filter.js +39 -0
- harness/trajectory_replay/assets/public-webarena-report.css +13 -0
- harness/trajectory_replay/assets/suite-filter.js +7 -0
- harness/trajectory_replay/assets/webarena-article.css +1 -0
- harness/trajectory_replay/assets/webarena-catalog.css +12 -0
- harness/trajectory_replay/assets/webarena-report.css +52 -0
- harness/trajectory_replay/assets/webarena-report.js +280 -0
- harness/trajectory_replay/browser_use_seeds.py +375 -0
- harness/trajectory_replay/catalog_acceptance.py +717 -0
- harness/trajectory_replay/catalog_action_audit.py +190 -0
- harness/trajectory_replay/catalog_binding.py +198 -0
- harness/trajectory_replay/catalog_comparison.py +423 -0
- harness/trajectory_replay/catalog_consolidation.py +330 -0
- harness/trajectory_replay/catalog_contracts.py +833 -0
- harness/trajectory_replay/catalog_release.py +80 -0
- harness/trajectory_replay/catalog_runner.py +832 -0
- harness/trajectory_replay/cli.py +309 -0
- harness/trajectory_replay/core/__init__.py +1 -0
- harness/trajectory_replay/core/diagnostics.py +263 -0
- harness/trajectory_replay/core/docker_runner.py +374 -0
- harness/trajectory_replay/core/executor.py +72 -0
- harness/trajectory_replay/core/metrics.py +365 -0
- harness/trajectory_replay/core/report.py +113 -0
- harness/trajectory_replay/core/suite.py +190 -0
- harness/trajectory_replay/diagnostics.py +685 -0
- harness/trajectory_replay/evaluator_transition.py +434 -0
- harness/trajectory_replay/executor.py +165 -0
- harness/trajectory_replay/expert_seeds.py +188 -0
- harness/trajectory_replay/failure_matrix.py +280 -0
- harness/trajectory_replay/interpretation.py +186 -0
- harness/trajectory_replay/messages.json +1002 -0
- harness/trajectory_replay/messages.py +11 -0
- harness/trajectory_replay/metrics.py +35 -0
- harness/trajectory_replay/participants.py +97 -0
- harness/trajectory_replay/playwright_tool.py +1331 -0
- harness/trajectory_replay/public_webarena/__init__.py +5 -0
- harness/trajectory_replay/public_webarena/cli.py +204 -0
- harness/trajectory_replay/public_webarena/config.py +79 -0
- harness/trajectory_replay/public_webarena/diagnostics.py +1163 -0
- harness/trajectory_replay/public_webarena/evaluators.py +155 -0
- harness/trajectory_replay/public_webarena/executor.py +695 -0
- harness/trajectory_replay/public_webarena/har.py +393 -0
- harness/trajectory_replay/public_webarena/manifest_repair.py +393 -0
- harness/trajectory_replay/public_webarena/metrics.py +47 -0
- harness/trajectory_replay/public_webarena/publication.py +492 -0
- harness/trajectory_replay/public_webarena/qualification.py +2765 -0
- harness/trajectory_replay/public_webarena/release.py +171 -0
- harness/trajectory_replay/public_webarena/report.py +1903 -0
- harness/trajectory_replay/public_webarena/response_builder.py +327 -0
- harness/trajectory_replay/public_webarena/result_audit.py +185 -0
- harness/trajectory_replay/public_webarena/runner.py +908 -0
- harness/trajectory_replay/public_webarena/suite.py +410 -0
- harness/trajectory_replay/public_webarena/targets.py +93 -0
- harness/trajectory_replay/public_webarena/webarena.py +735 -0
- harness/trajectory_replay/public_webarena/webarena_environment.py +735 -0
- harness/trajectory_replay/remote_execution.py +1581 -0
- harness/trajectory_replay/remote_runtime.py +1071 -0
- harness/trajectory_replay/report.py +1236 -0
- harness/trajectory_replay/report_compaction.py +217 -0
- harness/trajectory_replay/report_inputs.py +170 -0
- harness/trajectory_replay/report_languages.py +14 -0
- harness/trajectory_replay/resources/fixtures/app.html +11 -0
- harness/trajectory_replay/resources/fixtures/catalog.html +9 -0
- harness/trajectory_replay/resources/fixtures/core.html +18 -0
- harness/trajectory_replay/resources/fixtures/dynamic.html +11 -0
- harness/trajectory_replay/resources/fixtures/forms.html +11 -0
- harness/trajectory_replay/resources/fixtures/interaction-space.html +142 -0
- harness/trajectory_replay/resources/fixtures/page2.html +3 -0
- harness/trajectory_replay/resources/fixtures/page3.html +2 -0
- harness/trajectory_replay/retrieval_acceptance.py +1080 -0
- harness/trajectory_replay/runner.py +304 -0
- harness/trajectory_replay/runtime_identity.py +131 -0
- harness/trajectory_replay/suite.py +83 -0
- harness/trajectory_replay/webarena_corrections.py +1088 -0
- harness/trajectory_replay/webarena_dataset.py +684 -0
- harness/trajectory_replay/webarena_plan.py +658 -0
- harness/trajectory_replay/webarena_replay.py +1657 -0
- harness/trajectory_replay/webarena_report.py +2290 -0
- harness/trajectory_replay/webarena_report_check.py +944 -0
- harness/trajectory_replay/webarena_report_package.py +155 -0
- harness/trajectory_replay/webarena_report_publish.py +230 -0
- harness/trajectory_replay/webarena_report_visual.py +693 -0
- harness/trajectory_replay/webarena_results.py +644 -0
- harness/trajectory_replay/webarena_review.py +198 -0
- harness/trajectory_replay/webarena_runtime_source.py +119 -0
- harness/web_extract/__init__.py +1 -0
- harness/web_extract/analysis.py +683 -0
- harness/web_extract/assets/report.css +58 -0
- harness/web_extract/assets/report.js +431 -0
- harness/web_extract/certification.py +596 -0
- harness/web_extract/cli.py +436 -0
- harness/web_extract/dataset.py +274 -0
- harness/web_extract/example.py +473 -0
- harness/web_extract/firecrawl_backend.mjs +178 -0
- harness/web_extract/messages.json +1070 -0
- harness/web_extract/messages.py +31 -0
- harness/web_extract/model_comparison.py +613 -0
- harness/web_extract/pipeline.py +2580 -0
- harness/web_extract/publication.py +209 -0
- harness/web_extract/records.py +574 -0
- harness/web_extract/repair.py +657 -0
- harness/web_extract/report.py +6167 -0
- harness/web_extract/report_suite.py +353 -0
- harness/web_extract/report_ui.py +66 -0
- harness/web_extract/review.py +1580 -0
- harness/web_extract/self_hosted.py +978 -0
- harness/web_extract/snapshots.py +1197 -0
- harness/web_extract/swde.py +742 -0
- harness/web_fetch/__init__.py +5 -0
- harness/web_fetch/__main__.py +5 -0
- harness/web_fetch/accounts.py +130 -0
- harness/web_fetch/adjudication_evidence.py +767 -0
- harness/web_fetch/anomaly_review.py +346 -0
- harness/web_fetch/artifact_auth.py +135 -0
- harness/web_fetch/artifacts.py +521 -0
- harness/web_fetch/assembly.py +1324 -0
- harness/web_fetch/assets/cohort-diagnostic.css +18 -0
- harness/web_fetch/assets/cohort-diagnostic.js +2 -0
- harness/web_fetch/assets/methodology/annotation.png +0 -0
- harness/web_fetch/assets/methodology/scoring.png +0 -0
- harness/web_fetch/assets/methodology/validation.png +0 -0
- harness/web_fetch/assets/report-extra.css +64 -0
- harness/web_fetch/assets/report-loader.js +167 -0
- harness/web_fetch/assets/report-style.css +68 -0
- harness/web_fetch/assets/report.js +57 -0
- harness/web_fetch/assets/wcxb.css +1 -0
- harness/web_fetch/assets/wcxb.js +1 -0
- harness/web_fetch/assets/webmainbench-full.css +1 -0
- harness/web_fetch/assets/webmainbench-full.js +52 -0
- harness/web_fetch/benchmark_runner.py +190 -0
- harness/web_fetch/browser_observations.py +980 -0
- harness/web_fetch/call_boundary.py +142 -0
- harness/web_fetch/certification.py +683 -0
- harness/web_fetch/cli.py +394 -0
- harness/web_fetch/cohort_exclusions.py +38 -0
- harness/web_fetch/comparison_audit.py +479 -0
- harness/web_fetch/config.py +231 -0
- harness/web_fetch/container_resources.py +220 -0
- harness/web_fetch/data/browser-observation-contracts.json +25 -0
- harness/web_fetch/data/fixture-dataset.json +42 -0
- harness/web_fetch/data/fixture-providers.json +11 -0
- harness/web_fetch/data/latency-certifiers/34bdb7aedb25a9e5a0c64e0c549926ed7e71f747f544ec2035f65952c0177347.py.txt +142 -0
- harness/web_fetch/data/latency-certifiers/95a2f0a73578346998f0a4ad1256b2cef469a3f2db9b4b30550fc03fc5c259a8.py.txt +142 -0
- harness/web_fetch/data/native-projection-history.json +35 -0
- harness/web_fetch/data/native-projection-regressions.json +531 -0
- harness/web_fetch/data/report-methods.json +6687 -0
- harness/web_fetch/data/report-track-context.json +132 -0
- harness/web_fetch/data/wcxb-metric-bindings.json +12 -0
- harness/web_fetch/data/webmainbench-latency-source.json +18 -0
- harness/web_fetch/dataset_assembly.py +274 -0
- harness/web_fetch/diagnostics.py +73 -0
- harness/web_fetch/document_text.py +55 -0
- harness/web_fetch/dynamic_cohort.py +467 -0
- harness/web_fetch/evidence.py +753 -0
- harness/web_fetch/failure_diagnosis.py +190 -0
- harness/web_fetch/headless_runner.py +510 -0
- harness/web_fetch/html_to_markdown.mjs +21 -0
- harness/web_fetch/local_audit.py +453 -0
- harness/web_fetch/local_execution_contract.py +317 -0
- harness/web_fetch/local_execution_legacy.py +205 -0
- harness/web_fetch/local_revision.py +1192 -0
- harness/web_fetch/markdown.py +32 -0
- harness/web_fetch/messages.json +2377 -0
- harness/web_fetch/methods.py +451 -0
- harness/web_fetch/models.py +70 -0
- harness/web_fetch/native_projection.py +769 -0
- harness/web_fetch/native_routes.py +637 -0
- harness/web_fetch/network_dns.py +363 -0
- harness/web_fetch/network_preflight.py +119 -0
- harness/web_fetch/provider_adoption.py +321 -0
- harness/web_fetch/providers/__init__.py +4 -0
- harness/web_fetch/providers/base.py +274 -0
- harness/web_fetch/providers/cloudflare.py +816 -0
- harness/web_fetch/providers/cowagent.py +78 -0
- harness/web_fetch/providers/exa.py +177 -0
- harness/web_fetch/providers/firecrawl.py +454 -0
- harness/web_fetch/providers/fixture.py +49 -0
- harness/web_fetch/providers/headless.py +1147 -0
- harness/web_fetch/providers/headless_worker.py +512 -0
- harness/web_fetch/providers/hosted_fetch.py +250 -0
- harness/web_fetch/providers/http.py +87 -0
- harness/web_fetch/providers/http_client.py +112 -0
- harness/web_fetch/providers/lexmount.py +211 -0
- harness/web_fetch/providers/moli.py +889 -0
- harness/web_fetch/providers/runtime_identity.py +220 -0
- harness/web_fetch/providers/subprocess_jsonl.py +410 -0
- harness/web_fetch/providers/tavily.py +116 -0
- harness/web_fetch/public_benchmark_sample.py +160 -0
- harness/web_fetch/publication.py +26 -0
- harness/web_fetch/publication_check.py +379 -0
- harness/web_fetch/publication_privacy.py +1136 -0
- harness/web_fetch/publication_verify.py +390 -0
- harness/web_fetch/published_snapshot.py +379 -0
- harness/web_fetch/reference_freeze.py +699 -0
- harness/web_fetch/reference_inputs.py +387 -0
- harness/web_fetch/reference_lifecycle.py +1811 -0
- harness/web_fetch/reference_revision.py +408 -0
- harness/web_fetch/refresh_assembly.py +1723 -0
- harness/web_fetch/reliability.py +80 -0
- harness/web_fetch/render_evidence.py +215 -0
- harness/web_fetch/report-translations.json +3456 -0
- harness/web_fetch/report.py +4564 -0
- harness/web_fetch/report_build.py +1851 -0
- harness/web_fetch/report_capabilities.py +149 -0
- harness/web_fetch/report_changes.py +224 -0
- harness/web_fetch/report_check.py +1004 -0
- harness/web_fetch/report_components.py +512 -0
- harness/web_fetch/report_diagnostic.py +857 -0
- harness/web_fetch/report_explainer.py +1133 -0
- harness/web_fetch/report_i18n.py +942 -0
- harness/web_fetch/report_interfaces.py +262 -0
- harness/web_fetch/report_methods.py +159 -0
- harness/web_fetch/report_package.py +1000 -0
- harness/web_fetch/report_pricing.py +120 -0
- harness/web_fetch/report_publish.py +158 -0
- harness/web_fetch/report_quality.py +34 -0
- harness/web_fetch/report_reading.py +124 -0
- harness/web_fetch/report_resources.py +196 -0
- harness/web_fetch/report_saved_composition.py +149 -0
- harness/web_fetch/report_saved_matrix.py +2662 -0
- harness/web_fetch/report_style.py +9 -0
- harness/web_fetch/report_version.py +270 -0
- harness/web_fetch/rescoring.py +332 -0
- harness/web_fetch/resource_cohort.py +207 -0
- harness/web_fetch/result_ledger.py +119 -0
- harness/web_fetch/result_review.py +1252 -0
- harness/web_fetch/retrieval_hints.py +574 -0
- harness/web_fetch/review_execution.py +1638 -0
- harness/web_fetch/risk_control.py +524 -0
- harness/web_fetch/runner.py +1241 -0
- harness/web_fetch/safety.py +140 -0
- harness/web_fetch/scoring_revision.py +391 -0
- harness/web_fetch/source_capture.py +455 -0
- harness/web_fetch/speed_cohort.py +216 -0
- harness/web_fetch/storage.py +336 -0
- harness/web_fetch/task_outputs.py +346 -0
- harness/web_fetch/templates/interpretation-headless.css +326 -0
- harness/web_fetch/title_quality.py +93 -0
- harness/web_fetch/wcxb_dataset.py +168 -0
- harness/web_fetch/wcxb_latency.py +347 -0
- harness/web_fetch/wcxb_metrics.py +38 -0
- harness/web_fetch/wcxb_publication.py +147 -0
- harness/web_fetch/wcxb_report_check.py +753 -0
- harness/web_fetch/wcxb_scoring.py +667 -0
- harness/web_fetch/webmainbench_assembly.py +684 -0
- harness/web_fetch/webmainbench_case_correction.py +267 -0
- harness/web_fetch/webmainbench_certification.py +306 -0
- harness/web_fetch/webmainbench_correction.py +866 -0
- harness/web_fetch/webmainbench_diagnostics.py +1202 -0
- harness/web_fetch/webmainbench_evidence.py +205 -0
- harness/web_fetch/webmainbench_latency_correction.py +525 -0
- harness/web_fetch/webmainbench_latency_diagnostic.py +562 -0
- harness/web_fetch/webmainbench_report_check.py +900 -0
- harness/web_fetch/webmainbench_runner.py +680 -0
- harness/web_fetch/webmainbench_scoring.py +354 -0
- harness/web_fetch/workers/__init__.py +1 -0
- harness/web_fetch/workers/cowagent_worker.py +70 -0
- harness/web_fetch/workers/crawl4ai_worker.py +463 -0
- harness/web_fetch/workers/dsh_output.mjs +21 -0
- harness/web_fetch/workers/dsh_webfetch_worker.mts +100 -0
- harness/web_fetch/workers/langchain_webbase_worker.py +67 -0
- harness/web_fetch/workers/native_dns_darwin.c +108 -0
- harness/web_fetch/workers/openclaw_webfetch_worker.mts +91 -0
- harness/web_fetch/workers/public_dns.mjs +49 -0
- harness/web_search/__init__.py +1 -0
- harness/web_search/__main__.py +4 -0
- harness/web_search/adapters.py +371 -0
- harness/web_search/answers.py +326 -0
- harness/web_search/assets/comparison.css +5 -0
- harness/web_search/assets/comparison.js +45 -0
- harness/web_search/assets/delivery-bridge.js +35 -0
- harness/web_search/assets/delivery-loader.js +35 -0
- harness/web_search/assets/native-evidence-result.js +8 -0
- harness/web_search/assets/report.css +69 -0
- harness/web_search/assets/report.js +58 -0
- harness/web_search/cli.py +44 -0
- harness/web_search/comparison.py +843 -0
- harness/web_search/core.py +274 -0
- harness/web_search/data/native-normalization-regressions.json +354 -0
- harness/web_search/delivery.py +163 -0
- harness/web_search/diagnostics.py +132 -0
- harness/web_search/example.py +147 -0
- harness/web_search/messages.json +751 -0
- harness/web_search/messages.py +23 -0
- harness/web_search/output_levels.py +431 -0
- harness/web_search/pages.py +28 -0
- harness/web_search/profiles.py +220 -0
- harness/web_search/publication.py +208 -0
- harness/web_search/relevance.py +828 -0
- harness/web_search/report.py +4315 -0
- harness/web_search/review.py +1318 -0
- harness/web_search/revisions.py +1052 -0
- harness/web_search/runner.py +1013 -0
- harness/web_search/visual.py +164 -0
- harness/webarena_state.py +549 -0
- lexbench_webfetch/__init__.py +1 -0
- lexbench_webfetch/annotation_revisions.py +269 -0
- lexbench_webfetch/article_comments.py +331 -0
- lexbench_webfetch/article_dataset.py +285 -0
- lexbench_webfetch/article_posts.py +384 -0
- lexbench_webfetch/article_qualification.py +192 -0
- lexbench_webfetch/article_text.py +259 -0
- lexbench_webfetch/article_title.py +186 -0
- lexbench_webfetch/benchmark.py +591 -0
- lexbench_webfetch/body_coverage.py +582 -0
- lexbench_webfetch/cli.py +143 -0
- lexbench_webfetch/composite_tasks.py +2026 -0
- lexbench_webfetch/content_scopes.py +156 -0
- lexbench_webfetch/document_structure.py +326 -0
- lexbench_webfetch/dynamic_entry_matching.py +190 -0
- lexbench_webfetch/dynamic_lists.py +1658 -0
- lexbench_webfetch/dynamic_reference.py +25 -0
- lexbench_webfetch/evaluation.py +24 -0
- lexbench_webfetch/file_acceptance.py +72 -0
- lexbench_webfetch/inputs.py +129 -0
- lexbench_webfetch/list_verification.py +90 -0
- lexbench_webfetch/metrics.py +1441 -0
- lexbench_webfetch/models.py +93 -0
- lexbench_webfetch/ordered_tree.py +115 -0
- lexbench_webfetch/paths.py +41 -0
- lexbench_webfetch/reference_quality.py +1104 -0
- lexbench_webfetch/required_components.py +44 -0
- lexbench_webfetch/resource_urls.py +35 -0
- lexbench_webfetch/response_input.py +135 -0
- lexbench_webfetch/risk_control_dataset.py +512 -0
- lexbench_webfetch/scoring.py +452 -0
- lexbench_webfetch/scoring_regressions.py +215 -0
- lexbench_webfetch/stable_tasks.py +2144 -0
- lexbench_webfetch/structured_tables.py +793 -0
- lexbench_webfetch/versions.py +224 -0
- lexbench_websearch/__init__.py +5 -0
- lexbench_websearch/answer_rules.py +322 -0
- lexbench_websearch/answers.py +383 -0
- lexbench_websearch/cli.py +123 -0
- lexbench_websearch/contracts.py +58 -0
- lexbench_websearch/evidence_quality.py +109 -0
- lexbench_websearch/inputs.py +80 -0
- lexbench_websearch/outputs.py +161 -0
- lexbench_websearch/qualification.py +144 -0
- lexbench_websearch/relevance.py +346 -0
- lexbench_websearch/rubric.py +34 -0
- lexbench_websearch/scoring.py +354 -0
- lexbench_websearch/simpleqa.py +467 -0
- lexbench_websearch/versions.py +255 -0
- webstone-0.8.0.dist-info/METADATA +367 -0
- webstone-0.8.0.dist-info/RECORD +488 -0
- webstone-0.8.0.dist-info/WHEEL +5 -0
- webstone-0.8.0.dist-info/entry_points.txt +4 -0
- webstone-0.8.0.dist-info/licenses/LICENSE +202 -0
- webstone-0.8.0.dist-info/licenses/NOTICE +6 -0
- webstone-0.8.0.dist-info/licenses/THIRD_PARTY.md +22 -0
- webstone-0.8.0.dist-info/top_level.txt +3 -0
|
@@ -0,0 +1,353 @@
|
|
|
1
|
+
"""Compose independent record benchmarks through the maintained report owner."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from html import escape
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from bs4 import BeautifulSoup
|
|
10
|
+
|
|
11
|
+
from harness.web_extract import report as owner
|
|
12
|
+
from harness.web_extract.pipeline import digest, read_json, require_external_output, save_json
|
|
13
|
+
|
|
14
|
+
MODE = "field-extraction-suite"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def cohort_registry(entries):
|
|
18
|
+
"""Public scope authority: ordered cohorts and their exact child inputs."""
|
|
19
|
+
return [
|
|
20
|
+
{key: entry.get(key, "") for key in (
|
|
21
|
+
"id", "label", "summary_html", "content_sha256", "profile_sha256"
|
|
22
|
+
)}
|
|
23
|
+
for entry in entries
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def suite_inputs(profile):
|
|
28
|
+
"""Resolve private child inputs from the suite's hash-bound content."""
|
|
29
|
+
authority = read_json(profile)
|
|
30
|
+
binding = authority.get("research_report", {}).get("content", {})
|
|
31
|
+
if not binding.get("path"):
|
|
32
|
+
raise ValueError("Suite authority requires its bound content path")
|
|
33
|
+
content = profile.parent / binding["path"]
|
|
34
|
+
if content.is_symlink() or digest(content.read_bytes()) != binding.get("sha256"):
|
|
35
|
+
raise ValueError("Suite content differs from its bound input profile")
|
|
36
|
+
entries = read_json(content)["cohorts"]
|
|
37
|
+
if cohort_registry(entries) != authority.get("research_report", {}).get("cohorts"):
|
|
38
|
+
raise ValueError("Suite registry differs from its profile-authorized content")
|
|
39
|
+
for entry in entries:
|
|
40
|
+
for key in ("content", "profile"):
|
|
41
|
+
path = Path(entry[key])
|
|
42
|
+
require_external_output(path)
|
|
43
|
+
if path.is_symlink() or digest(path.read_bytes()) != entry.get(key + "_sha256"):
|
|
44
|
+
raise ValueError("Suite child input differs from its declared binding")
|
|
45
|
+
return content, entries
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def section(output, entry, language="zh"):
|
|
49
|
+
child = output / entry["id"]
|
|
50
|
+
manifest = read_json(child / "report-manifest.json")
|
|
51
|
+
metrics = read_json(child / "data/metrics.json")
|
|
52
|
+
methods = manifest["planned_methods"]
|
|
53
|
+
contracts = owner.selected_execution_contracts(
|
|
54
|
+
read_json(child / "public-evidence/sources.json").values()
|
|
55
|
+
)
|
|
56
|
+
latency_compared = (set(manifest["comparison_methods"])
|
|
57
|
+
if owner.latency_comparable(contracts, methods) else set())
|
|
58
|
+
def t(key, **values):
|
|
59
|
+
return owner.message(key, language, **values)
|
|
60
|
+
|
|
61
|
+
def score(method, family, field):
|
|
62
|
+
if method not in manifest["comparison_methods"]:
|
|
63
|
+
return None
|
|
64
|
+
scores = metrics[method]["scores"]
|
|
65
|
+
return scores[family][field] if scores is not None else None
|
|
66
|
+
|
|
67
|
+
quality, operation = [], []
|
|
68
|
+
for method in methods:
|
|
69
|
+
current = metrics[method]
|
|
70
|
+
latency = current["latency"]
|
|
71
|
+
quality.append(
|
|
72
|
+
[
|
|
73
|
+
escape(manifest["method_labels"][method])
|
|
74
|
+
+ (
|
|
75
|
+
"<br><small>" + escape(manifest["skipped_methods"][method]) + "</small>"
|
|
76
|
+
if method in manifest["skipped_methods"]
|
|
77
|
+
else ""
|
|
78
|
+
),
|
|
79
|
+
f"{current['executed']} / {current['planned']}",
|
|
80
|
+
owner.record_metric(score(method, "value_micro", "f1")),
|
|
81
|
+
owner.record_metric(score(method, "page_hit_macro", "f1")),
|
|
82
|
+
owner.record_metric(score(method, "exact_field_sets", "rate"), True),
|
|
83
|
+
f'<a href="{entry["id"]}/{language}/interpretation.html#{method}">'
|
|
84
|
+
+ t("suite_results") + "</a>",
|
|
85
|
+
]
|
|
86
|
+
)
|
|
87
|
+
operation.append(
|
|
88
|
+
[
|
|
89
|
+
escape(manifest["method_labels"][method]),
|
|
90
|
+
f"{latency['timed']} / {current['planned']}",
|
|
91
|
+
*[owner.record_metric(latency[key]) for key in ("mean", "p50", "p95")],
|
|
92
|
+
escape(owner.price_quote(manifest["prices"][method], language)),
|
|
93
|
+
f'<a href="{entry["id"]}/{language}/accounts.html">{t("suite_pricing")}</a>',
|
|
94
|
+
]
|
|
95
|
+
)
|
|
96
|
+
quality_table = owner.record_comparison(
|
|
97
|
+
t("suite_quality_headers"),
|
|
98
|
+
quality,
|
|
99
|
+
{
|
|
100
|
+
column: (key, "max", [score(m, family, field) for m in methods])
|
|
101
|
+
for column, key, family, field in [
|
|
102
|
+
(2, "micro-f1", "value_micro", "f1"),
|
|
103
|
+
(3, "macro-f1", "page_hit_macro", "f1"),
|
|
104
|
+
(4, "exact-rate", "exact_field_sets", "rate"),
|
|
105
|
+
]
|
|
106
|
+
},
|
|
107
|
+
)
|
|
108
|
+
time_table = owner.record_comparison(
|
|
109
|
+
t("suite_operation_headers"),
|
|
110
|
+
operation,
|
|
111
|
+
{
|
|
112
|
+
column: (key, "min" if latency_compared else "neutral", [
|
|
113
|
+
metrics[m]["latency"][key] if m in latency_compared else None for m in methods
|
|
114
|
+
])
|
|
115
|
+
for column, key in [(2, "mean"), (3, "p50"), (4, "p95")]
|
|
116
|
+
},
|
|
117
|
+
)
|
|
118
|
+
return (
|
|
119
|
+
f'<section data-cohort="{entry["id"]}" id="{entry["id"]}"><h2>{escape(entry["label"])} · '
|
|
120
|
+
+ t("suite_cases", count=manifest["case_count"]) + "</h2>"
|
|
121
|
+
+ entry.get("summary_html", "")
|
|
122
|
+
+ quality_table
|
|
123
|
+
+ "<h3>" + t("latency_and_pricing") + "</h3>"
|
|
124
|
+
+ time_table
|
|
125
|
+
+ f'<p><a href="{entry["id"]}/{language}/dataset.html">{t("suite_dataset")}</a> · '
|
|
126
|
+
f'<a href="{entry["id"]}/{language}/protocol.html">{t("suite_examples")}</a> · '
|
|
127
|
+
f'<a href="{entry["id"]}/{language}/results.html">{t("suite_case_results")}</a>'
|
|
128
|
+
"</p></section>"
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def build(output, content, *, profile, language=None):
|
|
133
|
+
require_external_output(output)
|
|
134
|
+
language = owner.resolve_report_language(output, language, supported=("zh",))
|
|
135
|
+
model, authority = read_json(content), read_json(profile)
|
|
136
|
+
if authority.get("research_report", {}).get("content", {}).get("sha256") != digest(
|
|
137
|
+
content.read_bytes()
|
|
138
|
+
):
|
|
139
|
+
raise ValueError("Suite content differs from its instance profile")
|
|
140
|
+
entries = model["cohorts"]
|
|
141
|
+
names = [entry["id"] for entry in entries]
|
|
142
|
+
if (
|
|
143
|
+
not names
|
|
144
|
+
or len(set(names)) != len(names)
|
|
145
|
+
or any(
|
|
146
|
+
name in {"zh", "en", "public-evidence", "assets"}
|
|
147
|
+
or not re.fullmatch(r"[a-z0-9]+(?:-[a-z0-9]+)*", name)
|
|
148
|
+
for name in names
|
|
149
|
+
)
|
|
150
|
+
):
|
|
151
|
+
raise ValueError("Suite cohorts need distinct safe directory names")
|
|
152
|
+
for entry in entries:
|
|
153
|
+
child_content = read_json(Path(entry["content"]))
|
|
154
|
+
if child_content.get("site_prefix") != entry["id"]:
|
|
155
|
+
raise ValueError("Each cohort must declare its suite directory")
|
|
156
|
+
for key in ("content", "profile"):
|
|
157
|
+
if entry.get(key + "_sha256") != digest(Path(entry[key]).read_bytes()):
|
|
158
|
+
raise ValueError("Suite child input differs from its declared binding")
|
|
159
|
+
if authority.get("research_report", {}).get("cohorts") != cohort_registry(entries):
|
|
160
|
+
raise ValueError("Suite registry differs from its profile-authorized content")
|
|
161
|
+
if owner.public_risk(profile.read_bytes()):
|
|
162
|
+
raise ValueError("Suite authority must contain only publication-safe configuration")
|
|
163
|
+
if output.exists() and any(output.iterdir()):
|
|
164
|
+
raise ValueError("Use an empty suite report directory")
|
|
165
|
+
output.mkdir(parents=True, exist_ok=True)
|
|
166
|
+
(output / "index.html").write_text(
|
|
167
|
+
'<!doctype html><html><head><meta charset="utf-8">'
|
|
168
|
+
'<meta http-equiv="refresh" content="0;url=zh/index.html"></head>'
|
|
169
|
+
'<body><a href="zh/index.html">Web Extract</a></body></html>',
|
|
170
|
+
encoding="utf-8",
|
|
171
|
+
)
|
|
172
|
+
bindings = []
|
|
173
|
+
for entry in entries:
|
|
174
|
+
owner.build(
|
|
175
|
+
output / entry["id"],
|
|
176
|
+
Path(entry["content"]),
|
|
177
|
+
home=Path(entry["home"]),
|
|
178
|
+
profile=Path(entry["profile"]),
|
|
179
|
+
language=language,
|
|
180
|
+
)
|
|
181
|
+
bindings.append(
|
|
182
|
+
{key: entry[key] for key in ("id", "label")}
|
|
183
|
+
| {
|
|
184
|
+
"summary_html": entry.get("summary_html", ""),
|
|
185
|
+
"manifest_sha256": digest(
|
|
186
|
+
(output / entry["id"] / "report-manifest.json").read_bytes()
|
|
187
|
+
),
|
|
188
|
+
}
|
|
189
|
+
)
|
|
190
|
+
for directory in ("zh", "en"):
|
|
191
|
+
(output / directory).mkdir()
|
|
192
|
+
(output / "style.css").write_text(owner.CSS, encoding="utf-8")
|
|
193
|
+
(output / "app.js").write_text(owner.JS, encoding="utf-8")
|
|
194
|
+
owner.copy_report_assets(output, content, model)
|
|
195
|
+
body = model.get("intro_html", "") + "".join(
|
|
196
|
+
section(output, entry, language) for entry in bindings
|
|
197
|
+
)
|
|
198
|
+
body += owner.report_metric_guide(language, records=True) + model.get("interpretation_html", "")
|
|
199
|
+
(output / "zh/index.html").write_text(
|
|
200
|
+
owner.page(
|
|
201
|
+
language, "index", model["title"], body, languages=(language,), state=MODE,
|
|
202
|
+
notice=owner.message("suite_notice", language),
|
|
203
|
+
navigation=[{"route": f'{e["id"]}/{language}/index', "label": e["label"]}
|
|
204
|
+
for e in bindings],
|
|
205
|
+
),
|
|
206
|
+
encoding="utf-8",
|
|
207
|
+
)
|
|
208
|
+
(output / "en/index.html").write_text(
|
|
209
|
+
owner.message("english_not_generated_notice", "en"), encoding="utf-8"
|
|
210
|
+
)
|
|
211
|
+
input_profile = owner.copy_input_profile(output, profile, preserve_bytes=True)
|
|
212
|
+
save_json(output / "suite.json", {"cohorts": bindings})
|
|
213
|
+
save_json(
|
|
214
|
+
output / "report-manifest.json",
|
|
215
|
+
{
|
|
216
|
+
"mode": MODE,
|
|
217
|
+
"template": owner.template_identity("web-extract"),
|
|
218
|
+
"template_version": owner.VERSION,
|
|
219
|
+
"webstone": owner.bind_build_version(output),
|
|
220
|
+
"renderer_sha256": digest(Path(owner.__file__).read_bytes()),
|
|
221
|
+
"composition_sha256": digest(Path(__file__).read_bytes()),
|
|
222
|
+
**owner.report_dependencies(),
|
|
223
|
+
"input_profile": input_profile,
|
|
224
|
+
"profile_sha256": digest(profile.read_bytes()),
|
|
225
|
+
"content_sha256": digest(content.read_bytes()),
|
|
226
|
+
"local_example": authority.get("local_example", False),
|
|
227
|
+
"dataset_kind": "local-example"
|
|
228
|
+
if authority.get("local_example", False)
|
|
229
|
+
else "published-upstream",
|
|
230
|
+
"languages": ["zh"],
|
|
231
|
+
"primary_language": "zh",
|
|
232
|
+
"cohorts": bindings,
|
|
233
|
+
"files": {
|
|
234
|
+
p.relative_to(output).as_posix(): digest(p.read_bytes())
|
|
235
|
+
for p in sorted(output.rglob("*"))
|
|
236
|
+
if p.is_file()
|
|
237
|
+
},
|
|
238
|
+
},
|
|
239
|
+
exclusive=True,
|
|
240
|
+
)
|
|
241
|
+
return _check_rendered_suite(output)
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def _check_rendered_suite(output, *, release=False, visual_receipt=None):
|
|
245
|
+
output = owner.open_report(output)
|
|
246
|
+
manifest = read_json(output / "report-manifest.json")
|
|
247
|
+
owner.check_report_identity(output, manifest, "web-extract")
|
|
248
|
+
owner.check_input_profile(output, manifest)
|
|
249
|
+
if (
|
|
250
|
+
manifest.get("mode") != MODE
|
|
251
|
+
or manifest.get("renderer_sha256") != digest(Path(owner.__file__).read_bytes())
|
|
252
|
+
or manifest.get("composition_sha256") != digest(Path(__file__).read_bytes())
|
|
253
|
+
):
|
|
254
|
+
raise ValueError("Suite renderer changed; rebuild required")
|
|
255
|
+
actual = {p.relative_to(output).as_posix() for p in output.rglob("*") if p.is_file()}
|
|
256
|
+
if actual != set(manifest["files"]) | {"report-manifest.json"}:
|
|
257
|
+
raise ValueError("Suite inventory changed")
|
|
258
|
+
for name, sha in manifest["files"].items():
|
|
259
|
+
path = output / name
|
|
260
|
+
if (
|
|
261
|
+
path.is_symlink()
|
|
262
|
+
or digest(path.read_bytes()) != sha
|
|
263
|
+
or owner.public_risk(path.read_bytes())
|
|
264
|
+
):
|
|
265
|
+
raise ValueError("Suite file changed or has a privacy risk: " + name)
|
|
266
|
+
entries = read_json(output / "suite.json")["cohorts"]
|
|
267
|
+
if entries != manifest["cohorts"]:
|
|
268
|
+
raise ValueError("Suite cohort bindings changed")
|
|
269
|
+
authority = read_json(output / manifest["input_profile"]["path"])
|
|
270
|
+
if (
|
|
271
|
+
digest((output / manifest["input_profile"]["path"]).read_bytes())
|
|
272
|
+
!= manifest.get("profile_sha256")
|
|
273
|
+
or authority.get("research_report", {}).get("content", {}).get("sha256")
|
|
274
|
+
!= manifest.get("content_sha256")
|
|
275
|
+
):
|
|
276
|
+
raise ValueError("Suite content differs from its bound input profile")
|
|
277
|
+
declared = authority.get("research_report", {}).get("cohorts")
|
|
278
|
+
if [{key: e.get(key, "") for key in ("id", "label", "summary_html")} for e in entries] != [
|
|
279
|
+
{key: e.get(key, "") for key in ("id", "label", "summary_html")} for e in declared or []
|
|
280
|
+
]:
|
|
281
|
+
raise ValueError("Suite registry differs from its profile-authorized content")
|
|
282
|
+
names = [entry["id"] for entry in entries]
|
|
283
|
+
if (
|
|
284
|
+
not names
|
|
285
|
+
or len(names) != len(set(names))
|
|
286
|
+
or any(
|
|
287
|
+
name in {"zh", "en", "public-evidence", "assets"}
|
|
288
|
+
or not re.fullmatch(r"[a-z0-9]+(?:-[a-z0-9]+)*", name)
|
|
289
|
+
for name in names
|
|
290
|
+
)
|
|
291
|
+
):
|
|
292
|
+
raise ValueError("Suite cohort directories changed")
|
|
293
|
+
visual = read_json(visual_receipt) if release and visual_receipt else {}
|
|
294
|
+
for entry, source in zip(entries, declared, strict=True):
|
|
295
|
+
child = output / entry["id"]
|
|
296
|
+
child_manifest = read_json(child / "report-manifest.json")
|
|
297
|
+
if any(child_manifest.get(key + "_sha256") != source.get(key + "_sha256")
|
|
298
|
+
for key in ("content", "profile")):
|
|
299
|
+
raise ValueError("Suite child input binding changed")
|
|
300
|
+
if digest((child / "report-manifest.json").read_bytes()) != entry["manifest_sha256"]:
|
|
301
|
+
raise ValueError("Suite cohort manifest changed")
|
|
302
|
+
receipt = visual.get("cohorts", {}).get(entry["id"], {})
|
|
303
|
+
if release and (
|
|
304
|
+
not receipt.get("path")
|
|
305
|
+
or digest(Path(receipt["path"]).read_bytes()) != receipt.get("sha256")
|
|
306
|
+
):
|
|
307
|
+
raise ValueError("Suite requires each cohort's bound visual receipt")
|
|
308
|
+
owner._check_record_cohort(
|
|
309
|
+
child, release=release, visual_receipt=Path(receipt["path"]) if release else None,
|
|
310
|
+
)
|
|
311
|
+
from harness.reporting.presentation import check_metric_guide, check_presentation
|
|
312
|
+
|
|
313
|
+
soup = check_presentation((output / "zh/index.html").read_text(encoding="utf-8"))
|
|
314
|
+
expected = BeautifulSoup(owner.page(
|
|
315
|
+
"zh", "index", "", "".join(section(output, entry) for entry in entries),
|
|
316
|
+
languages=("zh",), state=MODE,
|
|
317
|
+
navigation=[{"route": f'{e["id"]}/zh/index', "label": e["label"]} for e in entries],
|
|
318
|
+
), "html.parser")
|
|
319
|
+
if [str(node) for node in soup.select("[data-cohort]")] != [
|
|
320
|
+
str(node) for node in expected.select("[data-cohort]")
|
|
321
|
+
]:
|
|
322
|
+
raise ValueError("Suite comparison differs from its independent cohort results")
|
|
323
|
+
check_metric_guide(
|
|
324
|
+
str(soup),
|
|
325
|
+
owner.metric_definitions("zh", records=True),
|
|
326
|
+
title=owner.message("guide_title", "zh"),
|
|
327
|
+
headings=owner.message("guide_headings", "zh"),
|
|
328
|
+
)
|
|
329
|
+
if [(a.get("href"), a.get_text()) for a in soup.select("nav a")] != [
|
|
330
|
+
(entry["id"] + "/zh/index.html", entry["label"]) for entry in entries
|
|
331
|
+
]:
|
|
332
|
+
raise ValueError("Suite navigation differs from its benchmark registry")
|
|
333
|
+
notice = BeautifulSoup((output / "en/index.html").read_text(encoding="utf-8"), "html.parser")
|
|
334
|
+
if {p.name for p in (output / "en").glob("*.html")} != {"index.html"} or not notice.select_one(
|
|
335
|
+
'[data-language-notice] a[href="zh/index.html"]'
|
|
336
|
+
):
|
|
337
|
+
raise ValueError("Suite English notice is missing or stale")
|
|
338
|
+
links = owner.LinkIndex(errors="strict")
|
|
339
|
+
for path in [output / "zh/index.html", output / "en/index.html"]:
|
|
340
|
+
for value in links.page(path).values:
|
|
341
|
+
parsed = owner.urlsplit(value)
|
|
342
|
+
if parsed.scheme or parsed.netloc:
|
|
343
|
+
continue
|
|
344
|
+
target = output / owner.unquote(parsed.path) if parsed.path else path
|
|
345
|
+
if not target.resolve().is_relative_to(output) or not target.is_file():
|
|
346
|
+
raise ValueError("Broken suite navigation: " + value)
|
|
347
|
+
if parsed.fragment and owner.unquote(parsed.fragment) not in links.page(target).ids:
|
|
348
|
+
raise ValueError("Broken suite anchor: " + value)
|
|
349
|
+
if release:
|
|
350
|
+
if manifest.get("local_example"):
|
|
351
|
+
raise ValueError("Synthetic suites cannot be published as measured results")
|
|
352
|
+
owner.check_visual_receipt(output, manifest, visual_receipt, routes=("index",))
|
|
353
|
+
return manifest
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""Static report controls for field, method, source and diagnosis drilldown."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
from harness.web_extract.messages import message
|
|
7
|
+
|
|
8
|
+
ASSETS = Path(__file__).with_name("assets")
|
|
9
|
+
JS = (ASSETS / "report.js").read_text(encoding="utf-8")
|
|
10
|
+
|
|
11
|
+
CLIENT_MESSAGE_KEYS = (
|
|
12
|
+
"records_qualified_reference",
|
|
13
|
+
"showing_results",
|
|
14
|
+
"audited_reference",
|
|
15
|
+
"stages_client_transport",
|
|
16
|
+
"compare_all_methods_a24f79dd",
|
|
17
|
+
"complete_request_response_and_repair_history",
|
|
18
|
+
"current_call_latency",
|
|
19
|
+
"exact_error",
|
|
20
|
+
"failure_repair",
|
|
21
|
+
"field",
|
|
22
|
+
"field_labels_availability",
|
|
23
|
+
"field_labels_OutOfStock",
|
|
24
|
+
"field_labels_price",
|
|
25
|
+
"field_labels_sku",
|
|
26
|
+
"full_result",
|
|
27
|
+
"load_failed_close_and_retry_or_use_the_raw_file",
|
|
28
|
+
"loading_complete_evidence",
|
|
29
|
+
"no_accepted_value",
|
|
30
|
+
"no_recognizable_product_fields_returned",
|
|
31
|
+
"null",
|
|
32
|
+
"original_call_and_score_retained",
|
|
33
|
+
"outcome_243a87d2",
|
|
34
|
+
"outcomes_correct",
|
|
35
|
+
"outcomes_missing",
|
|
36
|
+
"outcomes_unexpected",
|
|
37
|
+
"outcomes_wrong",
|
|
38
|
+
"outcomes_partial",
|
|
39
|
+
"outcomes_invalid",
|
|
40
|
+
"outcomes_unavailable",
|
|
41
|
+
"output",
|
|
42
|
+
"stages_provider_schema",
|
|
43
|
+
"reference",
|
|
44
|
+
"reviewed_cause",
|
|
45
|
+
"see_field_judgments_and_complete_evidence_below",
|
|
46
|
+
"source_and_annotations_cf191383",
|
|
47
|
+
"source_evidence_and_audited_reference",
|
|
48
|
+
"stage_fb689e6a",
|
|
49
|
+
"stages_fields",
|
|
50
|
+
"stages_model_capacity",
|
|
51
|
+
"stages_navigation",
|
|
52
|
+
"stages_provider_fetch",
|
|
53
|
+
"stages_provider_response",
|
|
54
|
+
"structured_fields_returned_inspect_field_differences",
|
|
55
|
+
"treatment",
|
|
56
|
+
)
|
|
57
|
+
JS = JS.replace(
|
|
58
|
+
"__REPORT_MESSAGES__",
|
|
59
|
+
json.dumps(
|
|
60
|
+
{
|
|
61
|
+
language: {key: message(key, language, shown="{shown}") for key in CLIENT_MESSAGE_KEYS}
|
|
62
|
+
for language in ("en", "zh")
|
|
63
|
+
},
|
|
64
|
+
ensure_ascii=False,
|
|
65
|
+
),
|
|
66
|
+
)
|