agentinputlab 0.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentinputlab/__init__.py +28 -0
- agentinputlab/algorithms/__init__.py +6 -0
- agentinputlab/algorithms/acquire/__init__.py +145 -0
- agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
- agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
- agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
- agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
- agentinputlab/algorithms/acquire/hyde.py +296 -0
- agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
- agentinputlab/algorithms/acquire/hyde_external.py +148 -0
- agentinputlab/algorithms/acquire/ircot.py +28 -0
- agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
- agentinputlab/algorithms/acquire/ircot_native.py +355 -0
- agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
- agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
- agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
- agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
- agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
- agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
- agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
- agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
- agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
- agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
- agentinputlab/algorithms/clarify/__init__.py +61 -0
- agentinputlab/algorithms/clarify/_structured.py +64 -0
- agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
- agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
- agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
- agentinputlab/algorithms/clarify/future_turns.py +379 -0
- agentinputlab/algorithms/clarify/udcs.py +255 -0
- agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
- agentinputlab/algorithms/curate/__init__.py +65 -0
- agentinputlab/algorithms/curate/acon/__init__.py +27 -0
- agentinputlab/algorithms/curate/acon/common.py +284 -0
- agentinputlab/algorithms/curate/acon/common_support.py +136 -0
- agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
- agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
- agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
- agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
- agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
- agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
- agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
- agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
- agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
- agentinputlab/algorithms/curate/recomp/external.py +1129 -0
- agentinputlab/algorithms/curate/recomp/selective.py +304 -0
- agentinputlab/algorithms/decompose/__init__.py +107 -0
- agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
- agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
- agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
- agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
- agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
- agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
- agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
- agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
- agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
- agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
- agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
- agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
- agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
- agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
- agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
- agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
- agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
- agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
- agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
- agentinputlab/algorithms/reformulate/__init__.py +85 -0
- agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
- agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
- agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
- agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
- agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
- agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
- agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
- agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
- agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
- agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
- agentinputlab/algorithms/reformulate/rrr.py +696 -0
- agentinputlab/annotation/__init__.py +5 -0
- agentinputlab/annotation/independent.py +428 -0
- agentinputlab/annotation/server.py +233 -0
- agentinputlab/annotation/static/app.js +121 -0
- agentinputlab/annotation/static/index.html +60 -0
- agentinputlab/annotation/static/review.css +2 -0
- agentinputlab/annotation/static/review.html +10 -0
- agentinputlab/annotation/static/review.js +11 -0
- agentinputlab/annotation/static/styles.css +55 -0
- agentinputlab/annotation/store.py +444 -0
- agentinputlab/api.py +368 -0
- agentinputlab/artifacts/__init__.py +15 -0
- agentinputlab/artifacts/json_file.py +104 -0
- agentinputlab/artifacts/manifest.py +98 -0
- agentinputlab/artifacts/store.py +157 -0
- agentinputlab/backends/__init__.py +98 -0
- agentinputlab/backends/embedding/__init__.py +25 -0
- agentinputlab/backends/embedding/base.py +145 -0
- agentinputlab/backends/embedding/callable.py +75 -0
- agentinputlab/backends/embedding/replay.py +129 -0
- agentinputlab/backends/llm/__init__.py +31 -0
- agentinputlab/backends/llm/base.py +149 -0
- agentinputlab/backends/llm/callable.py +108 -0
- agentinputlab/backends/llm/decomp_completion.py +88 -0
- agentinputlab/backends/llm/final_content_stop.py +72 -0
- agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
- agentinputlab/backends/llm/openai_compatible.py +332 -0
- agentinputlab/backends/llm/transformers.py +171 -0
- agentinputlab/backends/retrieval/__init__.py +35 -0
- agentinputlab/backends/retrieval/base.py +50 -0
- agentinputlab/backends/retrieval/dense.py +777 -0
- agentinputlab/backends/retrieval/memory.py +119 -0
- agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
- agentinputlab/backends/user/__init__.py +15 -0
- agentinputlab/backends/user/base.py +66 -0
- agentinputlab/backends/user/callable.py +54 -0
- agentinputlab/backends/user/replay.py +89 -0
- agentinputlab/backends/vector.py +38 -0
- agentinputlab/benchmarks/__init__.py +246 -0
- agentinputlab/benchmarks/acquisition.py +106 -0
- agentinputlab/benchmarks/ale.py +540 -0
- agentinputlab/benchmarks/appworld_ul.py +669 -0
- agentinputlab/benchmarks/assets.py +226 -0
- agentinputlab/benchmarks/base.py +68 -0
- agentinputlab/benchmarks/beir_archive.py +235 -0
- agentinputlab/benchmarks/beir_sample.py +269 -0
- agentinputlab/benchmarks/bfcl_native.py +245 -0
- agentinputlab/benchmarks/bfcl_v3.py +607 -0
- agentinputlab/benchmarks/cache.py +218 -0
- agentinputlab/benchmarks/clariq.py +203 -0
- agentinputlab/benchmarks/data/registry.json +321 -0
- agentinputlab/benchmarks/data/smoke_assets.json +511 -0
- agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
- agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
- agentinputlab/benchmarks/execution.py +1111 -0
- agentinputlab/benchmarks/interactive.py +257 -0
- agentinputlab/benchmarks/ir_query.py +204 -0
- agentinputlab/benchmarks/longbench.py +203 -0
- agentinputlab/benchmarks/paired_replay.py +479 -0
- agentinputlab/benchmarks/preprocess.py +2313 -0
- agentinputlab/benchmarks/production.py +236 -0
- agentinputlab/benchmarks/qa.py +963 -0
- agentinputlab/benchmarks/registry.py +110 -0
- agentinputlab/benchmarks/retrieval.py +459 -0
- agentinputlab/benchmarks/runtime.py +509 -0
- agentinputlab/benchmarks/statebench.py +938 -0
- agentinputlab/benchmarks/tau3.py +725 -0
- agentinputlab/benchmarks/toolsandbox.py +626 -0
- agentinputlab/benchmarks/trec_dl_sample.py +263 -0
- agentinputlab/campaign.py +654 -0
- agentinputlab/campaign_runner.py +953 -0
- agentinputlab/cli/__init__.py +6 -0
- agentinputlab/cli/main.py +933 -0
- agentinputlab/connectors/__init__.py +46 -0
- agentinputlab/connectors/knowledge/__init__.py +4 -0
- agentinputlab/connectors/tools/__init__.py +30 -0
- agentinputlab/connectors/tools/base.py +57 -0
- agentinputlab/connectors/tools/mcp.py +222 -0
- agentinputlab/connectors/tools/memory.py +41 -0
- agentinputlab/connectors/tools/openapi.py +695 -0
- agentinputlab/connectors/tools/schema.py +102 -0
- agentinputlab/contracts/__init__.py +91 -0
- agentinputlab/contracts/availability.py +24 -0
- agentinputlab/contracts/dependency.py +58 -0
- agentinputlab/contracts/enums.py +82 -0
- agentinputlab/contracts/errors.py +63 -0
- agentinputlab/contracts/evidence.py +43 -0
- agentinputlab/contracts/intent.py +110 -0
- agentinputlab/contracts/lifecycle.py +164 -0
- agentinputlab/contracts/patch.py +167 -0
- agentinputlab/contracts/serialization.py +93 -0
- agentinputlab/contracts/state.py +127 -0
- agentinputlab/contracts/trace.py +59 -0
- agentinputlab/contracts/worker.py +42 -0
- agentinputlab/core/__init__.py +7 -0
- agentinputlab/data/__init__.py +32 -0
- agentinputlab/data/checksums.py +9 -0
- agentinputlab/data/downloader.py +235 -0
- agentinputlab/data/http.py +171 -0
- agentinputlab/data/index.py +21 -0
- agentinputlab/data/overlap.py +28 -0
- agentinputlab/data/preprocess.py +32 -0
- agentinputlab/data/storage.py +57 -0
- agentinputlab/data/trec_sample.py +427 -0
- agentinputlab/evaluation/__init__.py +5 -0
- agentinputlab/evaluation/outcomes.py +32 -0
- agentinputlab/experiments/__init__.py +278 -0
- agentinputlab/experiments/analysis.py +981 -0
- agentinputlab/experiments/artifacts.py +687 -0
- agentinputlab/experiments/automatic_measurement.py +678 -0
- agentinputlab/experiments/auxiliary_matrix.py +203 -0
- agentinputlab/experiments/bindings.py +430 -0
- agentinputlab/experiments/blueprints.py +147 -0
- agentinputlab/experiments/builders/__init__.py +45 -0
- agentinputlab/experiments/builders/executable_external_payload.py +423 -0
- agentinputlab/experiments/builders/external_process_arena.py +1018 -0
- agentinputlab/experiments/builders/registry_intervention.py +463 -0
- agentinputlab/experiments/cli.py +158 -0
- agentinputlab/experiments/controls.py +340 -0
- agentinputlab/experiments/driver_guard.py +205 -0
- agentinputlab/experiments/drivers/__init__.py +89 -0
- agentinputlab/experiments/drivers/external_arena.py +2908 -0
- agentinputlab/experiments/drivers/method_process.py +662 -0
- agentinputlab/experiments/drivers/parity_process.py +1484 -0
- agentinputlab/experiments/drivers/static_production.py +1574 -0
- agentinputlab/experiments/executor.py +448 -0
- agentinputlab/experiments/loader.py +605 -0
- agentinputlab/experiments/measurement.py +675 -0
- agentinputlab/experiments/paper_analysis.py +1818 -0
- agentinputlab/experiments/parity_identity.py +605 -0
- agentinputlab/experiments/plan.py +277 -0
- agentinputlab/experiments/production_builders.py +3739 -0
- agentinputlab/experiments/readiness.py +432 -0
- agentinputlab/experiments/schema.py +514 -0
- agentinputlab/experiments/secrets.py +177 -0
- agentinputlab/experiments/stateful_analysis.py +1738 -0
- agentinputlab/experiments/transfer_analysis.py +1316 -0
- agentinputlab/experiments/unseen_analysis.py +1025 -0
- agentinputlab/integrations/__init__.py +1 -0
- agentinputlab/integrations/action_space/__init__.py +1 -0
- agentinputlab/integrations/external_workers/__init__.py +76 -0
- agentinputlab/integrations/external_workers/ace.py +431 -0
- agentinputlab/integrations/external_workers/base.py +181 -0
- agentinputlab/integrations/external_workers/decomp_live.py +249 -0
- agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
- agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
- agentinputlab/integrations/external_workers/ircot.py +576 -0
- agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
- agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
- agentinputlab/integrations/external_workers/method.py +314 -0
- agentinputlab/integrations/external_workers/process.py +678 -0
- agentinputlab/integrations/external_workers/supervisor.py +1071 -0
- agentinputlab/integrations/full_architecture/__init__.py +1 -0
- agentinputlab/integrations/full_architecture/hyde.py +212 -0
- agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
- agentinputlab/integrations/live_history.py +222 -0
- agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
- agentinputlab/integrations/training/__init__.py +1 -0
- agentinputlab/metrics/__init__.py +50 -0
- agentinputlab/metrics/catalog.py +313 -0
- agentinputlab/metrics/harm.py +95 -0
- agentinputlab/metrics/paired.py +40 -0
- agentinputlab/metrics/qa.py +76 -0
- agentinputlab/metrics/text.py +33 -0
- agentinputlab/metrics/third_party.py +161 -0
- agentinputlab/models/__init__.py +4 -0
- agentinputlab/observability/__init__.py +56 -0
- agentinputlab/observability/budget.py +316 -0
- agentinputlab/observability/instrumentation.py +754 -0
- agentinputlab/observability/models.py +403 -0
- agentinputlab/observability/monitor.py +380 -0
- agentinputlab/optimizers/__init__.py +1 -0
- agentinputlab/optimizers/clarify/__init__.py +12 -0
- agentinputlab/optimizers/context/__init__.py +22 -0
- agentinputlab/optimizers/decompose/__init__.py +64 -0
- agentinputlab/optimizers/evidence/__init__.py +22 -0
- agentinputlab/optimizers/rewrite/__init__.py +30 -0
- agentinputlab/parity/__init__.py +5 -0
- agentinputlab/parity/base.py +206 -0
- agentinputlab/production_campaign.py +763 -0
- agentinputlab/py.typed +1 -0
- agentinputlab/recipes/__init__.py +333 -0
- agentinputlab/recipes/ace.py +418 -0
- agentinputlab/recipes/ace_bfcl.py +273 -0
- agentinputlab/recipes/ace_common_support.py +169 -0
- agentinputlab/recipes/ace_native_plan.py +322 -0
- agentinputlab/recipes/ace_online.py +711 -0
- agentinputlab/recipes/ace_retrieval.py +330 -0
- agentinputlab/recipes/ace_statebench.py +382 -0
- agentinputlab/recipes/ace_tau.py +463 -0
- agentinputlab/recipes/ace_toolsandbox.py +432 -0
- agentinputlab/recipes/acon_guideline.py +22 -0
- agentinputlab/recipes/adaptive_rag.py +822 -0
- agentinputlab/recipes/llmlingua2_inference.py +146 -0
- agentinputlab/recipes/llmlingua2_training.py +796 -0
- agentinputlab/recipes/mce.py +1486 -0
- agentinputlab/recipes/mce_final.py +335 -0
- agentinputlab/recipes/mce_text_fit.py +499 -0
- agentinputlab/recipes/mce_text_only.py +298 -0
- agentinputlab/recipes/qrag.py +729 -0
- agentinputlab/recipes/qrag_inference.py +364 -0
- agentinputlab/recipes/r2ag_features.py +236 -0
- agentinputlab/recipes/r2ag_inference.py +503 -0
- agentinputlab/recipes/r2ag_training.py +948 -0
- agentinputlab/recipes/recomp_abstractive.py +592 -0
- agentinputlab/recipes/recomp_extractive.py +525 -0
- agentinputlab/recipes/reformer.py +17 -0
- agentinputlab/recipes/rqrag.py +1516 -0
- agentinputlab/recipes/rqrag_common_support.py +193 -0
- agentinputlab/recipes/rqrag_completion.py +147 -0
- agentinputlab/recipes/skillopt.py +462 -0
- agentinputlab/recipes/skillopt_common_support.py +154 -0
- agentinputlab/recipes/skillopt_environment.py +165 -0
- agentinputlab/recipes/skillopt_environment_cli.py +5 -0
- agentinputlab/recipes/skillopt_retrieval.py +188 -0
- agentinputlab/recipes/smartsearch.py +897 -0
- agentinputlab/recipes/smartsearch_curriculum.py +283 -0
- agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
- agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
- agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
- agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
- agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
- agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
- agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
- agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
- agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
- agentinputlab/recipes/smartsearch_inference.py +524 -0
- agentinputlab/recipes/smartsearch_native_data.py +172 -0
- agentinputlab/recipes/smartsearch_policy.py +616 -0
- agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
- agentinputlab/recipes/smartsearch_sampling.py +232 -0
- agentinputlab/recipes/smartsearch_teacher.py +203 -0
- agentinputlab/recipes/smartsearch_training.py +265 -0
- agentinputlab/registry/__init__.py +46 -0
- agentinputlab/registry/beta.py +727 -0
- agentinputlab/registry/data/algorithms.json +1959 -0
- agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
- agentinputlab/registry/loader.py +100 -0
- agentinputlab/registry/result_manifest.py +184 -0
- agentinputlab/registry/runtime.py +195 -0
- agentinputlab/registry/schema.py +449 -0
- agentinputlab/runner/__init__.py +30 -0
- agentinputlab/runner/balance.py +248 -0
- agentinputlab/runner/cache.py +135 -0
- agentinputlab/runner/composite.py +130 -0
- agentinputlab/runner/engine.py +285 -0
- agentinputlab/runner/runtime_guard.py +268 -0
- agentinputlab/runner/worker.py +80 -0
- agentinputlab/runtime_bootstrap.py +288 -0
- agentinputlab/selectors/__init__.py +196 -0
- agentinputlab/selectors/advanced.py +1514 -0
- agentinputlab/selectors/artifacts.py +1087 -0
- agentinputlab/selectors/base.py +112 -0
- agentinputlab/selectors/campaigns.py +1893 -0
- agentinputlab/selectors/catalog.py +86 -0
- agentinputlab/selectors/discovery.py +1860 -0
- agentinputlab/selectors/evaluation.py +1348 -0
- agentinputlab/selectors/features.py +383 -0
- agentinputlab/selectors/learned.py +502 -0
- agentinputlab/selectors/no_op.py +28 -0
- agentinputlab/selectors/outcome_table.py +475 -0
- agentinputlab/selectors/producer.py +853 -0
- agentinputlab/selectors/rules.py +109 -0
- agentinputlab/selectors/runtime.py +420 -0
- agentinputlab/studies/__init__.py +16 -0
- agentinputlab/studies/design.py +165 -0
- agentinputlab/studies/paired.py +322 -0
- agentinputlab/studio/__init__.py +6 -0
- agentinputlab/studio/campaigns.py +819 -0
- agentinputlab/studio/engine.py +1408 -0
- agentinputlab/studio/model_backends.py +425 -0
- agentinputlab/studio/prepared.py +410 -0
- agentinputlab/studio/server.py +613 -0
- agentinputlab/studio/static/app.js +397 -0
- agentinputlab/studio/static/campaigns.html +175 -0
- agentinputlab/studio/static/campaigns.js +426 -0
- agentinputlab/studio/static/index.html +192 -0
- agentinputlab/studio/static/result.html +103 -0
- agentinputlab/studio/static/result.js +273 -0
- agentinputlab/studio/static/styles.css +467 -0
- agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
- agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
- agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
- agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
- agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
- agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
- agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
- agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
- agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
- workers/README.md +94 -0
- workers/ace/README.md +261 -0
- workers/ace/__init__.py +1 -0
- workers/ace/compatibility_manifest.json +39 -0
- workers/ace/driver.py +1682 -0
- workers/acon/README.md +215 -0
- workers/acon/__init__.py +5 -0
- workers/acon/compatibility_manifest.json +93 -0
- workers/acon/driver.py +1859 -0
- workers/acon/fitted.py +395 -0
- workers/acon/guideline.py +470 -0
- workers/acon/guideline_source_manifest.json +20 -0
- workers/adaptive_rag/README.md +96 -0
- workers/adaptive_rag/__init__.py +15 -0
- workers/adaptive_rag/compatibility_manifest.json +61 -0
- workers/adaptive_rag/driver.py +1470 -0
- workers/adaptive_rag/fixture_runtime.py +163 -0
- workers/adaptive_rag/runtime.py +252 -0
- workers/ask_or_assume/README.md +124 -0
- workers/ask_or_assume/__init__.py +1 -0
- workers/ask_or_assume/compatibility_manifest.json +36 -0
- workers/ask_or_assume/driver.py +1227 -0
- workers/decomp/README.md +133 -0
- workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
- workers/decomp/__init__.py +5 -0
- workers/decomp/compatibility_manifest.json +205 -0
- workers/decomp/driver.py +1015 -0
- workers/decomp/live.py +137 -0
- workers/decomp/runtime.py +231 -0
- workers/infocqr/README.md +106 -0
- workers/infocqr/__init__.py +1 -0
- workers/infocqr/compatibility_manifest.json +84 -0
- workers/infocqr/driver.py +1766 -0
- workers/ircot/README.md +167 -0
- workers/ircot/__init__.py +7 -0
- workers/ircot/compatibility_manifest.json +121 -0
- workers/ircot/controlled.py +457 -0
- workers/ircot/driver.py +1373 -0
- workers/ircot/fixture_bindings.json +39 -0
- workers/ircot/fixture_config.json +48 -0
- workers/ircot/fixture_prompt.txt +2 -0
- workers/ircot/native_qa.py +398 -0
- workers/ircot/qa_compatibility_manifest.json +57 -0
- workers/ircot/runtime.py +97 -0
- workers/llmcompiler/README.md +160 -0
- workers/llmcompiler/__init__.py +1 -0
- workers/llmcompiler/compatibility_manifest.json +45 -0
- workers/llmcompiler/driver.py +1514 -0
- workers/llmcompiler/export_prompts.py +74 -0
- workers/llmcompiler/live.py +174 -0
- workers/llmcompiler/patch_manifest.json +147 -0
- workers/mce/README.md +198 -0
- workers/mce/__init__.py +29 -0
- workers/mce/compatibility_manifest.json +115 -0
- workers/mce/driver.py +2585 -0
- workers/mce/final_eval.py +173 -0
- workers/official_arenas/README.md +25 -0
- workers/official_arenas/__init__.py +1 -0
- workers/official_arenas/capture_records.py +110 -0
- workers/official_arenas/driver.py +619 -0
- workers/official_arenas/protocol.py +334 -0
- workers/official_arenas/providers/__init__.py +1 -0
- workers/official_arenas/providers/_live_history.py +228 -0
- workers/official_arenas/providers/_message_compat.py +77 -0
- workers/official_arenas/providers/appworld_ul.py +196 -0
- workers/official_arenas/providers/fixture.py +112 -0
- workers/official_arenas/providers/statebench.py +960 -0
- workers/official_arenas/providers/tau3.py +1672 -0
- workers/official_arenas/providers/tau_retail.py +586 -0
- workers/official_arenas/providers/toolsandbox.py +1321 -0
- workers/official_arenas/providers/tooltalk.py +699 -0
- workers/official_arenas/retail_capture.py +188 -0
- workers/official_arenas/retail_driver.py +275 -0
- workers/qrag/README.md +172 -0
- workers/qrag/__init__.py +1 -0
- workers/qrag/compatibility_manifest.json +64 -0
- workers/qrag/driver.py +673 -0
- workers/r2ag/README.md +154 -0
- workers/r2ag/__init__.py +5 -0
- workers/r2ag/compatibility_manifest.json +55 -0
- workers/r2ag/driver.py +1287 -0
- workers/r2ag/feature_schema.json +8 -0
- workers/r2ag/runtime.py +589 -0
- workers/recomp/README.md +120 -0
- workers/recomp/__init__.py +1 -0
- workers/recomp/compatibility_manifest.json +43 -0
- workers/recomp/driver.py +1653 -0
- workers/reformer/README.md +125 -0
- workers/reformer/__init__.py +1 -0
- workers/reformer/compatibility_manifest.json +68 -0
- workers/reformer/driver.py +1459 -0
- workers/reformer/induction.py +457 -0
- workers/rrr/README.md +126 -0
- workers/rrr/__init__.py +1 -0
- workers/rrr/compatibility_manifest.json +57 -0
- workers/rrr/driver.py +1133 -0
- workers/smartsearch/README.md +331 -0
- workers/smartsearch/__init__.py +1 -0
- workers/smartsearch/compatibility_manifest.json +41 -0
- workers/smartsearch/driver.py +1643 -0
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
<!doctype html>
|
|
2
|
+
<html lang="en">
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="utf-8">
|
|
5
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
6
|
+
<meta name="color-scheme" content="dark">
|
|
7
|
+
<title>Experiment result · AgentInputLab Studio</title>
|
|
8
|
+
<link rel="stylesheet" href="/styles.css">
|
|
9
|
+
<script src="/result.js" defer></script>
|
|
10
|
+
</head>
|
|
11
|
+
<body>
|
|
12
|
+
<div class="app-shell">
|
|
13
|
+
<aside class="sidebar">
|
|
14
|
+
<a class="brand" href="/">
|
|
15
|
+
<span class="brand-mark" aria-hidden="true">AI</span>
|
|
16
|
+
<span><strong>AgentInputLab</strong><small>Experiment Studio</small></span>
|
|
17
|
+
</a>
|
|
18
|
+
<nav class="primary-nav" aria-label="Studio navigation">
|
|
19
|
+
<a class="nav-link" href="/#configure"><span>01</span>Configure</a>
|
|
20
|
+
<a class="nav-link" href="/campaigns"><span>02</span>Campaigns</a>
|
|
21
|
+
<a class="nav-link" href="/#runs"><span>03</span>Run queue</a>
|
|
22
|
+
<a class="nav-link active" href="/result"><span>04</span>Results</a>
|
|
23
|
+
</nav>
|
|
24
|
+
<div class="sidebar-spacer"></div>
|
|
25
|
+
<div class="local-policy">
|
|
26
|
+
<div class="policy-heading"><span class="status-dot"></span>Measured locally</div>
|
|
27
|
+
<p>Provider billing tokens are never inferred. Replay results report a clearly labeled word-token estimate.</p>
|
|
28
|
+
</div>
|
|
29
|
+
<div class="build-label">STUDIO / ALPHA 01</div>
|
|
30
|
+
</aside>
|
|
31
|
+
|
|
32
|
+
<main class="main-canvas result-canvas">
|
|
33
|
+
<header class="result-header entrance">
|
|
34
|
+
<div>
|
|
35
|
+
<a class="back-link" href="/#runs">← Back to queue</a>
|
|
36
|
+
<p class="eyebrow">EXPERIMENT RESULT</p>
|
|
37
|
+
<h1 id="result-title">Select a run</h1>
|
|
38
|
+
<p class="result-subtitle" id="result-subtitle">Open a result from the queue to inspect its measurements and trace.</p>
|
|
39
|
+
</div>
|
|
40
|
+
<span class="run-status waiting" id="run-status">WAITING</span>
|
|
41
|
+
</header>
|
|
42
|
+
|
|
43
|
+
<div id="result-empty" class="result-empty entrance entrance-delay-1">
|
|
44
|
+
<span class="empty-glyph">↳</span>
|
|
45
|
+
<h2>No run selected</h2>
|
|
46
|
+
<p>Queue an experiment, then open its result to see quality, usage, timing, failures, and the full algorithm trace.</p>
|
|
47
|
+
<a class="primary-action inline-action" href="/">Configure a run <b>↗</b></a>
|
|
48
|
+
</div>
|
|
49
|
+
|
|
50
|
+
<div id="result-content" hidden>
|
|
51
|
+
<section class="metric-strip entrance entrance-delay-1" aria-label="Result summary">
|
|
52
|
+
<div><span>Quality</span><strong id="metric-quality">—</strong><small id="metric-quality-delta">top-1 accuracy</small></div>
|
|
53
|
+
<div><span>Cost</span><strong id="metric-cost">—</strong><small>measured USD</small></div>
|
|
54
|
+
<div><span>Tokens</span><strong id="metric-tokens">—</strong><small>estimated output words</small></div>
|
|
55
|
+
<div><span>Latency</span><strong id="metric-latency">—</strong><small>wall clock</small></div>
|
|
56
|
+
<div><span>Failures</span><strong id="metric-failures">—</strong><small>examples</small></div>
|
|
57
|
+
</section>
|
|
58
|
+
|
|
59
|
+
<section class="result-section entrance entrance-delay-2" aria-labelledby="quality-heading">
|
|
60
|
+
<div class="result-section-heading"><div><p class="section-index">01 / EFFECT</p><h2 id="quality-heading">Paired quality</h2></div><p id="scope-label">Synthetic smoke result</p></div>
|
|
61
|
+
<div class="quality-layout">
|
|
62
|
+
<div class="bar-compare">
|
|
63
|
+
<div class="bar-row"><span>Baseline</span><div class="bar-track"><i id="baseline-bar"></i></div><strong id="baseline-value">—</strong></div>
|
|
64
|
+
<div class="bar-row treatment"><span>Treatment</span><div class="bar-track"><i id="treatment-bar"></i></div><strong id="treatment-value">—</strong></div>
|
|
65
|
+
</div>
|
|
66
|
+
<dl class="effect-list">
|
|
67
|
+
<div><dt>Mean effect</dt><dd id="effect-value">—</dd></div>
|
|
68
|
+
<div><dt>Win / tie / harm</dt><dd id="rates-value">—</dd></div>
|
|
69
|
+
<div><dt>Intent recall</dt><dd id="intent-value">—</dd></div>
|
|
70
|
+
<div><dt>Token Jaccard</dt><dd id="jaccard-value">—</dd></div>
|
|
71
|
+
</dl>
|
|
72
|
+
</div>
|
|
73
|
+
</section>
|
|
74
|
+
|
|
75
|
+
<section class="result-section" aria-labelledby="runtime-heading">
|
|
76
|
+
<div class="result-section-heading"><div><p class="section-index">02 / RUNTIME</p><h2 id="runtime-heading">Usage and timing</h2></div><p>Instrumented at the Studio boundary</p></div>
|
|
77
|
+
<div class="runtime-columns">
|
|
78
|
+
<dl id="usage-list" class="dense-list"></dl>
|
|
79
|
+
<dl id="timing-list" class="dense-list"></dl>
|
|
80
|
+
<dl id="fairness-list" class="dense-list"></dl>
|
|
81
|
+
</div>
|
|
82
|
+
</section>
|
|
83
|
+
|
|
84
|
+
<section class="result-section" aria-labelledby="samples-heading">
|
|
85
|
+
<div class="result-section-heading"><div><p class="section-index">03 / EXAMPLES</p><h2 id="samples-heading">Sample outcomes</h2></div><p>Open a row to inspect its state transformation</p></div>
|
|
86
|
+
<div id="sample-list" class="sample-list"></div>
|
|
87
|
+
</section>
|
|
88
|
+
|
|
89
|
+
<section class="result-section" aria-labelledby="failure-heading">
|
|
90
|
+
<div class="result-section-heading"><div><p class="section-index">04 / FAILURES</p><h2 id="failure-heading">Failure ledger</h2></div><p id="failure-summary">No failures recorded</p></div>
|
|
91
|
+
<div id="failure-list" class="failure-list"></div>
|
|
92
|
+
</section>
|
|
93
|
+
|
|
94
|
+
<section class="result-section" aria-labelledby="trace-heading">
|
|
95
|
+
<div class="result-section-heading"><div><p class="section-index">05 / TRACE</p><h2 id="trace-heading">Execution trace</h2></div><button id="toggle-json" class="text-button" type="button">Raw JSON</button></div>
|
|
96
|
+
<div id="trace-list" class="trace-list"></div>
|
|
97
|
+
<pre id="raw-json" class="raw-json" hidden></pre>
|
|
98
|
+
</section>
|
|
99
|
+
</div>
|
|
100
|
+
</main>
|
|
101
|
+
</div>
|
|
102
|
+
</body>
|
|
103
|
+
</html>
|
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
const el = (id) => document.getElementById(id);
|
|
4
|
+
let activeRecord = null;
|
|
5
|
+
let pollTimer = null;
|
|
6
|
+
|
|
7
|
+
async function requestJSON(url) {
|
|
8
|
+
const response = await fetch(url);
|
|
9
|
+
const payload = await response.json().catch(() => ({}));
|
|
10
|
+
if (!response.ok) throw new Error(payload?.error?.message || `Request failed (${response.status})`);
|
|
11
|
+
return payload;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
function formatPercent(value, signed = false) {
|
|
15
|
+
if (!Number.isFinite(value)) return "—";
|
|
16
|
+
const prefix = signed && value > 0 ? "+" : "";
|
|
17
|
+
return `${prefix}${(value * 100).toFixed(1)}%`;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function formatNumber(value, digits = 2) {
|
|
21
|
+
return Number.isFinite(value) ? value.toFixed(digits) : "—";
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function formatStatus(status) {
|
|
25
|
+
return String(status || "waiting").replaceAll("_", " ").toUpperCase();
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function backendAuditLabel(backend) {
|
|
29
|
+
const backendId = typeof backend?.backend_id === "string"
|
|
30
|
+
? backend.backend_id
|
|
31
|
+
: "backend identity unavailable";
|
|
32
|
+
const fingerprint = backend?.actual_backend_fingerprint_sha256
|
|
33
|
+
|| backend?.model_id_sha256;
|
|
34
|
+
return typeof fingerprint === "string" && /^[0-9a-f]{64}$/.test(fingerprint)
|
|
35
|
+
? `${backendId} · ${fingerprint.slice(0, 12)}`
|
|
36
|
+
: backendId;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function setStatus(status) {
|
|
40
|
+
const badge = el("run-status");
|
|
41
|
+
badge.className = `run-status ${status || "waiting"}`;
|
|
42
|
+
badge.textContent = formatStatus(status);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function setDefinitionList(target, entries) {
|
|
46
|
+
target.replaceChildren();
|
|
47
|
+
for (const [label, value] of entries) {
|
|
48
|
+
const row = document.createElement("div");
|
|
49
|
+
const term = document.createElement("dt");
|
|
50
|
+
const description = document.createElement("dd");
|
|
51
|
+
term.textContent = label;
|
|
52
|
+
description.textContent = value;
|
|
53
|
+
row.append(term, description);
|
|
54
|
+
target.append(row);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function createText(tag, text, className = "") {
|
|
59
|
+
const node = document.createElement(tag);
|
|
60
|
+
node.textContent = text;
|
|
61
|
+
if (className) node.className = className;
|
|
62
|
+
return node;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function renderSamples(samples) {
|
|
66
|
+
const list = el("sample-list");
|
|
67
|
+
list.replaceChildren();
|
|
68
|
+
for (const [index, sample] of samples.entries()) {
|
|
69
|
+
const details = document.createElement("details");
|
|
70
|
+
details.className = "sample-item";
|
|
71
|
+
const summary = document.createElement("summary");
|
|
72
|
+
summary.append(
|
|
73
|
+
createText("span", String(index + 1).padStart(2, "0"), "sample-index"),
|
|
74
|
+
createText("span", sample.original_query, "sample-query"),
|
|
75
|
+
createText("span", `B ${formatPercent(sample.baseline_score)}`, "sample-score"),
|
|
76
|
+
createText("span", `T ${formatPercent(sample.treatment_score)}`, "sample-score"),
|
|
77
|
+
createText("span", "+", "sample-chevron"),
|
|
78
|
+
);
|
|
79
|
+
const detail = document.createElement("div");
|
|
80
|
+
detail.className = "sample-detail";
|
|
81
|
+
const before = document.createElement("div");
|
|
82
|
+
before.append(createText("h3", "ORIGINAL STATE"), createText("p", sample.original_query));
|
|
83
|
+
const after = document.createElement("div");
|
|
84
|
+
after.append(
|
|
85
|
+
createText("h3", "TRANSFORMED STATE"),
|
|
86
|
+
createText("p", sample.transformed_query),
|
|
87
|
+
);
|
|
88
|
+
if (sample.transformed_instruction) {
|
|
89
|
+
after.append(createText("h3", "INSTRUCTION"), createText("p", sample.transformed_instruction));
|
|
90
|
+
}
|
|
91
|
+
const metadata = document.createElement("dl");
|
|
92
|
+
setDefinitionList(metadata, [
|
|
93
|
+
["Target", sample.target_document_id],
|
|
94
|
+
["Baseline top", sample.baseline_top_document_id || "none"],
|
|
95
|
+
["Treatment top", sample.treatment_top_document_id || "none"],
|
|
96
|
+
["Latency", `${formatNumber(sample.latency_ms, 3)} ms`],
|
|
97
|
+
["Patch", sample.patch_hash.slice(0, 12)],
|
|
98
|
+
]);
|
|
99
|
+
after.append(metadata);
|
|
100
|
+
detail.append(before, after);
|
|
101
|
+
details.append(summary, detail);
|
|
102
|
+
list.append(details);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function renderFailures(failures) {
|
|
107
|
+
const list = el("failure-list");
|
|
108
|
+
list.replaceChildren();
|
|
109
|
+
el("failure-summary").textContent = failures.length ? `${failures.length} failures recorded` : "No failures recorded";
|
|
110
|
+
if (!failures.length) {
|
|
111
|
+
list.append(createText("p", "All requested examples reached a terminal result.", "no-failures"));
|
|
112
|
+
return;
|
|
113
|
+
}
|
|
114
|
+
for (const failure of failures) {
|
|
115
|
+
const row = document.createElement("div");
|
|
116
|
+
row.className = "failure-item";
|
|
117
|
+
row.append(
|
|
118
|
+
createText("strong", `${failure.example_id} · ${failure.category}`),
|
|
119
|
+
createText("span", failure.message),
|
|
120
|
+
createText("span", `${formatNumber(failure.latency_ms, 2)} ms`),
|
|
121
|
+
);
|
|
122
|
+
list.append(row);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function summarizePayload(payload) {
|
|
127
|
+
const rendered = JSON.stringify(payload);
|
|
128
|
+
return rendered.length > 420 ? `${rendered.slice(0, 419)}…` : rendered;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function renderTrace(samples) {
|
|
132
|
+
const list = el("trace-list");
|
|
133
|
+
list.replaceChildren();
|
|
134
|
+
for (const sample of samples) {
|
|
135
|
+
const trace = sample.trace;
|
|
136
|
+
const group = document.createElement("div");
|
|
137
|
+
group.className = "trace-group";
|
|
138
|
+
const identity = document.createElement("div");
|
|
139
|
+
identity.append(
|
|
140
|
+
createText("strong", sample.example_id),
|
|
141
|
+
createText("small", trace.release_id),
|
|
142
|
+
createText("small", sample.patch_hash),
|
|
143
|
+
);
|
|
144
|
+
const events = document.createElement("div");
|
|
145
|
+
events.className = "trace-events";
|
|
146
|
+
for (const event of trace.events || []) {
|
|
147
|
+
const item = document.createElement("div");
|
|
148
|
+
item.className = "trace-event";
|
|
149
|
+
item.append(
|
|
150
|
+
createText("strong", `${event.phase} · ${event.name}`),
|
|
151
|
+
createText("p", summarizePayload(event.payload)),
|
|
152
|
+
);
|
|
153
|
+
events.append(item);
|
|
154
|
+
}
|
|
155
|
+
for (const event of sample.observability?.snapshot?.events || []) {
|
|
156
|
+
const item = document.createElement("div");
|
|
157
|
+
item.className = "trace-event monitor-event";
|
|
158
|
+
item.append(
|
|
159
|
+
createText("strong", `monitor · ${event.kind} · ${event.component || "run"}`),
|
|
160
|
+
createText("p", `${formatNumber(event.elapsed_ms, 3)} ms · ${summarizePayload(event.payload)}`),
|
|
161
|
+
);
|
|
162
|
+
events.append(item);
|
|
163
|
+
}
|
|
164
|
+
if (!(trace.events || []).length) {
|
|
165
|
+
events.append(createText("p", "Typed NO_OP trace contains no method-specific events."));
|
|
166
|
+
}
|
|
167
|
+
group.append(identity, events);
|
|
168
|
+
list.append(group);
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
function renderResult(record) {
|
|
173
|
+
activeRecord = record;
|
|
174
|
+
setStatus(record.status);
|
|
175
|
+
el("result-title").textContent = record.config.release_id;
|
|
176
|
+
el("result-subtitle").textContent = `${record.config.dataset_id} · run ${record.id.slice(0, 12)} · created ${record.created_at}`;
|
|
177
|
+
if (!record.result) {
|
|
178
|
+
el("result-empty").hidden = false;
|
|
179
|
+
el("result-empty").querySelector("h2").textContent = record.status === "failed" ? "Run failed before producing a result" : "Run is in progress";
|
|
180
|
+
el("result-empty").querySelector("p").textContent = record.error?.message || "The result page will update automatically when the local worker finishes.";
|
|
181
|
+
return;
|
|
182
|
+
}
|
|
183
|
+
el("result-empty").hidden = true;
|
|
184
|
+
el("result-content").hidden = false;
|
|
185
|
+
const summary = record.result.summary;
|
|
186
|
+
const quality = summary.quality;
|
|
187
|
+
const usage = summary.usage;
|
|
188
|
+
const timing = summary.timing;
|
|
189
|
+
el("metric-quality").textContent = formatPercent(quality.treatment);
|
|
190
|
+
el("metric-quality-delta").textContent = `${formatPercent(quality.mean_effect, true)} vs baseline`;
|
|
191
|
+
el("metric-cost").textContent = `$${formatNumber(usage.cost_usd, 4)}`;
|
|
192
|
+
el("metric-tokens").textContent = String(usage.output_tokens_estimate ?? "—");
|
|
193
|
+
el("metric-latency").textContent = `${formatNumber(timing.wall_ms, 1)} ms`;
|
|
194
|
+
el("metric-failures").textContent = String(summary.failure_count);
|
|
195
|
+
el("scope-label").textContent = record.result.result_scope.replaceAll("_", " ");
|
|
196
|
+
el("baseline-value").textContent = formatPercent(quality.baseline);
|
|
197
|
+
el("treatment-value").textContent = formatPercent(quality.treatment);
|
|
198
|
+
el("effect-value").textContent = formatPercent(quality.mean_effect, true);
|
|
199
|
+
el("rates-value").textContent = `${formatPercent(quality.win_rate)} / ${formatPercent(quality.tie_rate)} / ${formatPercent(quality.harm_rate)}`;
|
|
200
|
+
el("intent-value").textContent = formatPercent(quality.intent_token_recall);
|
|
201
|
+
el("jaccard-value").textContent = formatPercent(quality.token_jaccard);
|
|
202
|
+
requestAnimationFrame(() => {
|
|
203
|
+
el("baseline-bar").style.width = `${Math.max(0, Math.min(100, quality.baseline * 100))}%`;
|
|
204
|
+
el("treatment-bar").style.width = `${Math.max(0, Math.min(100, quality.treatment * 100))}%`;
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
setDefinitionList(el("usage-list"), [
|
|
208
|
+
["Model calls", String(usage.model_calls)],
|
|
209
|
+
["Optimizer retrieval", String(usage.optimizer_retrieval_calls)],
|
|
210
|
+
["Evaluation retrieval", String(usage.evaluation_retrieval_calls)],
|
|
211
|
+
["Tool / embedding", `${usage.tool_calls} / ${usage.embedding_calls}`],
|
|
212
|
+
["Core tokens", `${usage.core_input_tokens} in / ${usage.core_output_tokens} out`],
|
|
213
|
+
["Input estimate", String(usage.input_tokens_estimate)],
|
|
214
|
+
["Output estimate", String(usage.output_tokens_estimate)],
|
|
215
|
+
["CPU", `${formatNumber(usage.cpu_seconds, 6)} s`],
|
|
216
|
+
["Cost basis", usage.cost_basis],
|
|
217
|
+
]);
|
|
218
|
+
setDefinitionList(el("timing-list"), [
|
|
219
|
+
["Wall", `${formatNumber(timing.wall_ms, 3)} ms`],
|
|
220
|
+
["Mean sample", `${formatNumber(timing.mean_sample_ms, 3)} ms`],
|
|
221
|
+
["Median sample", `${formatNumber(timing.median_sample_ms, 3)} ms`],
|
|
222
|
+
["P95 sample", `${formatNumber(timing.p95_sample_ms, 3)} ms`],
|
|
223
|
+
["Core intervention", `${formatNumber(timing.core_intervention_ms, 3)} ms`],
|
|
224
|
+
["Core mean", `${formatNumber(timing.core_mean_intervention_ms, 3)} ms`],
|
|
225
|
+
["Started", record.started_at || "—"],
|
|
226
|
+
["Finished", record.finished_at || "—"],
|
|
227
|
+
]);
|
|
228
|
+
const audit = record.result.fairness_audit;
|
|
229
|
+
setDefinitionList(el("fairness-list"), [
|
|
230
|
+
["Seed", String(audit.seed)],
|
|
231
|
+
["Fingerprint drift", String(audit.fingerprint_drift)],
|
|
232
|
+
["Retriever", audit.retriever_before.retriever_id],
|
|
233
|
+
["Corpus", audit.retriever_before.corpus_hash.slice(0, 12)],
|
|
234
|
+
["Backend", backendAuditLabel(audit.backend)],
|
|
235
|
+
["Dataset revision", audit.dataset_revision],
|
|
236
|
+
]);
|
|
237
|
+
renderSamples(record.result.samples);
|
|
238
|
+
renderFailures(record.result.failures);
|
|
239
|
+
renderTrace(record.result.samples);
|
|
240
|
+
el("raw-json").textContent = JSON.stringify(record, null, 2);
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
async function loadRun(runId) {
|
|
244
|
+
try {
|
|
245
|
+
const record = await requestJSON(`/api/runs/${encodeURIComponent(runId)}`);
|
|
246
|
+
renderResult(record);
|
|
247
|
+
if (["queued", "running"].includes(record.status)) {
|
|
248
|
+
clearTimeout(pollTimer);
|
|
249
|
+
pollTimer = setTimeout(() => loadRun(runId), 750);
|
|
250
|
+
}
|
|
251
|
+
} catch (error) {
|
|
252
|
+
setStatus("failed");
|
|
253
|
+
el("result-empty").hidden = false;
|
|
254
|
+
el("result-empty").querySelector("h2").textContent = "Result is unavailable";
|
|
255
|
+
el("result-empty").querySelector("p").textContent = error.message;
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
function init() {
|
|
260
|
+
const runId = new URLSearchParams(window.location.search).get("run");
|
|
261
|
+
el("toggle-json").addEventListener("click", () => {
|
|
262
|
+
const raw = el("raw-json");
|
|
263
|
+
raw.hidden = !raw.hidden;
|
|
264
|
+
el("toggle-json").textContent = raw.hidden ? "Raw JSON" : "Hide JSON";
|
|
265
|
+
});
|
|
266
|
+
if (!runId) return;
|
|
267
|
+
el("result-empty").hidden = false;
|
|
268
|
+
el("result-empty").querySelector("h2").textContent = "Loading run";
|
|
269
|
+
el("result-empty").querySelector("p").textContent = "Reading the local run record and measurements.";
|
|
270
|
+
loadRun(runId);
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
document.addEventListener("DOMContentLoaded", init);
|