agentinputlab 0.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentinputlab/__init__.py +28 -0
- agentinputlab/algorithms/__init__.py +6 -0
- agentinputlab/algorithms/acquire/__init__.py +145 -0
- agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
- agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
- agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
- agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
- agentinputlab/algorithms/acquire/hyde.py +296 -0
- agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
- agentinputlab/algorithms/acquire/hyde_external.py +148 -0
- agentinputlab/algorithms/acquire/ircot.py +28 -0
- agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
- agentinputlab/algorithms/acquire/ircot_native.py +355 -0
- agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
- agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
- agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
- agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
- agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
- agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
- agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
- agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
- agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
- agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
- agentinputlab/algorithms/clarify/__init__.py +61 -0
- agentinputlab/algorithms/clarify/_structured.py +64 -0
- agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
- agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
- agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
- agentinputlab/algorithms/clarify/future_turns.py +379 -0
- agentinputlab/algorithms/clarify/udcs.py +255 -0
- agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
- agentinputlab/algorithms/curate/__init__.py +65 -0
- agentinputlab/algorithms/curate/acon/__init__.py +27 -0
- agentinputlab/algorithms/curate/acon/common.py +284 -0
- agentinputlab/algorithms/curate/acon/common_support.py +136 -0
- agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
- agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
- agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
- agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
- agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
- agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
- agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
- agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
- agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
- agentinputlab/algorithms/curate/recomp/external.py +1129 -0
- agentinputlab/algorithms/curate/recomp/selective.py +304 -0
- agentinputlab/algorithms/decompose/__init__.py +107 -0
- agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
- agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
- agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
- agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
- agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
- agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
- agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
- agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
- agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
- agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
- agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
- agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
- agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
- agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
- agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
- agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
- agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
- agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
- agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
- agentinputlab/algorithms/reformulate/__init__.py +85 -0
- agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
- agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
- agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
- agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
- agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
- agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
- agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
- agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
- agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
- agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
- agentinputlab/algorithms/reformulate/rrr.py +696 -0
- agentinputlab/annotation/__init__.py +5 -0
- agentinputlab/annotation/independent.py +428 -0
- agentinputlab/annotation/server.py +233 -0
- agentinputlab/annotation/static/app.js +121 -0
- agentinputlab/annotation/static/index.html +60 -0
- agentinputlab/annotation/static/review.css +2 -0
- agentinputlab/annotation/static/review.html +10 -0
- agentinputlab/annotation/static/review.js +11 -0
- agentinputlab/annotation/static/styles.css +55 -0
- agentinputlab/annotation/store.py +444 -0
- agentinputlab/api.py +368 -0
- agentinputlab/artifacts/__init__.py +15 -0
- agentinputlab/artifacts/json_file.py +104 -0
- agentinputlab/artifacts/manifest.py +98 -0
- agentinputlab/artifacts/store.py +157 -0
- agentinputlab/backends/__init__.py +98 -0
- agentinputlab/backends/embedding/__init__.py +25 -0
- agentinputlab/backends/embedding/base.py +145 -0
- agentinputlab/backends/embedding/callable.py +75 -0
- agentinputlab/backends/embedding/replay.py +129 -0
- agentinputlab/backends/llm/__init__.py +31 -0
- agentinputlab/backends/llm/base.py +149 -0
- agentinputlab/backends/llm/callable.py +108 -0
- agentinputlab/backends/llm/decomp_completion.py +88 -0
- agentinputlab/backends/llm/final_content_stop.py +72 -0
- agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
- agentinputlab/backends/llm/openai_compatible.py +332 -0
- agentinputlab/backends/llm/transformers.py +171 -0
- agentinputlab/backends/retrieval/__init__.py +35 -0
- agentinputlab/backends/retrieval/base.py +50 -0
- agentinputlab/backends/retrieval/dense.py +777 -0
- agentinputlab/backends/retrieval/memory.py +119 -0
- agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
- agentinputlab/backends/user/__init__.py +15 -0
- agentinputlab/backends/user/base.py +66 -0
- agentinputlab/backends/user/callable.py +54 -0
- agentinputlab/backends/user/replay.py +89 -0
- agentinputlab/backends/vector.py +38 -0
- agentinputlab/benchmarks/__init__.py +246 -0
- agentinputlab/benchmarks/acquisition.py +106 -0
- agentinputlab/benchmarks/ale.py +540 -0
- agentinputlab/benchmarks/appworld_ul.py +669 -0
- agentinputlab/benchmarks/assets.py +226 -0
- agentinputlab/benchmarks/base.py +68 -0
- agentinputlab/benchmarks/beir_archive.py +235 -0
- agentinputlab/benchmarks/beir_sample.py +269 -0
- agentinputlab/benchmarks/bfcl_native.py +245 -0
- agentinputlab/benchmarks/bfcl_v3.py +607 -0
- agentinputlab/benchmarks/cache.py +218 -0
- agentinputlab/benchmarks/clariq.py +203 -0
- agentinputlab/benchmarks/data/registry.json +321 -0
- agentinputlab/benchmarks/data/smoke_assets.json +511 -0
- agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
- agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
- agentinputlab/benchmarks/execution.py +1111 -0
- agentinputlab/benchmarks/interactive.py +257 -0
- agentinputlab/benchmarks/ir_query.py +204 -0
- agentinputlab/benchmarks/longbench.py +203 -0
- agentinputlab/benchmarks/paired_replay.py +479 -0
- agentinputlab/benchmarks/preprocess.py +2313 -0
- agentinputlab/benchmarks/production.py +236 -0
- agentinputlab/benchmarks/qa.py +963 -0
- agentinputlab/benchmarks/registry.py +110 -0
- agentinputlab/benchmarks/retrieval.py +459 -0
- agentinputlab/benchmarks/runtime.py +509 -0
- agentinputlab/benchmarks/statebench.py +938 -0
- agentinputlab/benchmarks/tau3.py +725 -0
- agentinputlab/benchmarks/toolsandbox.py +626 -0
- agentinputlab/benchmarks/trec_dl_sample.py +263 -0
- agentinputlab/campaign.py +654 -0
- agentinputlab/campaign_runner.py +953 -0
- agentinputlab/cli/__init__.py +6 -0
- agentinputlab/cli/main.py +933 -0
- agentinputlab/connectors/__init__.py +46 -0
- agentinputlab/connectors/knowledge/__init__.py +4 -0
- agentinputlab/connectors/tools/__init__.py +30 -0
- agentinputlab/connectors/tools/base.py +57 -0
- agentinputlab/connectors/tools/mcp.py +222 -0
- agentinputlab/connectors/tools/memory.py +41 -0
- agentinputlab/connectors/tools/openapi.py +695 -0
- agentinputlab/connectors/tools/schema.py +102 -0
- agentinputlab/contracts/__init__.py +91 -0
- agentinputlab/contracts/availability.py +24 -0
- agentinputlab/contracts/dependency.py +58 -0
- agentinputlab/contracts/enums.py +82 -0
- agentinputlab/contracts/errors.py +63 -0
- agentinputlab/contracts/evidence.py +43 -0
- agentinputlab/contracts/intent.py +110 -0
- agentinputlab/contracts/lifecycle.py +164 -0
- agentinputlab/contracts/patch.py +167 -0
- agentinputlab/contracts/serialization.py +93 -0
- agentinputlab/contracts/state.py +127 -0
- agentinputlab/contracts/trace.py +59 -0
- agentinputlab/contracts/worker.py +42 -0
- agentinputlab/core/__init__.py +7 -0
- agentinputlab/data/__init__.py +32 -0
- agentinputlab/data/checksums.py +9 -0
- agentinputlab/data/downloader.py +235 -0
- agentinputlab/data/http.py +171 -0
- agentinputlab/data/index.py +21 -0
- agentinputlab/data/overlap.py +28 -0
- agentinputlab/data/preprocess.py +32 -0
- agentinputlab/data/storage.py +57 -0
- agentinputlab/data/trec_sample.py +427 -0
- agentinputlab/evaluation/__init__.py +5 -0
- agentinputlab/evaluation/outcomes.py +32 -0
- agentinputlab/experiments/__init__.py +278 -0
- agentinputlab/experiments/analysis.py +981 -0
- agentinputlab/experiments/artifacts.py +687 -0
- agentinputlab/experiments/automatic_measurement.py +678 -0
- agentinputlab/experiments/auxiliary_matrix.py +203 -0
- agentinputlab/experiments/bindings.py +430 -0
- agentinputlab/experiments/blueprints.py +147 -0
- agentinputlab/experiments/builders/__init__.py +45 -0
- agentinputlab/experiments/builders/executable_external_payload.py +423 -0
- agentinputlab/experiments/builders/external_process_arena.py +1018 -0
- agentinputlab/experiments/builders/registry_intervention.py +463 -0
- agentinputlab/experiments/cli.py +158 -0
- agentinputlab/experiments/controls.py +340 -0
- agentinputlab/experiments/driver_guard.py +205 -0
- agentinputlab/experiments/drivers/__init__.py +89 -0
- agentinputlab/experiments/drivers/external_arena.py +2908 -0
- agentinputlab/experiments/drivers/method_process.py +662 -0
- agentinputlab/experiments/drivers/parity_process.py +1484 -0
- agentinputlab/experiments/drivers/static_production.py +1574 -0
- agentinputlab/experiments/executor.py +448 -0
- agentinputlab/experiments/loader.py +605 -0
- agentinputlab/experiments/measurement.py +675 -0
- agentinputlab/experiments/paper_analysis.py +1818 -0
- agentinputlab/experiments/parity_identity.py +605 -0
- agentinputlab/experiments/plan.py +277 -0
- agentinputlab/experiments/production_builders.py +3739 -0
- agentinputlab/experiments/readiness.py +432 -0
- agentinputlab/experiments/schema.py +514 -0
- agentinputlab/experiments/secrets.py +177 -0
- agentinputlab/experiments/stateful_analysis.py +1738 -0
- agentinputlab/experiments/transfer_analysis.py +1316 -0
- agentinputlab/experiments/unseen_analysis.py +1025 -0
- agentinputlab/integrations/__init__.py +1 -0
- agentinputlab/integrations/action_space/__init__.py +1 -0
- agentinputlab/integrations/external_workers/__init__.py +76 -0
- agentinputlab/integrations/external_workers/ace.py +431 -0
- agentinputlab/integrations/external_workers/base.py +181 -0
- agentinputlab/integrations/external_workers/decomp_live.py +249 -0
- agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
- agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
- agentinputlab/integrations/external_workers/ircot.py +576 -0
- agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
- agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
- agentinputlab/integrations/external_workers/method.py +314 -0
- agentinputlab/integrations/external_workers/process.py +678 -0
- agentinputlab/integrations/external_workers/supervisor.py +1071 -0
- agentinputlab/integrations/full_architecture/__init__.py +1 -0
- agentinputlab/integrations/full_architecture/hyde.py +212 -0
- agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
- agentinputlab/integrations/live_history.py +222 -0
- agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
- agentinputlab/integrations/training/__init__.py +1 -0
- agentinputlab/metrics/__init__.py +50 -0
- agentinputlab/metrics/catalog.py +313 -0
- agentinputlab/metrics/harm.py +95 -0
- agentinputlab/metrics/paired.py +40 -0
- agentinputlab/metrics/qa.py +76 -0
- agentinputlab/metrics/text.py +33 -0
- agentinputlab/metrics/third_party.py +161 -0
- agentinputlab/models/__init__.py +4 -0
- agentinputlab/observability/__init__.py +56 -0
- agentinputlab/observability/budget.py +316 -0
- agentinputlab/observability/instrumentation.py +754 -0
- agentinputlab/observability/models.py +403 -0
- agentinputlab/observability/monitor.py +380 -0
- agentinputlab/optimizers/__init__.py +1 -0
- agentinputlab/optimizers/clarify/__init__.py +12 -0
- agentinputlab/optimizers/context/__init__.py +22 -0
- agentinputlab/optimizers/decompose/__init__.py +64 -0
- agentinputlab/optimizers/evidence/__init__.py +22 -0
- agentinputlab/optimizers/rewrite/__init__.py +30 -0
- agentinputlab/parity/__init__.py +5 -0
- agentinputlab/parity/base.py +206 -0
- agentinputlab/production_campaign.py +763 -0
- agentinputlab/py.typed +1 -0
- agentinputlab/recipes/__init__.py +333 -0
- agentinputlab/recipes/ace.py +418 -0
- agentinputlab/recipes/ace_bfcl.py +273 -0
- agentinputlab/recipes/ace_common_support.py +169 -0
- agentinputlab/recipes/ace_native_plan.py +322 -0
- agentinputlab/recipes/ace_online.py +711 -0
- agentinputlab/recipes/ace_retrieval.py +330 -0
- agentinputlab/recipes/ace_statebench.py +382 -0
- agentinputlab/recipes/ace_tau.py +463 -0
- agentinputlab/recipes/ace_toolsandbox.py +432 -0
- agentinputlab/recipes/acon_guideline.py +22 -0
- agentinputlab/recipes/adaptive_rag.py +822 -0
- agentinputlab/recipes/llmlingua2_inference.py +146 -0
- agentinputlab/recipes/llmlingua2_training.py +796 -0
- agentinputlab/recipes/mce.py +1486 -0
- agentinputlab/recipes/mce_final.py +335 -0
- agentinputlab/recipes/mce_text_fit.py +499 -0
- agentinputlab/recipes/mce_text_only.py +298 -0
- agentinputlab/recipes/qrag.py +729 -0
- agentinputlab/recipes/qrag_inference.py +364 -0
- agentinputlab/recipes/r2ag_features.py +236 -0
- agentinputlab/recipes/r2ag_inference.py +503 -0
- agentinputlab/recipes/r2ag_training.py +948 -0
- agentinputlab/recipes/recomp_abstractive.py +592 -0
- agentinputlab/recipes/recomp_extractive.py +525 -0
- agentinputlab/recipes/reformer.py +17 -0
- agentinputlab/recipes/rqrag.py +1516 -0
- agentinputlab/recipes/rqrag_common_support.py +193 -0
- agentinputlab/recipes/rqrag_completion.py +147 -0
- agentinputlab/recipes/skillopt.py +462 -0
- agentinputlab/recipes/skillopt_common_support.py +154 -0
- agentinputlab/recipes/skillopt_environment.py +165 -0
- agentinputlab/recipes/skillopt_environment_cli.py +5 -0
- agentinputlab/recipes/skillopt_retrieval.py +188 -0
- agentinputlab/recipes/smartsearch.py +897 -0
- agentinputlab/recipes/smartsearch_curriculum.py +283 -0
- agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
- agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
- agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
- agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
- agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
- agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
- agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
- agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
- agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
- agentinputlab/recipes/smartsearch_inference.py +524 -0
- agentinputlab/recipes/smartsearch_native_data.py +172 -0
- agentinputlab/recipes/smartsearch_policy.py +616 -0
- agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
- agentinputlab/recipes/smartsearch_sampling.py +232 -0
- agentinputlab/recipes/smartsearch_teacher.py +203 -0
- agentinputlab/recipes/smartsearch_training.py +265 -0
- agentinputlab/registry/__init__.py +46 -0
- agentinputlab/registry/beta.py +727 -0
- agentinputlab/registry/data/algorithms.json +1959 -0
- agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
- agentinputlab/registry/loader.py +100 -0
- agentinputlab/registry/result_manifest.py +184 -0
- agentinputlab/registry/runtime.py +195 -0
- agentinputlab/registry/schema.py +449 -0
- agentinputlab/runner/__init__.py +30 -0
- agentinputlab/runner/balance.py +248 -0
- agentinputlab/runner/cache.py +135 -0
- agentinputlab/runner/composite.py +130 -0
- agentinputlab/runner/engine.py +285 -0
- agentinputlab/runner/runtime_guard.py +268 -0
- agentinputlab/runner/worker.py +80 -0
- agentinputlab/runtime_bootstrap.py +288 -0
- agentinputlab/selectors/__init__.py +196 -0
- agentinputlab/selectors/advanced.py +1514 -0
- agentinputlab/selectors/artifacts.py +1087 -0
- agentinputlab/selectors/base.py +112 -0
- agentinputlab/selectors/campaigns.py +1893 -0
- agentinputlab/selectors/catalog.py +86 -0
- agentinputlab/selectors/discovery.py +1860 -0
- agentinputlab/selectors/evaluation.py +1348 -0
- agentinputlab/selectors/features.py +383 -0
- agentinputlab/selectors/learned.py +502 -0
- agentinputlab/selectors/no_op.py +28 -0
- agentinputlab/selectors/outcome_table.py +475 -0
- agentinputlab/selectors/producer.py +853 -0
- agentinputlab/selectors/rules.py +109 -0
- agentinputlab/selectors/runtime.py +420 -0
- agentinputlab/studies/__init__.py +16 -0
- agentinputlab/studies/design.py +165 -0
- agentinputlab/studies/paired.py +322 -0
- agentinputlab/studio/__init__.py +6 -0
- agentinputlab/studio/campaigns.py +819 -0
- agentinputlab/studio/engine.py +1408 -0
- agentinputlab/studio/model_backends.py +425 -0
- agentinputlab/studio/prepared.py +410 -0
- agentinputlab/studio/server.py +613 -0
- agentinputlab/studio/static/app.js +397 -0
- agentinputlab/studio/static/campaigns.html +175 -0
- agentinputlab/studio/static/campaigns.js +426 -0
- agentinputlab/studio/static/index.html +192 -0
- agentinputlab/studio/static/result.html +103 -0
- agentinputlab/studio/static/result.js +273 -0
- agentinputlab/studio/static/styles.css +467 -0
- agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
- agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
- agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
- agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
- agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
- agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
- agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
- agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
- agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
- workers/README.md +94 -0
- workers/ace/README.md +261 -0
- workers/ace/__init__.py +1 -0
- workers/ace/compatibility_manifest.json +39 -0
- workers/ace/driver.py +1682 -0
- workers/acon/README.md +215 -0
- workers/acon/__init__.py +5 -0
- workers/acon/compatibility_manifest.json +93 -0
- workers/acon/driver.py +1859 -0
- workers/acon/fitted.py +395 -0
- workers/acon/guideline.py +470 -0
- workers/acon/guideline_source_manifest.json +20 -0
- workers/adaptive_rag/README.md +96 -0
- workers/adaptive_rag/__init__.py +15 -0
- workers/adaptive_rag/compatibility_manifest.json +61 -0
- workers/adaptive_rag/driver.py +1470 -0
- workers/adaptive_rag/fixture_runtime.py +163 -0
- workers/adaptive_rag/runtime.py +252 -0
- workers/ask_or_assume/README.md +124 -0
- workers/ask_or_assume/__init__.py +1 -0
- workers/ask_or_assume/compatibility_manifest.json +36 -0
- workers/ask_or_assume/driver.py +1227 -0
- workers/decomp/README.md +133 -0
- workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
- workers/decomp/__init__.py +5 -0
- workers/decomp/compatibility_manifest.json +205 -0
- workers/decomp/driver.py +1015 -0
- workers/decomp/live.py +137 -0
- workers/decomp/runtime.py +231 -0
- workers/infocqr/README.md +106 -0
- workers/infocqr/__init__.py +1 -0
- workers/infocqr/compatibility_manifest.json +84 -0
- workers/infocqr/driver.py +1766 -0
- workers/ircot/README.md +167 -0
- workers/ircot/__init__.py +7 -0
- workers/ircot/compatibility_manifest.json +121 -0
- workers/ircot/controlled.py +457 -0
- workers/ircot/driver.py +1373 -0
- workers/ircot/fixture_bindings.json +39 -0
- workers/ircot/fixture_config.json +48 -0
- workers/ircot/fixture_prompt.txt +2 -0
- workers/ircot/native_qa.py +398 -0
- workers/ircot/qa_compatibility_manifest.json +57 -0
- workers/ircot/runtime.py +97 -0
- workers/llmcompiler/README.md +160 -0
- workers/llmcompiler/__init__.py +1 -0
- workers/llmcompiler/compatibility_manifest.json +45 -0
- workers/llmcompiler/driver.py +1514 -0
- workers/llmcompiler/export_prompts.py +74 -0
- workers/llmcompiler/live.py +174 -0
- workers/llmcompiler/patch_manifest.json +147 -0
- workers/mce/README.md +198 -0
- workers/mce/__init__.py +29 -0
- workers/mce/compatibility_manifest.json +115 -0
- workers/mce/driver.py +2585 -0
- workers/mce/final_eval.py +173 -0
- workers/official_arenas/README.md +25 -0
- workers/official_arenas/__init__.py +1 -0
- workers/official_arenas/capture_records.py +110 -0
- workers/official_arenas/driver.py +619 -0
- workers/official_arenas/protocol.py +334 -0
- workers/official_arenas/providers/__init__.py +1 -0
- workers/official_arenas/providers/_live_history.py +228 -0
- workers/official_arenas/providers/_message_compat.py +77 -0
- workers/official_arenas/providers/appworld_ul.py +196 -0
- workers/official_arenas/providers/fixture.py +112 -0
- workers/official_arenas/providers/statebench.py +960 -0
- workers/official_arenas/providers/tau3.py +1672 -0
- workers/official_arenas/providers/tau_retail.py +586 -0
- workers/official_arenas/providers/toolsandbox.py +1321 -0
- workers/official_arenas/providers/tooltalk.py +699 -0
- workers/official_arenas/retail_capture.py +188 -0
- workers/official_arenas/retail_driver.py +275 -0
- workers/qrag/README.md +172 -0
- workers/qrag/__init__.py +1 -0
- workers/qrag/compatibility_manifest.json +64 -0
- workers/qrag/driver.py +673 -0
- workers/r2ag/README.md +154 -0
- workers/r2ag/__init__.py +5 -0
- workers/r2ag/compatibility_manifest.json +55 -0
- workers/r2ag/driver.py +1287 -0
- workers/r2ag/feature_schema.json +8 -0
- workers/r2ag/runtime.py +589 -0
- workers/recomp/README.md +120 -0
- workers/recomp/__init__.py +1 -0
- workers/recomp/compatibility_manifest.json +43 -0
- workers/recomp/driver.py +1653 -0
- workers/reformer/README.md +125 -0
- workers/reformer/__init__.py +1 -0
- workers/reformer/compatibility_manifest.json +68 -0
- workers/reformer/driver.py +1459 -0
- workers/reformer/induction.py +457 -0
- workers/rrr/README.md +126 -0
- workers/rrr/__init__.py +1 -0
- workers/rrr/compatibility_manifest.json +57 -0
- workers/rrr/driver.py +1133 -0
- workers/smartsearch/README.md +331 -0
- workers/smartsearch/__init__.py +1 -0
- workers/smartsearch/compatibility_manifest.json +41 -0
- workers/smartsearch/driver.py +1643 -0
|
@@ -0,0 +1,1516 @@
|
|
|
1
|
+
"""RQ-RAG tree inference with controlled and official-checkpoint profiles.
|
|
2
|
+
|
|
3
|
+
The pinned upstream source and released checkpoint have no declared license.
|
|
4
|
+
The official-checkpoint adapter therefore remains fail-closed until explicit
|
|
5
|
+
license clearance and the three audited inference fixes are attested.
|
|
6
|
+
|
|
7
|
+
``RQRAGMethodTreeSearch`` is a separate, runnable controlled profile. It keeps
|
|
8
|
+
the paper's four special-token actions, breadth-first query/action tree,
|
|
9
|
+
retrieval after every query action, and confidence/PPL/ensemble leaf selection,
|
|
10
|
+
while using AgentInputLab's replaceable ``LLMBackend`` and ``Retriever`` seams.
|
|
11
|
+
It is method-level reproduction infrastructure, not a claim of released-7B
|
|
12
|
+
checkpoint parity.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import math
|
|
18
|
+
from collections import deque
|
|
19
|
+
from collections.abc import Mapping, Sequence
|
|
20
|
+
from dataclasses import asdict, dataclass
|
|
21
|
+
from types import MappingProxyType
|
|
22
|
+
from typing import Any
|
|
23
|
+
|
|
24
|
+
from agentinputlab.backends.llm.base import GenerationRequest, GenerationResponse, LLMBackend
|
|
25
|
+
from agentinputlab.backends.retrieval.base import RetrievalResult, Retriever
|
|
26
|
+
from agentinputlab.contracts import (
|
|
27
|
+
AlgorithmOutputError,
|
|
28
|
+
BudgetExceededError,
|
|
29
|
+
CapabilityError,
|
|
30
|
+
CompositeResult,
|
|
31
|
+
ContextBlock,
|
|
32
|
+
ExecutionTrace,
|
|
33
|
+
ExternalDependencyError,
|
|
34
|
+
FrozenAgentState,
|
|
35
|
+
InterventionContext,
|
|
36
|
+
SemanticFamily,
|
|
37
|
+
TraceEvent,
|
|
38
|
+
TracePhase,
|
|
39
|
+
ValidationError,
|
|
40
|
+
)
|
|
41
|
+
from agentinputlab.contracts.serialization import content_hash, freeze_mapping
|
|
42
|
+
from agentinputlab.integrations.external_workers import WorkerClient
|
|
43
|
+
from agentinputlab.integrations.external_workers.method import (
|
|
44
|
+
MethodWorkerRequest,
|
|
45
|
+
call_worker,
|
|
46
|
+
finite_number,
|
|
47
|
+
mapping_sequence,
|
|
48
|
+
nonnegative_integer,
|
|
49
|
+
required_text,
|
|
50
|
+
resolve_worker,
|
|
51
|
+
string_tuple,
|
|
52
|
+
worker_fingerprint,
|
|
53
|
+
)
|
|
54
|
+
from agentinputlab.observability import external_usage_counters
|
|
55
|
+
|
|
56
|
+
RQ_RAG_UPSTREAM_COMMIT = "df5f2597eec9c229e1d54a6884bf236236870b32"
|
|
57
|
+
RQ_RAG_METHOD_RELEASE_ID = "recipe.rqrag.method_tree_search"
|
|
58
|
+
_UNCLEARED_LICENSE_IDS = frozenset({"", "NONE", "NOASSERTION", "UNKNOWN", "UNLICENSED"})
|
|
59
|
+
_QUERY_TYPES = frozenset({"rewrite", "decompose", "disambiguate", "direct_response"})
|
|
60
|
+
RQ_RAG_ACTION_TOKENS: Mapping[str, str] = freeze_mapping(
|
|
61
|
+
{
|
|
62
|
+
"rewrite": "[S_Rewritten_Query]",
|
|
63
|
+
"decompose": "[S_Decomposed_Query]",
|
|
64
|
+
"disambiguate": "[S_Disambiguated_Query]",
|
|
65
|
+
"direct_response": "[A_Response]",
|
|
66
|
+
}
|
|
67
|
+
)
|
|
68
|
+
_QUERY_ACTIONS = ("rewrite", "decompose", "disambiguate")
|
|
69
|
+
_RESERVED_PROTOCOL_TOKENS = (
|
|
70
|
+
*RQ_RAG_ACTION_TOKENS.values(),
|
|
71
|
+
"[EOS]",
|
|
72
|
+
"[R_Evidences]",
|
|
73
|
+
"[/R_Evidences]",
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _require_sha256(value: str, *, label: str) -> str:
|
|
78
|
+
if not isinstance(value, str):
|
|
79
|
+
raise ValidationError(f"{label} must be text")
|
|
80
|
+
normalized = value.strip().lower()
|
|
81
|
+
if len(normalized) != 64 or any(
|
|
82
|
+
character not in "0123456789abcdef" for character in normalized
|
|
83
|
+
):
|
|
84
|
+
raise ValidationError(f"{label} must be a lowercase SHA-256 digest")
|
|
85
|
+
return normalized
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
@dataclass(frozen=True, slots=True)
|
|
89
|
+
class RQRAGLicenseClearance:
|
|
90
|
+
"""Auditable approval for source, checkpoint, and training-data terms."""
|
|
91
|
+
|
|
92
|
+
source_license_id: str
|
|
93
|
+
checkpoint_license_id: str
|
|
94
|
+
dataset_license_id: str
|
|
95
|
+
approved_by: str
|
|
96
|
+
approval_reference: str
|
|
97
|
+
|
|
98
|
+
def __post_init__(self) -> None:
|
|
99
|
+
for field_name in (
|
|
100
|
+
"source_license_id",
|
|
101
|
+
"checkpoint_license_id",
|
|
102
|
+
"dataset_license_id",
|
|
103
|
+
):
|
|
104
|
+
raw_license_id = getattr(self, field_name)
|
|
105
|
+
if not isinstance(raw_license_id, str):
|
|
106
|
+
raise ValidationError(f"RQ-RAG {field_name} must be text")
|
|
107
|
+
license_id = raw_license_id.strip().upper()
|
|
108
|
+
if license_id in _UNCLEARED_LICENSE_IDS:
|
|
109
|
+
raise ValidationError(f"RQ-RAG {field_name} is not cleared")
|
|
110
|
+
if (
|
|
111
|
+
not isinstance(self.approved_by, str)
|
|
112
|
+
or not isinstance(self.approval_reference, str)
|
|
113
|
+
or not self.approved_by.strip()
|
|
114
|
+
or not self.approval_reference.strip()
|
|
115
|
+
):
|
|
116
|
+
raise ValidationError("RQ-RAG license clearance requires approver and reference")
|
|
117
|
+
|
|
118
|
+
@property
|
|
119
|
+
def clearance_id(self) -> str:
|
|
120
|
+
return content_hash(self)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
@dataclass(frozen=True, slots=True)
|
|
124
|
+
class RQRAGPatchAttestation:
|
|
125
|
+
"""Attest the minimal upstream fixes without redefining tree decoding."""
|
|
126
|
+
|
|
127
|
+
patch_manifest_sha256: str
|
|
128
|
+
attested_by: str
|
|
129
|
+
attestation_reference: str
|
|
130
|
+
tree_false_regression_passed: bool
|
|
131
|
+
oracle_false_regression_passed: bool
|
|
132
|
+
duplicate_stop_regression_passed: bool
|
|
133
|
+
semantic_patch_only: bool
|
|
134
|
+
upstream_commit: str = RQ_RAG_UPSTREAM_COMMIT
|
|
135
|
+
|
|
136
|
+
def __post_init__(self) -> None:
|
|
137
|
+
object.__setattr__(
|
|
138
|
+
self,
|
|
139
|
+
"patch_manifest_sha256",
|
|
140
|
+
_require_sha256(
|
|
141
|
+
self.patch_manifest_sha256,
|
|
142
|
+
label="RQ-RAG patch_manifest_sha256",
|
|
143
|
+
),
|
|
144
|
+
)
|
|
145
|
+
if self.upstream_commit != RQ_RAG_UPSTREAM_COMMIT:
|
|
146
|
+
raise ValidationError("RQ-RAG patch attestation targets the wrong upstream commit")
|
|
147
|
+
if (
|
|
148
|
+
not isinstance(self.attested_by, str)
|
|
149
|
+
or not isinstance(self.attestation_reference, str)
|
|
150
|
+
or not self.attested_by.strip()
|
|
151
|
+
or not self.attestation_reference.strip()
|
|
152
|
+
):
|
|
153
|
+
raise ValidationError("RQ-RAG patch attestation requires attester and reference")
|
|
154
|
+
if any(
|
|
155
|
+
value is not True
|
|
156
|
+
for value in (
|
|
157
|
+
self.tree_false_regression_passed,
|
|
158
|
+
self.oracle_false_regression_passed,
|
|
159
|
+
self.duplicate_stop_regression_passed,
|
|
160
|
+
self.semantic_patch_only,
|
|
161
|
+
)
|
|
162
|
+
):
|
|
163
|
+
raise ValidationError("RQ-RAG patch attestation must pass all fail-closed gates")
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
@dataclass(frozen=True, slots=True)
|
|
167
|
+
class RQRAGControls:
|
|
168
|
+
max_depth: int = 4
|
|
169
|
+
max_nodes: int = 32
|
|
170
|
+
max_documents: int = 100
|
|
171
|
+
max_output_tokens: int = 4096
|
|
172
|
+
|
|
173
|
+
def __post_init__(self) -> None:
|
|
174
|
+
values = (self.max_depth, self.max_nodes, self.max_documents, self.max_output_tokens)
|
|
175
|
+
if any(isinstance(value, bool) or not isinstance(value, int) for value in values):
|
|
176
|
+
raise ValidationError("RQ-RAG limits must be integers")
|
|
177
|
+
if min(*values) < 1:
|
|
178
|
+
raise ValidationError("RQ-RAG limits must be positive")
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
@dataclass(frozen=True, slots=True)
|
|
182
|
+
class RQRAGTreeNode:
|
|
183
|
+
node_id: str
|
|
184
|
+
parent_id: str | None
|
|
185
|
+
depth: int
|
|
186
|
+
query_type: str
|
|
187
|
+
control_token: str
|
|
188
|
+
query: str
|
|
189
|
+
document_ids: tuple[str, ...]
|
|
190
|
+
score: float
|
|
191
|
+
|
|
192
|
+
@classmethod
|
|
193
|
+
def from_mapping(cls, value: Mapping[str, Any]) -> RQRAGTreeNode:
|
|
194
|
+
parent = value.get("parent_id")
|
|
195
|
+
if parent is not None and (not isinstance(parent, str) or not parent.strip()):
|
|
196
|
+
raise AlgorithmOutputError("RQ-RAG parent_id must be null or non-empty text")
|
|
197
|
+
query_type = required_text(value, "query_type", label="RQ-RAG tree node")
|
|
198
|
+
if query_type not in _QUERY_TYPES:
|
|
199
|
+
raise AlgorithmOutputError("RQ-RAG tree node has an unsupported query_type")
|
|
200
|
+
return cls(
|
|
201
|
+
node_id=required_text(value, "node_id", label="RQ-RAG tree node"),
|
|
202
|
+
parent_id=parent,
|
|
203
|
+
depth=nonnegative_integer(value.get("depth"), label="RQ-RAG tree node depth"),
|
|
204
|
+
query_type=query_type,
|
|
205
|
+
control_token=required_text(
|
|
206
|
+
value,
|
|
207
|
+
"control_token",
|
|
208
|
+
label="RQ-RAG tree node",
|
|
209
|
+
),
|
|
210
|
+
query=required_text(value, "query", label="RQ-RAG tree node"),
|
|
211
|
+
document_ids=string_tuple(
|
|
212
|
+
value.get("document_ids", ()),
|
|
213
|
+
label="RQ-RAG tree node document_ids",
|
|
214
|
+
unique=True,
|
|
215
|
+
),
|
|
216
|
+
score=finite_number(value.get("score"), label="RQ-RAG tree node score"),
|
|
217
|
+
)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
@dataclass(frozen=True, slots=True)
|
|
221
|
+
class _RQRAGEvidence:
|
|
222
|
+
document_id: str
|
|
223
|
+
text: str
|
|
224
|
+
source: str
|
|
225
|
+
score: float | None
|
|
226
|
+
retrieved_by_node_ids: tuple[str, ...]
|
|
227
|
+
metadata: Mapping[str, Any]
|
|
228
|
+
|
|
229
|
+
def __post_init__(self) -> None:
|
|
230
|
+
object.__setattr__(self, "metadata", freeze_mapping(self.metadata))
|
|
231
|
+
|
|
232
|
+
@classmethod
|
|
233
|
+
def from_mapping(cls, value: Mapping[str, Any]) -> _RQRAGEvidence:
|
|
234
|
+
metadata = value.get("metadata", {})
|
|
235
|
+
if not isinstance(metadata, Mapping):
|
|
236
|
+
raise AlgorithmOutputError("RQ-RAG evidence metadata must be a mapping")
|
|
237
|
+
score_value = value.get("score")
|
|
238
|
+
return cls(
|
|
239
|
+
document_id=required_text(value, "document_id", label="RQ-RAG evidence"),
|
|
240
|
+
text=required_text(value, "text", label="RQ-RAG evidence"),
|
|
241
|
+
source=required_text(value, "source", label="RQ-RAG evidence"),
|
|
242
|
+
score=(
|
|
243
|
+
None
|
|
244
|
+
if score_value is None
|
|
245
|
+
else finite_number(score_value, label="RQ-RAG evidence score")
|
|
246
|
+
),
|
|
247
|
+
retrieved_by_node_ids=string_tuple(
|
|
248
|
+
value.get("retrieved_by_node_ids"),
|
|
249
|
+
label="RQ-RAG evidence retrieved_by_node_ids",
|
|
250
|
+
allow_empty=False,
|
|
251
|
+
unique=True,
|
|
252
|
+
),
|
|
253
|
+
metadata=dict(metadata),
|
|
254
|
+
)
|
|
255
|
+
|
|
256
|
+
def as_context_block(self) -> ContextBlock:
|
|
257
|
+
return ContextBlock(
|
|
258
|
+
block_id=self.document_id,
|
|
259
|
+
text=self.text,
|
|
260
|
+
source=self.source,
|
|
261
|
+
score=self.score,
|
|
262
|
+
metadata={
|
|
263
|
+
**dict(self.metadata),
|
|
264
|
+
"rqrag_retrieved_by_node_ids": self.retrieved_by_node_ids,
|
|
265
|
+
},
|
|
266
|
+
)
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
@dataclass(frozen=True, slots=True)
|
|
270
|
+
class RQRAGMethodControls:
|
|
271
|
+
"""Controls for the backend-neutral method-level tree search.
|
|
272
|
+
|
|
273
|
+
``max_depth`` counts retrieval actions, not the synthetic root or terminal
|
|
274
|
+
answer nodes. A depth of one therefore evaluates a direct answer plus one
|
|
275
|
+
rewrite, decomposition, and disambiguation branch, followed by an answer
|
|
276
|
+
for each branch.
|
|
277
|
+
"""
|
|
278
|
+
|
|
279
|
+
max_depth: int = 1
|
|
280
|
+
max_nodes: int = 32
|
|
281
|
+
max_documents: int = 100
|
|
282
|
+
retrieval_top_k: int = 5
|
|
283
|
+
max_output_tokens_per_call: int = 256
|
|
284
|
+
query_temperature: float = 1.0
|
|
285
|
+
answer_temperature: float = 0.0
|
|
286
|
+
selection_mode: str = "confidence"
|
|
287
|
+
query_actions: tuple[str, ...] = _QUERY_ACTIONS
|
|
288
|
+
allow_partial_tree: bool = False
|
|
289
|
+
|
|
290
|
+
def __post_init__(self) -> None:
|
|
291
|
+
integer_values = (
|
|
292
|
+
self.max_depth,
|
|
293
|
+
self.max_nodes,
|
|
294
|
+
self.max_documents,
|
|
295
|
+
self.retrieval_top_k,
|
|
296
|
+
self.max_output_tokens_per_call,
|
|
297
|
+
)
|
|
298
|
+
if any(isinstance(value, bool) or not isinstance(value, int) for value in integer_values):
|
|
299
|
+
raise ValidationError("RQ-RAG method controls must be integers")
|
|
300
|
+
if self.max_depth < 0:
|
|
301
|
+
raise ValidationError("RQ-RAG method max_depth must be non-negative")
|
|
302
|
+
if self.max_nodes < 2:
|
|
303
|
+
raise ValidationError("RQ-RAG method max_nodes must allow a root and answer")
|
|
304
|
+
if (
|
|
305
|
+
min(
|
|
306
|
+
self.max_documents,
|
|
307
|
+
self.retrieval_top_k,
|
|
308
|
+
self.max_output_tokens_per_call,
|
|
309
|
+
)
|
|
310
|
+
< 1
|
|
311
|
+
):
|
|
312
|
+
raise ValidationError("RQ-RAG method document/token limits must be positive")
|
|
313
|
+
temperatures = (self.query_temperature, self.answer_temperature)
|
|
314
|
+
if any(
|
|
315
|
+
isinstance(value, bool)
|
|
316
|
+
or not isinstance(value, (int, float))
|
|
317
|
+
or not math.isfinite(value)
|
|
318
|
+
or value < 0
|
|
319
|
+
for value in temperatures
|
|
320
|
+
):
|
|
321
|
+
raise ValidationError("RQ-RAG method temperatures must be finite and non-negative")
|
|
322
|
+
if self.selection_mode not in {"confidence", "ppl", "ensemble"}:
|
|
323
|
+
raise ValidationError("RQ-RAG selection_mode must be confidence, ppl, or ensemble")
|
|
324
|
+
if not isinstance(self.allow_partial_tree, bool):
|
|
325
|
+
raise ValidationError("RQ-RAG allow_partial_tree must be boolean")
|
|
326
|
+
actions = tuple(self.query_actions)
|
|
327
|
+
if (
|
|
328
|
+
not actions
|
|
329
|
+
or len(set(actions)) != len(actions)
|
|
330
|
+
or any(action not in _QUERY_ACTIONS for action in actions)
|
|
331
|
+
):
|
|
332
|
+
raise ValidationError(
|
|
333
|
+
"RQ-RAG query_actions must be unique rewrite/decompose/disambiguate actions"
|
|
334
|
+
)
|
|
335
|
+
object.__setattr__(self, "query_actions", actions)
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
@dataclass(frozen=True, slots=True)
|
|
339
|
+
class RQRAGCandidateScore:
|
|
340
|
+
"""Model-derived scores used by the paper's post-hoc leaf selectors."""
|
|
341
|
+
|
|
342
|
+
confidence: float | None
|
|
343
|
+
perplexity: float | None
|
|
344
|
+
source: str
|
|
345
|
+
|
|
346
|
+
def __post_init__(self) -> None:
|
|
347
|
+
if self.confidence is not None and (
|
|
348
|
+
isinstance(self.confidence, bool)
|
|
349
|
+
or not isinstance(self.confidence, (int, float))
|
|
350
|
+
or not math.isfinite(self.confidence)
|
|
351
|
+
or not 0 <= self.confidence <= 1
|
|
352
|
+
):
|
|
353
|
+
raise AlgorithmOutputError("RQ-RAG confidence must be finite in [0, 1]")
|
|
354
|
+
if self.perplexity is not None and (
|
|
355
|
+
isinstance(self.perplexity, bool)
|
|
356
|
+
or not isinstance(self.perplexity, (int, float))
|
|
357
|
+
or not math.isfinite(self.perplexity)
|
|
358
|
+
or self.perplexity <= 0
|
|
359
|
+
):
|
|
360
|
+
raise AlgorithmOutputError("RQ-RAG perplexity must be finite and positive")
|
|
361
|
+
if not isinstance(self.source, str) or not self.source.strip():
|
|
362
|
+
raise AlgorithmOutputError("RQ-RAG candidate score requires a source")
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
@dataclass(frozen=True, slots=True)
|
|
366
|
+
class _MethodPath:
|
|
367
|
+
node_id: str
|
|
368
|
+
retrieval_depth: int
|
|
369
|
+
prompt: str
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
@dataclass(frozen=True, slots=True)
|
|
373
|
+
class _AnswerCandidate:
|
|
374
|
+
node_id: str
|
|
375
|
+
answer: str
|
|
376
|
+
score: RQRAGCandidateScore
|
|
377
|
+
ordinal: int
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
@dataclass(slots=True)
|
|
381
|
+
class _EvidenceAccumulator:
|
|
382
|
+
document_id: str
|
|
383
|
+
text: str
|
|
384
|
+
source: str
|
|
385
|
+
score: float
|
|
386
|
+
metadata: Mapping[str, Any]
|
|
387
|
+
node_ids: list[str]
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
@dataclass(slots=True)
|
|
391
|
+
class RQRAGMethodTreeSearch:
|
|
392
|
+
"""Run RQ-RAG's action tree over replaceable model/retrieval backends.
|
|
393
|
+
|
|
394
|
+
The injected model must return a score for terminal answers, either through
|
|
395
|
+
``raw["rqrag_score"]`` (``confidence`` and/or ``perplexity``) or through
|
|
396
|
+
OpenAI-compatible token log-probabilities. Perplexity must explicitly have
|
|
397
|
+
``perplexity_scope="full_path"`` because upstream scores the whole candidate
|
|
398
|
+
path; completion-only API log-probabilities are accepted for controlled
|
|
399
|
+
confidence selection only. This keeps leaf selection model-derived and
|
|
400
|
+
prevents an unreported lexical heuristic from changing the treatment.
|
|
401
|
+
"""
|
|
402
|
+
|
|
403
|
+
controls: RQRAGMethodControls = RQRAGMethodControls()
|
|
404
|
+
method_id: str = "recipe.rqrag"
|
|
405
|
+
release_id: str = RQ_RAG_METHOD_RELEASE_ID
|
|
406
|
+
family: SemanticFamily = SemanticFamily.COMPOSITE
|
|
407
|
+
|
|
408
|
+
@staticmethod
|
|
409
|
+
def _require_backend(value: object) -> LLMBackend:
|
|
410
|
+
if value is None:
|
|
411
|
+
raise ExternalDependencyError("RQ-RAG method tree search requires an LLMBackend")
|
|
412
|
+
if (
|
|
413
|
+
not hasattr(value, "generate")
|
|
414
|
+
or not hasattr(value, "fingerprint")
|
|
415
|
+
or not hasattr(value, "model_id")
|
|
416
|
+
):
|
|
417
|
+
raise CapabilityError("RQ-RAG backend does not implement LLMBackend")
|
|
418
|
+
return value # type: ignore[return-value]
|
|
419
|
+
|
|
420
|
+
@staticmethod
|
|
421
|
+
def _require_retriever(value: object) -> Retriever:
|
|
422
|
+
if value is None:
|
|
423
|
+
raise ExternalDependencyError("RQ-RAG method tree search requires a Retriever")
|
|
424
|
+
if (
|
|
425
|
+
not hasattr(value, "retrieve")
|
|
426
|
+
or not hasattr(value, "fingerprint")
|
|
427
|
+
or not hasattr(value, "retriever_id")
|
|
428
|
+
):
|
|
429
|
+
raise CapabilityError("RQ-RAG retriever does not implement Retriever")
|
|
430
|
+
return value # type: ignore[return-value]
|
|
431
|
+
|
|
432
|
+
def run(
|
|
433
|
+
self,
|
|
434
|
+
state: FrozenAgentState,
|
|
435
|
+
context: InterventionContext,
|
|
436
|
+
) -> CompositeResult:
|
|
437
|
+
if not state.query.strip():
|
|
438
|
+
raise ValidationError("RQ-RAG requires a non-empty query")
|
|
439
|
+
if state.tool_names:
|
|
440
|
+
raise ValidationError(
|
|
441
|
+
"RQ-RAG method tree search does not consume tool schemas; use a retrieval backend"
|
|
442
|
+
)
|
|
443
|
+
_reject_protocol_tokens(state.query, label="RQ-RAG input query")
|
|
444
|
+
backend = self._require_backend(context.backend)
|
|
445
|
+
retriever = self._require_retriever(context.retriever)
|
|
446
|
+
backend_identity = _dependency_identity(backend.fingerprint(), label="RQ-RAG backend")
|
|
447
|
+
retriever_identity = _dependency_identity(
|
|
448
|
+
retriever.fingerprint(),
|
|
449
|
+
label="RQ-RAG retriever",
|
|
450
|
+
)
|
|
451
|
+
|
|
452
|
+
initial_prompt = _method_initial_prompt(state)
|
|
453
|
+
root = RQRAGTreeNode(
|
|
454
|
+
node_id="rqrag-node-0000",
|
|
455
|
+
parent_id=None,
|
|
456
|
+
depth=0,
|
|
457
|
+
query_type="root",
|
|
458
|
+
control_token="[ROOT]",
|
|
459
|
+
query=state.query.strip(),
|
|
460
|
+
document_ids=(),
|
|
461
|
+
score=0.0,
|
|
462
|
+
)
|
|
463
|
+
nodes: list[RQRAGTreeNode] = [root]
|
|
464
|
+
frontier: deque[_MethodPath] = deque((_MethodPath(root.node_id, 0, initial_prompt),))
|
|
465
|
+
candidates: list[_AnswerCandidate] = []
|
|
466
|
+
evidence_accumulators: dict[str, _EvidenceAccumulator] = {}
|
|
467
|
+
evidence_order: list[str] = []
|
|
468
|
+
model_calls = 0
|
|
469
|
+
retrieval_calls = 0
|
|
470
|
+
input_tokens = 0
|
|
471
|
+
output_tokens = 0
|
|
472
|
+
known_cost_usd = 0.0
|
|
473
|
+
unpriced_model_calls = 0
|
|
474
|
+
escaped_evidence_tokens = 0
|
|
475
|
+
incomplete = False
|
|
476
|
+
incomplete_reasons: set[str] = set()
|
|
477
|
+
|
|
478
|
+
def can_generate() -> bool:
|
|
479
|
+
return (
|
|
480
|
+
model_calls < state.budget.max_model_calls
|
|
481
|
+
and output_tokens < state.budget.max_output_tokens
|
|
482
|
+
and len(nodes) < self.controls.max_nodes
|
|
483
|
+
)
|
|
484
|
+
|
|
485
|
+
def generate_action(
|
|
486
|
+
path: _MethodPath,
|
|
487
|
+
action: str,
|
|
488
|
+
) -> tuple[str, RQRAGCandidateScore, GenerationResponse]:
|
|
489
|
+
nonlocal model_calls
|
|
490
|
+
nonlocal input_tokens
|
|
491
|
+
nonlocal output_tokens
|
|
492
|
+
nonlocal known_cost_usd
|
|
493
|
+
nonlocal unpriced_model_calls
|
|
494
|
+
if not can_generate():
|
|
495
|
+
raise BudgetExceededError("RQ-RAG has no remaining model/tree budget")
|
|
496
|
+
control_token = RQ_RAG_ACTION_TOKENS[action]
|
|
497
|
+
remaining_output = state.budget.max_output_tokens - output_tokens
|
|
498
|
+
request = GenerationRequest(
|
|
499
|
+
prompt=f"{path.prompt}{control_token}",
|
|
500
|
+
temperature=(
|
|
501
|
+
self.controls.answer_temperature
|
|
502
|
+
if action == "direct_response"
|
|
503
|
+
else self.controls.query_temperature
|
|
504
|
+
),
|
|
505
|
+
max_tokens=min(
|
|
506
|
+
self.controls.max_output_tokens_per_call,
|
|
507
|
+
remaining_output,
|
|
508
|
+
),
|
|
509
|
+
seed=context.seed + model_calls,
|
|
510
|
+
stop=("[EOS]",),
|
|
511
|
+
logprobs=(
|
|
512
|
+
action == "direct_response"
|
|
513
|
+
and self.controls.selection_mode in {"confidence", "ensemble"}
|
|
514
|
+
),
|
|
515
|
+
top_logprobs=(
|
|
516
|
+
5
|
|
517
|
+
if action == "direct_response"
|
|
518
|
+
and self.controls.selection_mode in {"confidence", "ensemble"}
|
|
519
|
+
else None
|
|
520
|
+
),
|
|
521
|
+
metadata={
|
|
522
|
+
"method_id": self.method_id,
|
|
523
|
+
"release_id": self.release_id,
|
|
524
|
+
"rqrag_action": action,
|
|
525
|
+
"rqrag_control_token": control_token,
|
|
526
|
+
"rqrag_parent_node_id": path.node_id,
|
|
527
|
+
"rqrag_retrieval_depth": path.retrieval_depth,
|
|
528
|
+
"paper_checkpoint_parity": False,
|
|
529
|
+
},
|
|
530
|
+
)
|
|
531
|
+
response = backend.generate(request)
|
|
532
|
+
model_calls += 1
|
|
533
|
+
if not isinstance(response, GenerationResponse):
|
|
534
|
+
raise AlgorithmOutputError("RQ-RAG backend must return a GenerationResponse")
|
|
535
|
+
generated = _parse_action_completion(
|
|
536
|
+
response.text,
|
|
537
|
+
action=action,
|
|
538
|
+
control_token=control_token,
|
|
539
|
+
)
|
|
540
|
+
score = _candidate_score(response)
|
|
541
|
+
input_tokens += response.usage.input_tokens
|
|
542
|
+
output_tokens += response.usage.output_tokens
|
|
543
|
+
if response.usage.output_tokens > request.max_tokens:
|
|
544
|
+
raise BudgetExceededError(
|
|
545
|
+
"RQ-RAG backend reported more output tokens than requested"
|
|
546
|
+
)
|
|
547
|
+
if output_tokens > state.budget.max_output_tokens:
|
|
548
|
+
raise BudgetExceededError("RQ-RAG exceeded max_output_tokens")
|
|
549
|
+
if (
|
|
550
|
+
state.budget.max_context_tokens is not None
|
|
551
|
+
and input_tokens > state.budget.max_context_tokens
|
|
552
|
+
):
|
|
553
|
+
raise BudgetExceededError("RQ-RAG exceeded max_context_tokens")
|
|
554
|
+
call_cost = response.usage.cost_usd
|
|
555
|
+
pricing = context.pricing_catalog
|
|
556
|
+
if (
|
|
557
|
+
call_cost is None
|
|
558
|
+
and pricing is not None
|
|
559
|
+
and hasattr(pricing, "has_price")
|
|
560
|
+
and pricing.has_price(backend.model_id)
|
|
561
|
+
):
|
|
562
|
+
call_cost = pricing.estimate(
|
|
563
|
+
backend.model_id,
|
|
564
|
+
input_tokens=response.usage.input_tokens,
|
|
565
|
+
output_tokens=response.usage.output_tokens,
|
|
566
|
+
)
|
|
567
|
+
if call_cost is None:
|
|
568
|
+
unpriced_model_calls += 1
|
|
569
|
+
if state.budget.max_cost_usd is not None:
|
|
570
|
+
raise BudgetExceededError(
|
|
571
|
+
"RQ-RAG cannot enforce max_cost_usd for an unpriced model call"
|
|
572
|
+
)
|
|
573
|
+
else:
|
|
574
|
+
known_cost_usd += call_cost
|
|
575
|
+
if (
|
|
576
|
+
state.budget.max_cost_usd is not None
|
|
577
|
+
and known_cost_usd > state.budget.max_cost_usd
|
|
578
|
+
):
|
|
579
|
+
raise BudgetExceededError("RQ-RAG exceeded max_cost_usd")
|
|
580
|
+
return generated, score, response
|
|
581
|
+
|
|
582
|
+
while frontier:
|
|
583
|
+
path = frontier.popleft()
|
|
584
|
+
if not can_generate():
|
|
585
|
+
incomplete = True
|
|
586
|
+
incomplete_reasons.add("model_or_tree_budget")
|
|
587
|
+
break
|
|
588
|
+
|
|
589
|
+
answer, answer_score, _ = generate_action(path, "direct_response")
|
|
590
|
+
_require_selection_score(answer_score, self.controls.selection_mode)
|
|
591
|
+
answer_node = RQRAGTreeNode(
|
|
592
|
+
node_id=_next_node_id(len(nodes)),
|
|
593
|
+
parent_id=path.node_id,
|
|
594
|
+
depth=path.retrieval_depth + 1,
|
|
595
|
+
query_type="direct_response",
|
|
596
|
+
control_token=RQ_RAG_ACTION_TOKENS["direct_response"],
|
|
597
|
+
query=answer,
|
|
598
|
+
document_ids=(),
|
|
599
|
+
score=_tree_score(answer_score, self.controls.selection_mode),
|
|
600
|
+
)
|
|
601
|
+
nodes.append(answer_node)
|
|
602
|
+
candidates.append(
|
|
603
|
+
_AnswerCandidate(
|
|
604
|
+
node_id=answer_node.node_id,
|
|
605
|
+
answer=answer,
|
|
606
|
+
score=answer_score,
|
|
607
|
+
ordinal=len(candidates),
|
|
608
|
+
)
|
|
609
|
+
)
|
|
610
|
+
|
|
611
|
+
if path.retrieval_depth >= self.controls.max_depth:
|
|
612
|
+
continue
|
|
613
|
+
for action in self.controls.query_actions:
|
|
614
|
+
if retrieval_calls >= state.budget.max_retrieval_calls:
|
|
615
|
+
incomplete = True
|
|
616
|
+
incomplete_reasons.add("retrieval_budget")
|
|
617
|
+
break
|
|
618
|
+
if not can_generate():
|
|
619
|
+
incomplete = True
|
|
620
|
+
incomplete_reasons.add("model_or_tree_budget")
|
|
621
|
+
break
|
|
622
|
+
query, query_score, _ = generate_action(path, action)
|
|
623
|
+
raw_results = tuple(retriever.retrieve(query, top_k=self.controls.retrieval_top_k))
|
|
624
|
+
retrieval_calls += 1
|
|
625
|
+
results = _validate_retrieval_results(
|
|
626
|
+
raw_results,
|
|
627
|
+
top_k=self.controls.retrieval_top_k,
|
|
628
|
+
)
|
|
629
|
+
query_node_id = _next_node_id(len(nodes))
|
|
630
|
+
document_ids: list[str] = []
|
|
631
|
+
formatted_evidence: list[str] = []
|
|
632
|
+
for result in results:
|
|
633
|
+
document = result.document
|
|
634
|
+
accumulator = evidence_accumulators.get(document.doc_id)
|
|
635
|
+
if accumulator is None:
|
|
636
|
+
if len(evidence_accumulators) >= self.controls.max_documents:
|
|
637
|
+
incomplete = True
|
|
638
|
+
incomplete_reasons.add("document_cap")
|
|
639
|
+
continue
|
|
640
|
+
accumulator = _EvidenceAccumulator(
|
|
641
|
+
document_id=document.doc_id,
|
|
642
|
+
text=document.text,
|
|
643
|
+
source=f"rqrag:{retriever.retriever_id}",
|
|
644
|
+
score=float(result.score),
|
|
645
|
+
metadata={
|
|
646
|
+
"title": document.title,
|
|
647
|
+
"corpus_revision": document.corpus_revision,
|
|
648
|
+
"rank": result.rank,
|
|
649
|
+
},
|
|
650
|
+
node_ids=[],
|
|
651
|
+
)
|
|
652
|
+
evidence_accumulators[document.doc_id] = accumulator
|
|
653
|
+
evidence_order.append(document.doc_id)
|
|
654
|
+
else:
|
|
655
|
+
_validate_repeated_document(accumulator, result)
|
|
656
|
+
if query_node_id not in accumulator.node_ids:
|
|
657
|
+
accumulator.node_ids.append(query_node_id)
|
|
658
|
+
if document.doc_id not in document_ids:
|
|
659
|
+
document_ids.append(document.doc_id)
|
|
660
|
+
title = document.title.strip() or document.doc_id
|
|
661
|
+
escaped_title, title_count = _escape_protocol_tokens(title)
|
|
662
|
+
escaped_text, text_count = _escape_protocol_tokens(document.text)
|
|
663
|
+
escaped_evidence_tokens += title_count + text_count
|
|
664
|
+
formatted_evidence.append(f"Title: {escaped_title}\nText: {escaped_text}")
|
|
665
|
+
query_node = RQRAGTreeNode(
|
|
666
|
+
node_id=query_node_id,
|
|
667
|
+
parent_id=path.node_id,
|
|
668
|
+
depth=path.retrieval_depth + 1,
|
|
669
|
+
query_type=action,
|
|
670
|
+
control_token=RQ_RAG_ACTION_TOKENS[action],
|
|
671
|
+
query=query,
|
|
672
|
+
document_ids=tuple(document_ids),
|
|
673
|
+
score=_tree_score(query_score, self.controls.selection_mode),
|
|
674
|
+
)
|
|
675
|
+
nodes.append(query_node)
|
|
676
|
+
evidence_text = "\n".join(formatted_evidence)
|
|
677
|
+
next_prompt = (
|
|
678
|
+
f"{path.prompt}{RQ_RAG_ACTION_TOKENS[action]}{query}[EOS]"
|
|
679
|
+
f"[R_Evidences]{evidence_text}[/R_Evidences]\n"
|
|
680
|
+
)
|
|
681
|
+
frontier.append(
|
|
682
|
+
_MethodPath(
|
|
683
|
+
node_id=query_node.node_id,
|
|
684
|
+
retrieval_depth=path.retrieval_depth + 1,
|
|
685
|
+
prompt=next_prompt,
|
|
686
|
+
)
|
|
687
|
+
)
|
|
688
|
+
|
|
689
|
+
if not candidates:
|
|
690
|
+
raise BudgetExceededError("RQ-RAG could not generate any terminal answer")
|
|
691
|
+
if incomplete and not self.controls.allow_partial_tree:
|
|
692
|
+
reasons = ", ".join(sorted(incomplete_reasons)) or "unknown"
|
|
693
|
+
raise BudgetExceededError(
|
|
694
|
+
"RQ-RAG tree is incomplete and non-comparable under the current budget "
|
|
695
|
+
f"({reasons}); set allow_partial_tree=True only for explicitly anytime runs"
|
|
696
|
+
)
|
|
697
|
+
selected = _select_candidate(candidates, self.controls.selection_mode)
|
|
698
|
+
node_by_id = {node.node_id: node for node in nodes}
|
|
699
|
+
selected_path = _node_ancestry(selected.node_id, node_by_id)
|
|
700
|
+
validation_controls = RQRAGControls(
|
|
701
|
+
max_depth=self.controls.max_depth + 1,
|
|
702
|
+
max_nodes=self.controls.max_nodes,
|
|
703
|
+
max_documents=self.controls.max_documents,
|
|
704
|
+
max_output_tokens=max(1, state.budget.max_output_tokens),
|
|
705
|
+
)
|
|
706
|
+
_validate_tree(
|
|
707
|
+
tuple(nodes),
|
|
708
|
+
selected_node_id=selected.node_id,
|
|
709
|
+
selected_path=selected_path,
|
|
710
|
+
controls=validation_controls,
|
|
711
|
+
)
|
|
712
|
+
evidence_records = tuple(
|
|
713
|
+
_RQRAGEvidence(
|
|
714
|
+
document_id=document_id,
|
|
715
|
+
text=evidence_accumulators[document_id].text,
|
|
716
|
+
source=evidence_accumulators[document_id].source,
|
|
717
|
+
score=evidence_accumulators[document_id].score,
|
|
718
|
+
retrieved_by_node_ids=tuple(evidence_accumulators[document_id].node_ids),
|
|
719
|
+
metadata=evidence_accumulators[document_id].metadata,
|
|
720
|
+
)
|
|
721
|
+
for document_id in evidence_order
|
|
722
|
+
)
|
|
723
|
+
evidence_by_id = _validate_document_lineage(tuple(nodes), evidence_records)
|
|
724
|
+
selected_document_ids = _selected_document_order(tuple(nodes), selected_path)
|
|
725
|
+
evidence = tuple(
|
|
726
|
+
evidence_by_id[document_id].as_context_block() for document_id in selected_document_ids
|
|
727
|
+
)
|
|
728
|
+
termination = "budget_exhausted" if incomplete else "selected_leaf"
|
|
729
|
+
trace_events: list[TraceEvent] = [
|
|
730
|
+
TraceEvent(
|
|
731
|
+
TracePhase.PREPARE,
|
|
732
|
+
"method_tree_configuration",
|
|
733
|
+
{
|
|
734
|
+
"upstream_commit": RQ_RAG_UPSTREAM_COMMIT,
|
|
735
|
+
"action_tokens": dict(RQ_RAG_ACTION_TOKENS),
|
|
736
|
+
"controls": asdict(self.controls),
|
|
737
|
+
"backend": backend_identity,
|
|
738
|
+
"retriever": retriever_identity,
|
|
739
|
+
"input_context_blocks": len(state.context),
|
|
740
|
+
"input_messages": len(state.messages),
|
|
741
|
+
"metadata_forwarded": False,
|
|
742
|
+
"paper_checkpoint_parity": False,
|
|
743
|
+
},
|
|
744
|
+
)
|
|
745
|
+
]
|
|
746
|
+
trace_events.extend(
|
|
747
|
+
TraceEvent(
|
|
748
|
+
TracePhase.OBSERVE,
|
|
749
|
+
"tree_node",
|
|
750
|
+
{
|
|
751
|
+
"node_id": node.node_id,
|
|
752
|
+
"parent_id": node.parent_id,
|
|
753
|
+
"depth": node.depth,
|
|
754
|
+
"query_type": node.query_type,
|
|
755
|
+
"control_token": node.control_token,
|
|
756
|
+
"query": node.query,
|
|
757
|
+
"document_ids": node.document_ids,
|
|
758
|
+
"score": node.score,
|
|
759
|
+
},
|
|
760
|
+
)
|
|
761
|
+
for node in nodes
|
|
762
|
+
)
|
|
763
|
+
trace_events.append(
|
|
764
|
+
TraceEvent(
|
|
765
|
+
TracePhase.FINALIZE,
|
|
766
|
+
"tree_selection",
|
|
767
|
+
{
|
|
768
|
+
"selected_node_id": selected.node_id,
|
|
769
|
+
"selected_path": selected_path,
|
|
770
|
+
"selected_document_ids": selected_document_ids,
|
|
771
|
+
"selection_mode": self.controls.selection_mode,
|
|
772
|
+
"allow_partial_tree": self.controls.allow_partial_tree,
|
|
773
|
+
"confidence": selected.score.confidence,
|
|
774
|
+
"perplexity": selected.score.perplexity,
|
|
775
|
+
"score_source": selected.score.source,
|
|
776
|
+
"termination": termination,
|
|
777
|
+
"incomplete_reasons": tuple(sorted(incomplete_reasons)),
|
|
778
|
+
},
|
|
779
|
+
)
|
|
780
|
+
)
|
|
781
|
+
warnings = ["controlled method-level RQ-RAG; no released-7B checkpoint parity claim"]
|
|
782
|
+
if incomplete:
|
|
783
|
+
warnings.append("partial anytime tree; exclude from complete-tree confirmatory cells")
|
|
784
|
+
if unpriced_model_calls:
|
|
785
|
+
warnings.append("one or more model calls did not report price")
|
|
786
|
+
if escaped_evidence_tokens:
|
|
787
|
+
warnings.append("reserved RQ-RAG tokens in retrieved text were prompt-escaped")
|
|
788
|
+
trace = ExecutionTrace(
|
|
789
|
+
method_id=self.method_id,
|
|
790
|
+
release_id=self.release_id,
|
|
791
|
+
events=tuple(trace_events),
|
|
792
|
+
counters={
|
|
793
|
+
"model_calls": model_calls,
|
|
794
|
+
"retrieval_calls": retrieval_calls,
|
|
795
|
+
"input_tokens": input_tokens,
|
|
796
|
+
"output_tokens": output_tokens,
|
|
797
|
+
"cost_usd": known_cost_usd,
|
|
798
|
+
"unpriced_model_calls": unpriced_model_calls,
|
|
799
|
+
"tree_nodes": len(nodes),
|
|
800
|
+
"tree_depth": max(node.depth for node in nodes),
|
|
801
|
+
"answer_candidates": len(candidates),
|
|
802
|
+
"selected_documents": len(evidence),
|
|
803
|
+
"escaped_evidence_tokens": escaped_evidence_tokens,
|
|
804
|
+
},
|
|
805
|
+
warnings=tuple(warnings),
|
|
806
|
+
)
|
|
807
|
+
treatment_identity = content_hash(
|
|
808
|
+
{
|
|
809
|
+
"upstream_commit": RQ_RAG_UPSTREAM_COMMIT,
|
|
810
|
+
"release_id": self.release_id,
|
|
811
|
+
"controls": asdict(self.controls),
|
|
812
|
+
"backend": backend_identity,
|
|
813
|
+
"retriever": retriever_identity,
|
|
814
|
+
}
|
|
815
|
+
)
|
|
816
|
+
return CompositeResult(
|
|
817
|
+
method_id=self.method_id,
|
|
818
|
+
release_id=self.release_id,
|
|
819
|
+
source_state_hash=state.state_hash,
|
|
820
|
+
final_answer=selected.answer,
|
|
821
|
+
evidence=evidence,
|
|
822
|
+
trace=trace,
|
|
823
|
+
implementation_identity=(
|
|
824
|
+
f"controlled_method_tree:chanchimin/RQ-RAG@{RQ_RAG_UPSTREAM_COMMIT}"
|
|
825
|
+
),
|
|
826
|
+
profile_kind="controlled_adapter",
|
|
827
|
+
metadata={
|
|
828
|
+
"implementation_origin": "controlled_clean_room_method_tree",
|
|
829
|
+
"paper_checkpoint_parity": False,
|
|
830
|
+
"treatment_identity_sha256": treatment_identity,
|
|
831
|
+
"backend_fingerprint": backend_identity,
|
|
832
|
+
"retriever_fingerprint": retriever_identity,
|
|
833
|
+
"selected_node_id": selected.node_id,
|
|
834
|
+
"selected_path": selected_path,
|
|
835
|
+
"selected_document_ids": selected_document_ids,
|
|
836
|
+
"selection_mode": self.controls.selection_mode,
|
|
837
|
+
"allow_partial_tree": self.controls.allow_partial_tree,
|
|
838
|
+
"comparison_eligible": not incomplete,
|
|
839
|
+
"score_source": selected.score.source,
|
|
840
|
+
"termination": termination,
|
|
841
|
+
"incomplete_reasons": tuple(sorted(incomplete_reasons)),
|
|
842
|
+
"oracle": False,
|
|
843
|
+
"tree_decode": True,
|
|
844
|
+
},
|
|
845
|
+
)
|
|
846
|
+
|
|
847
|
+
|
|
848
|
+
@dataclass(slots=True)
|
|
849
|
+
class RQRAGReleased7BExternal:
|
|
850
|
+
"""Preserve released RQ-RAG tree inference behind explicit legal gates."""
|
|
851
|
+
|
|
852
|
+
client: WorkerClient | None = None
|
|
853
|
+
checkpoint_revision: str = ""
|
|
854
|
+
tokenizer_revision: str = ""
|
|
855
|
+
retriever_revision: str = ""
|
|
856
|
+
token_protocol_sha256: str = ""
|
|
857
|
+
license_clearance: RQRAGLicenseClearance | None = None
|
|
858
|
+
patch_attestation: RQRAGPatchAttestation | None = None
|
|
859
|
+
controls: RQRAGControls = RQRAGControls()
|
|
860
|
+
method_id: str = "recipe.rqrag"
|
|
861
|
+
release_id: str = "recipe.rqrag.released_7b_external"
|
|
862
|
+
family: SemanticFamily = SemanticFamily.COMPOSITE
|
|
863
|
+
|
|
864
|
+
def __post_init__(self) -> None:
|
|
865
|
+
for name in ("checkpoint_revision", "tokenizer_revision", "retriever_revision"):
|
|
866
|
+
value = getattr(self, name)
|
|
867
|
+
if not isinstance(value, str) or not value.strip():
|
|
868
|
+
raise ValidationError(f"RQ-RAG {name} must not be empty")
|
|
869
|
+
self.token_protocol_sha256 = _require_sha256(
|
|
870
|
+
self.token_protocol_sha256,
|
|
871
|
+
label="RQ-RAG token_protocol_sha256",
|
|
872
|
+
)
|
|
873
|
+
|
|
874
|
+
def run(
|
|
875
|
+
self,
|
|
876
|
+
state: FrozenAgentState,
|
|
877
|
+
context: InterventionContext,
|
|
878
|
+
) -> CompositeResult:
|
|
879
|
+
if self.license_clearance is None:
|
|
880
|
+
raise ExternalDependencyError(
|
|
881
|
+
"RQ-RAG is license-gated; source/checkpoint/data clearance is required"
|
|
882
|
+
)
|
|
883
|
+
if self.patch_attestation is None:
|
|
884
|
+
raise ExternalDependencyError("RQ-RAG requires the audited three-fix patch attestation")
|
|
885
|
+
if not state.query.strip():
|
|
886
|
+
raise ValidationError("RQ-RAG requires a non-empty query")
|
|
887
|
+
worker = resolve_worker(self.client, context, label="RQ-RAG")
|
|
888
|
+
identity = worker_fingerprint(worker, label="RQ-RAG")
|
|
889
|
+
request = MethodWorkerRequest(
|
|
890
|
+
method_id=self.method_id,
|
|
891
|
+
release_id=self.release_id,
|
|
892
|
+
upstream_commit=RQ_RAG_UPSTREAM_COMMIT,
|
|
893
|
+
state_hash=state.state_hash,
|
|
894
|
+
payload={
|
|
895
|
+
"task_id": state.task_id,
|
|
896
|
+
"query": state.query,
|
|
897
|
+
"instruction": state.instruction,
|
|
898
|
+
"messages": state.messages,
|
|
899
|
+
"checkpoint_revision": self.checkpoint_revision,
|
|
900
|
+
"tokenizer_revision": self.tokenizer_revision,
|
|
901
|
+
"retriever_revision": self.retriever_revision,
|
|
902
|
+
"token_protocol_sha256": self.token_protocol_sha256,
|
|
903
|
+
"license_clearance_id": self.license_clearance.clearance_id,
|
|
904
|
+
"patch_attestation": asdict(self.patch_attestation),
|
|
905
|
+
"controls": asdict(self.controls),
|
|
906
|
+
"seed": context.seed,
|
|
907
|
+
"oracle": False,
|
|
908
|
+
"tree_decode": True,
|
|
909
|
+
},
|
|
910
|
+
dependency_fingerprints={"worker": identity},
|
|
911
|
+
minimum_reservation={"model_calls": 1},
|
|
912
|
+
protocol_version="agentinputlab.rqrag.v1",
|
|
913
|
+
)
|
|
914
|
+
response = call_worker(worker, request)
|
|
915
|
+
payload = response.payload
|
|
916
|
+
for key, expected in (
|
|
917
|
+
("checkpoint_revision", self.checkpoint_revision),
|
|
918
|
+
("tokenizer_revision", self.tokenizer_revision),
|
|
919
|
+
("retriever_revision", self.retriever_revision),
|
|
920
|
+
("token_protocol_sha256", self.token_protocol_sha256),
|
|
921
|
+
("license_clearance_id", self.license_clearance.clearance_id),
|
|
922
|
+
("patch_manifest_sha256", self.patch_attestation.patch_manifest_sha256),
|
|
923
|
+
):
|
|
924
|
+
if required_text(payload, key, label="RQ-RAG response") != expected:
|
|
925
|
+
raise AlgorithmOutputError(f"RQ-RAG {key} mismatch")
|
|
926
|
+
if payload.get("tree_decode") is not True or payload.get("oracle") is not False:
|
|
927
|
+
raise AlgorithmOutputError("RQ-RAG must use non-oracle tree decoding")
|
|
928
|
+
_validate_regression_echo(payload.get("regression_gates"))
|
|
929
|
+
|
|
930
|
+
nodes = tuple(
|
|
931
|
+
RQRAGTreeNode.from_mapping(row)
|
|
932
|
+
for row in mapping_sequence(payload.get("nodes"), label="RQ-RAG nodes")
|
|
933
|
+
)
|
|
934
|
+
selected_node_id = required_text(
|
|
935
|
+
payload,
|
|
936
|
+
"selected_node_id",
|
|
937
|
+
label="RQ-RAG response",
|
|
938
|
+
)
|
|
939
|
+
selected_path = string_tuple(
|
|
940
|
+
payload.get("selected_path"),
|
|
941
|
+
label="RQ-RAG selected_path",
|
|
942
|
+
allow_empty=False,
|
|
943
|
+
unique=True,
|
|
944
|
+
)
|
|
945
|
+
_validate_tree(
|
|
946
|
+
nodes,
|
|
947
|
+
selected_node_id=selected_node_id,
|
|
948
|
+
selected_path=selected_path,
|
|
949
|
+
controls=self.controls,
|
|
950
|
+
)
|
|
951
|
+
evidence_records = tuple(
|
|
952
|
+
_RQRAGEvidence.from_mapping(row)
|
|
953
|
+
for row in mapping_sequence(payload.get("evidence"), label="RQ-RAG evidence")
|
|
954
|
+
)
|
|
955
|
+
evidence_by_id = _validate_document_lineage(nodes, evidence_records)
|
|
956
|
+
if len(evidence_records) > self.controls.max_documents:
|
|
957
|
+
raise AlgorithmOutputError("RQ-RAG tree evidence exceeded max_documents")
|
|
958
|
+
selected_document_ids = string_tuple(
|
|
959
|
+
payload.get("selected_document_ids"),
|
|
960
|
+
label="RQ-RAG selected_document_ids",
|
|
961
|
+
unique=True,
|
|
962
|
+
)
|
|
963
|
+
expected_selected = _selected_document_order(nodes, selected_path)
|
|
964
|
+
if selected_document_ids != expected_selected:
|
|
965
|
+
raise AlgorithmOutputError(
|
|
966
|
+
"RQ-RAG selected documents disagree with the selected tree path"
|
|
967
|
+
)
|
|
968
|
+
if len(selected_document_ids) > self.controls.max_documents:
|
|
969
|
+
raise AlgorithmOutputError("RQ-RAG selected evidence exceeds max_documents")
|
|
970
|
+
|
|
971
|
+
retrieval_node_ids = string_tuple(
|
|
972
|
+
payload.get("retrieval_call_node_ids"),
|
|
973
|
+
label="RQ-RAG retrieval_call_node_ids",
|
|
974
|
+
unique=True,
|
|
975
|
+
)
|
|
976
|
+
expected_retrieval_nodes = tuple(
|
|
977
|
+
node.node_id for node in nodes if node.query_type != "direct_response"
|
|
978
|
+
)
|
|
979
|
+
if retrieval_node_ids != expected_retrieval_nodes:
|
|
980
|
+
raise AlgorithmOutputError(
|
|
981
|
+
"RQ-RAG retrieval-call lineage disagrees with non-terminal query nodes"
|
|
982
|
+
)
|
|
983
|
+
if response.usage.retrieval_calls != len(retrieval_node_ids):
|
|
984
|
+
raise AlgorithmOutputError("RQ-RAG retrieval usage disagrees with tree trajectory")
|
|
985
|
+
if response.usage.model_calls < 1:
|
|
986
|
+
raise AlgorithmOutputError("RQ-RAG inference requires a model call")
|
|
987
|
+
|
|
988
|
+
selection_mode = required_text(payload, "selection_mode", label="RQ-RAG response")
|
|
989
|
+
if selection_mode not in {"ppl", "confidence", "ensemble"}:
|
|
990
|
+
raise AlgorithmOutputError("RQ-RAG selection_mode is unsupported")
|
|
991
|
+
final_answer = required_text(payload, "final_answer", label="RQ-RAG response")
|
|
992
|
+
termination = required_text(payload, "termination", label="RQ-RAG response")
|
|
993
|
+
if termination not in {"selected_leaf", "budget_exhausted", "model_stop"}:
|
|
994
|
+
raise AlgorithmOutputError("RQ-RAG termination is unsupported")
|
|
995
|
+
if termination == "selected_leaf" and any(
|
|
996
|
+
node.parent_id == selected_node_id for node in nodes
|
|
997
|
+
):
|
|
998
|
+
raise AlgorithmOutputError("RQ-RAG selected_leaf termination selected a non-leaf")
|
|
999
|
+
|
|
1000
|
+
evidence = tuple(evidence_by_id[item].as_context_block() for item in selected_document_ids)
|
|
1001
|
+
trace_events: list[TraceEvent] = [
|
|
1002
|
+
TraceEvent(
|
|
1003
|
+
TracePhase.PREPARE,
|
|
1004
|
+
"gated_tree_inference",
|
|
1005
|
+
{
|
|
1006
|
+
"license_clearance_id": self.license_clearance.clearance_id,
|
|
1007
|
+
"patch_manifest_sha256": self.patch_attestation.patch_manifest_sha256,
|
|
1008
|
+
"token_protocol_sha256": self.token_protocol_sha256,
|
|
1009
|
+
},
|
|
1010
|
+
)
|
|
1011
|
+
]
|
|
1012
|
+
trace_events.extend(
|
|
1013
|
+
TraceEvent(
|
|
1014
|
+
TracePhase.OBSERVE,
|
|
1015
|
+
"tree_node",
|
|
1016
|
+
{
|
|
1017
|
+
"node_id": node.node_id,
|
|
1018
|
+
"parent_id": node.parent_id,
|
|
1019
|
+
"depth": node.depth,
|
|
1020
|
+
"query_type": node.query_type,
|
|
1021
|
+
"control_token": node.control_token,
|
|
1022
|
+
"query": node.query,
|
|
1023
|
+
"document_ids": node.document_ids,
|
|
1024
|
+
"score": node.score,
|
|
1025
|
+
},
|
|
1026
|
+
)
|
|
1027
|
+
for node in nodes
|
|
1028
|
+
)
|
|
1029
|
+
trace_events.append(
|
|
1030
|
+
TraceEvent(
|
|
1031
|
+
TracePhase.FINALIZE,
|
|
1032
|
+
"tree_selection",
|
|
1033
|
+
{
|
|
1034
|
+
"selected_node_id": selected_node_id,
|
|
1035
|
+
"selected_path": selected_path,
|
|
1036
|
+
"selected_document_ids": selected_document_ids,
|
|
1037
|
+
"selection_mode": selection_mode,
|
|
1038
|
+
"termination": termination,
|
|
1039
|
+
},
|
|
1040
|
+
)
|
|
1041
|
+
)
|
|
1042
|
+
trace = ExecutionTrace(
|
|
1043
|
+
method_id=self.method_id,
|
|
1044
|
+
release_id=self.release_id,
|
|
1045
|
+
events=tuple(trace_events),
|
|
1046
|
+
counters={
|
|
1047
|
+
**external_usage_counters(response.usage),
|
|
1048
|
+
"worker_calls": 1,
|
|
1049
|
+
"tree_nodes": len(nodes),
|
|
1050
|
+
"tree_depth": max(node.depth for node in nodes),
|
|
1051
|
+
"selected_documents": len(evidence),
|
|
1052
|
+
},
|
|
1053
|
+
)
|
|
1054
|
+
return CompositeResult(
|
|
1055
|
+
method_id=self.method_id,
|
|
1056
|
+
release_id=self.release_id,
|
|
1057
|
+
source_state_hash=state.state_hash,
|
|
1058
|
+
final_answer=final_answer,
|
|
1059
|
+
evidence=evidence,
|
|
1060
|
+
trace=trace,
|
|
1061
|
+
implementation_identity=(
|
|
1062
|
+
f"thin_external_wrapper:chanchimin/RQ-RAG@{RQ_RAG_UPSTREAM_COMMIT}"
|
|
1063
|
+
),
|
|
1064
|
+
profile_kind="official_native",
|
|
1065
|
+
metadata={
|
|
1066
|
+
"implementation_origin": "thin_external_wrapper",
|
|
1067
|
+
"license_clearance_id": self.license_clearance.clearance_id,
|
|
1068
|
+
"patch_manifest_sha256": self.patch_attestation.patch_manifest_sha256,
|
|
1069
|
+
"checkpoint_revision": self.checkpoint_revision,
|
|
1070
|
+
"tokenizer_revision": self.tokenizer_revision,
|
|
1071
|
+
"retriever_revision": self.retriever_revision,
|
|
1072
|
+
"token_protocol_sha256": self.token_protocol_sha256,
|
|
1073
|
+
"selected_node_id": selected_node_id,
|
|
1074
|
+
"selected_path": selected_path,
|
|
1075
|
+
"selection_mode": selection_mode,
|
|
1076
|
+
"termination": termination,
|
|
1077
|
+
"worker_fingerprint": identity,
|
|
1078
|
+
"license_gate": "explicitly_cleared_external_only",
|
|
1079
|
+
},
|
|
1080
|
+
)
|
|
1081
|
+
|
|
1082
|
+
|
|
1083
|
+
def _dependency_identity(value: object, *, label: str) -> Mapping[str, Any]:
|
|
1084
|
+
if not isinstance(value, Mapping) or not value:
|
|
1085
|
+
raise CapabilityError(f"{label} fingerprint must be a non-empty mapping")
|
|
1086
|
+
try:
|
|
1087
|
+
frozen = freeze_mapping(value)
|
|
1088
|
+
content_hash(frozen)
|
|
1089
|
+
except (TypeError, ValueError) as exc:
|
|
1090
|
+
raise ValidationError(f"{label} fingerprint is not canonical: {exc}") from exc
|
|
1091
|
+
return frozen
|
|
1092
|
+
|
|
1093
|
+
|
|
1094
|
+
def _reject_protocol_tokens(value: str, *, label: str) -> None:
|
|
1095
|
+
found = tuple(token for token in _RESERVED_PROTOCOL_TOKENS if token in value)
|
|
1096
|
+
if found:
|
|
1097
|
+
raise ValidationError(f"{label} contains reserved RQ-RAG protocol tokens: {found}")
|
|
1098
|
+
|
|
1099
|
+
|
|
1100
|
+
def _method_initial_prompt(state: FrozenAgentState) -> str:
|
|
1101
|
+
_reject_protocol_tokens(state.instruction, label="RQ-RAG instruction")
|
|
1102
|
+
for index, message in enumerate(state.messages):
|
|
1103
|
+
_reject_protocol_tokens(
|
|
1104
|
+
message.content,
|
|
1105
|
+
label=f"RQ-RAG message {index} content",
|
|
1106
|
+
)
|
|
1107
|
+
for index, block in enumerate(state.context):
|
|
1108
|
+
_reject_protocol_tokens(
|
|
1109
|
+
block.text,
|
|
1110
|
+
label=f"RQ-RAG input context {index}",
|
|
1111
|
+
)
|
|
1112
|
+
system = state.instruction.strip() or (
|
|
1113
|
+
"Answer the question concisely. Refine or decompose the query and use "
|
|
1114
|
+
"retrieved evidence when it improves the answer."
|
|
1115
|
+
)
|
|
1116
|
+
parts = [f"<s><|system|>\n{system}</s>\n"]
|
|
1117
|
+
if state.messages:
|
|
1118
|
+
parts.append("<|history|>\n")
|
|
1119
|
+
parts.extend(f"{message.role}: {message.content}\n" for message in state.messages)
|
|
1120
|
+
parts.append("</|history|>\n")
|
|
1121
|
+
if state.context:
|
|
1122
|
+
parts.append("<|provided_context|>\n")
|
|
1123
|
+
parts.extend(
|
|
1124
|
+
f"Context {index + 1}: {block.text}\n" for index, block in enumerate(state.context)
|
|
1125
|
+
)
|
|
1126
|
+
parts.append("</|provided_context|>\n")
|
|
1127
|
+
parts.append(f"<|user|>\n{state.query.strip()}</s>\n<|assistant|>\n")
|
|
1128
|
+
return "".join(parts)
|
|
1129
|
+
|
|
1130
|
+
|
|
1131
|
+
def _parse_action_completion(
|
|
1132
|
+
value: object,
|
|
1133
|
+
*,
|
|
1134
|
+
action: str,
|
|
1135
|
+
control_token: str,
|
|
1136
|
+
) -> str:
|
|
1137
|
+
if not isinstance(value, str) or not value.strip():
|
|
1138
|
+
raise AlgorithmOutputError(f"RQ-RAG {action} generation returned empty text")
|
|
1139
|
+
text = value.strip()
|
|
1140
|
+
if control_token in text:
|
|
1141
|
+
text = text.rsplit(control_token, 1)[1].lstrip()
|
|
1142
|
+
if "[EOS]" in text:
|
|
1143
|
+
text, trailing = text.split("[EOS]", 1)
|
|
1144
|
+
if trailing.strip():
|
|
1145
|
+
raise AlgorithmOutputError("RQ-RAG completion has content after [EOS]")
|
|
1146
|
+
text = text.strip()
|
|
1147
|
+
if not text:
|
|
1148
|
+
raise AlgorithmOutputError(f"RQ-RAG {action} generation parsed as empty")
|
|
1149
|
+
unexpected = tuple(token for token in _RESERVED_PROTOCOL_TOKENS if token in text)
|
|
1150
|
+
if unexpected:
|
|
1151
|
+
raise AlgorithmOutputError(
|
|
1152
|
+
f"RQ-RAG {action} completion contains nested protocol tokens: {unexpected}"
|
|
1153
|
+
)
|
|
1154
|
+
return text
|
|
1155
|
+
|
|
1156
|
+
|
|
1157
|
+
def _optional_finite_number(value: object, *, label: str) -> float | None:
|
|
1158
|
+
if value is None:
|
|
1159
|
+
return None
|
|
1160
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value):
|
|
1161
|
+
raise AlgorithmOutputError(f"{label} must be a finite number")
|
|
1162
|
+
return float(value)
|
|
1163
|
+
|
|
1164
|
+
|
|
1165
|
+
def _candidate_score(response: GenerationResponse) -> RQRAGCandidateScore:
|
|
1166
|
+
raw = response.raw
|
|
1167
|
+
explicit: object = raw.get("rqrag_score")
|
|
1168
|
+
source = "response.raw.rqrag_score"
|
|
1169
|
+
if explicit is None:
|
|
1170
|
+
choices = raw.get("choices")
|
|
1171
|
+
if (
|
|
1172
|
+
isinstance(choices, Sequence)
|
|
1173
|
+
and not isinstance(choices, (str, bytes))
|
|
1174
|
+
and choices
|
|
1175
|
+
and isinstance(choices[0], Mapping)
|
|
1176
|
+
):
|
|
1177
|
+
explicit = choices[0].get("rqrag_score")
|
|
1178
|
+
source = "response.raw.choices[0].rqrag_score"
|
|
1179
|
+
if explicit is not None:
|
|
1180
|
+
if not isinstance(explicit, Mapping):
|
|
1181
|
+
raise AlgorithmOutputError("RQ-RAG rqrag_score must be a mapping")
|
|
1182
|
+
confidence = _optional_finite_number(
|
|
1183
|
+
explicit.get("confidence"),
|
|
1184
|
+
label="RQ-RAG confidence",
|
|
1185
|
+
)
|
|
1186
|
+
perplexity = _optional_finite_number(
|
|
1187
|
+
explicit.get("perplexity"),
|
|
1188
|
+
label="RQ-RAG perplexity",
|
|
1189
|
+
)
|
|
1190
|
+
if perplexity is not None and explicit.get("perplexity_scope") != "full_path":
|
|
1191
|
+
raise AlgorithmOutputError(
|
|
1192
|
+
"RQ-RAG perplexity must declare perplexity_scope='full_path'"
|
|
1193
|
+
)
|
|
1194
|
+
return RQRAGCandidateScore(confidence, perplexity, source)
|
|
1195
|
+
|
|
1196
|
+
token_confidences = _openai_content_confidences(raw)
|
|
1197
|
+
if token_confidences:
|
|
1198
|
+
return RQRAGCandidateScore(
|
|
1199
|
+
confidence=sum(token_confidences) / len(token_confidences),
|
|
1200
|
+
perplexity=None,
|
|
1201
|
+
source="openai_compatible_completion_logprobs",
|
|
1202
|
+
)
|
|
1203
|
+
return RQRAGCandidateScore(None, None, "unreported")
|
|
1204
|
+
|
|
1205
|
+
|
|
1206
|
+
def _openai_content_confidences(raw: Mapping[str, Any]) -> tuple[float, ...]:
|
|
1207
|
+
choices = raw.get("choices")
|
|
1208
|
+
if (
|
|
1209
|
+
not isinstance(choices, Sequence)
|
|
1210
|
+
or isinstance(choices, (str, bytes))
|
|
1211
|
+
or not choices
|
|
1212
|
+
or not isinstance(choices[0], Mapping)
|
|
1213
|
+
):
|
|
1214
|
+
return ()
|
|
1215
|
+
logprobs = choices[0].get("logprobs")
|
|
1216
|
+
if not isinstance(logprobs, Mapping):
|
|
1217
|
+
return ()
|
|
1218
|
+
content = logprobs.get("content")
|
|
1219
|
+
if not isinstance(content, Sequence) or isinstance(content, (str, bytes)) or not content:
|
|
1220
|
+
return ()
|
|
1221
|
+
values: list[float] = []
|
|
1222
|
+
for item in content:
|
|
1223
|
+
if not isinstance(item, Mapping):
|
|
1224
|
+
raise AlgorithmOutputError("RQ-RAG token logprobs must be mappings")
|
|
1225
|
+
top_logprobs = item.get("top_logprobs")
|
|
1226
|
+
if (
|
|
1227
|
+
not isinstance(top_logprobs, Sequence)
|
|
1228
|
+
or isinstance(top_logprobs, (str, bytes))
|
|
1229
|
+
or not top_logprobs
|
|
1230
|
+
):
|
|
1231
|
+
raise AlgorithmOutputError(
|
|
1232
|
+
"RQ-RAG confidence scoring requires non-empty top_logprobs "
|
|
1233
|
+
"for every generated token"
|
|
1234
|
+
)
|
|
1235
|
+
candidate_logprobs: list[float] = []
|
|
1236
|
+
for top_item in top_logprobs:
|
|
1237
|
+
if not isinstance(top_item, Mapping):
|
|
1238
|
+
raise AlgorithmOutputError("RQ-RAG top token logprobs must be mappings")
|
|
1239
|
+
top_value = _optional_finite_number(
|
|
1240
|
+
top_item.get("logprob"),
|
|
1241
|
+
label="RQ-RAG top token logprob",
|
|
1242
|
+
)
|
|
1243
|
+
if top_value is not None:
|
|
1244
|
+
candidate_logprobs.append(top_value)
|
|
1245
|
+
logprob = max(candidate_logprobs) if candidate_logprobs else None
|
|
1246
|
+
if logprob is None:
|
|
1247
|
+
raise AlgorithmOutputError("RQ-RAG token logprob is missing")
|
|
1248
|
+
if logprob > 0:
|
|
1249
|
+
raise AlgorithmOutputError("RQ-RAG token logprob must not be positive")
|
|
1250
|
+
values.append(math.exp(logprob))
|
|
1251
|
+
return tuple(values)
|
|
1252
|
+
|
|
1253
|
+
|
|
1254
|
+
def _require_selection_score(score: RQRAGCandidateScore, selection_mode: str) -> None:
|
|
1255
|
+
if selection_mode == "ppl" and score.perplexity is None:
|
|
1256
|
+
raise AlgorithmOutputError("RQ-RAG ppl selection requires model-derived perplexity")
|
|
1257
|
+
if selection_mode in {"confidence", "ensemble"} and score.confidence is None:
|
|
1258
|
+
raise AlgorithmOutputError(
|
|
1259
|
+
f"RQ-RAG {selection_mode} selection requires model-derived confidence"
|
|
1260
|
+
)
|
|
1261
|
+
|
|
1262
|
+
|
|
1263
|
+
def _tree_score(score: RQRAGCandidateScore, selection_mode: str) -> float:
|
|
1264
|
+
if selection_mode == "ppl":
|
|
1265
|
+
if score.perplexity is not None:
|
|
1266
|
+
return -score.perplexity
|
|
1267
|
+
if score.confidence is not None:
|
|
1268
|
+
return score.confidence
|
|
1269
|
+
return 0.0
|
|
1270
|
+
if score.confidence is not None:
|
|
1271
|
+
return score.confidence
|
|
1272
|
+
if score.perplexity is not None:
|
|
1273
|
+
return 1.0 / score.perplexity
|
|
1274
|
+
return 0.0
|
|
1275
|
+
|
|
1276
|
+
|
|
1277
|
+
def _select_candidate(
|
|
1278
|
+
candidates: Sequence[_AnswerCandidate],
|
|
1279
|
+
selection_mode: str,
|
|
1280
|
+
) -> _AnswerCandidate:
|
|
1281
|
+
if not candidates:
|
|
1282
|
+
raise AlgorithmOutputError("RQ-RAG selection requires answer candidates")
|
|
1283
|
+
if selection_mode == "ppl":
|
|
1284
|
+
return min(
|
|
1285
|
+
candidates,
|
|
1286
|
+
key=lambda item: (
|
|
1287
|
+
float(item.score.perplexity),
|
|
1288
|
+
item.ordinal,
|
|
1289
|
+
),
|
|
1290
|
+
)
|
|
1291
|
+
if selection_mode == "confidence":
|
|
1292
|
+
return max(
|
|
1293
|
+
candidates,
|
|
1294
|
+
key=lambda item: (
|
|
1295
|
+
float(item.score.confidence),
|
|
1296
|
+
-item.ordinal,
|
|
1297
|
+
),
|
|
1298
|
+
)
|
|
1299
|
+
|
|
1300
|
+
totals: dict[str, float] = {}
|
|
1301
|
+
first_ordinal: dict[str, int] = {}
|
|
1302
|
+
for candidate in candidates:
|
|
1303
|
+
totals.setdefault(candidate.answer, 0.0)
|
|
1304
|
+
totals[candidate.answer] += float(candidate.score.confidence)
|
|
1305
|
+
first_ordinal.setdefault(candidate.answer, candidate.ordinal)
|
|
1306
|
+
selected_answer = max(
|
|
1307
|
+
totals,
|
|
1308
|
+
key=lambda answer: (totals[answer], -first_ordinal[answer]),
|
|
1309
|
+
)
|
|
1310
|
+
return max(
|
|
1311
|
+
(candidate for candidate in candidates if candidate.answer == selected_answer),
|
|
1312
|
+
key=lambda item: (float(item.score.confidence), -item.ordinal),
|
|
1313
|
+
)
|
|
1314
|
+
|
|
1315
|
+
|
|
1316
|
+
def _next_node_id(index: int) -> str:
|
|
1317
|
+
return f"rqrag-node-{index:04d}"
|
|
1318
|
+
|
|
1319
|
+
|
|
1320
|
+
def _node_ancestry(
|
|
1321
|
+
node_id: str,
|
|
1322
|
+
node_by_id: Mapping[str, RQRAGTreeNode],
|
|
1323
|
+
) -> tuple[str, ...]:
|
|
1324
|
+
path: list[str] = []
|
|
1325
|
+
seen: set[str] = set()
|
|
1326
|
+
cursor: str | None = node_id
|
|
1327
|
+
while cursor is not None:
|
|
1328
|
+
if cursor in seen or cursor not in node_by_id:
|
|
1329
|
+
raise AlgorithmOutputError("RQ-RAG tree ancestry is cyclic or incomplete")
|
|
1330
|
+
seen.add(cursor)
|
|
1331
|
+
path.append(cursor)
|
|
1332
|
+
cursor = node_by_id[cursor].parent_id
|
|
1333
|
+
return tuple(reversed(path))
|
|
1334
|
+
|
|
1335
|
+
|
|
1336
|
+
def _validate_retrieval_results(
|
|
1337
|
+
results: tuple[RetrievalResult, ...],
|
|
1338
|
+
*,
|
|
1339
|
+
top_k: int,
|
|
1340
|
+
) -> tuple[RetrievalResult, ...]:
|
|
1341
|
+
if len(results) > top_k:
|
|
1342
|
+
raise AlgorithmOutputError("RQ-RAG retriever returned more than top_k results")
|
|
1343
|
+
seen: set[str] = set()
|
|
1344
|
+
for expected_rank, result in enumerate(results, start=1):
|
|
1345
|
+
if not isinstance(result, RetrievalResult):
|
|
1346
|
+
raise AlgorithmOutputError("RQ-RAG retriever must return RetrievalResult instances")
|
|
1347
|
+
if result.rank != expected_rank:
|
|
1348
|
+
raise AlgorithmOutputError("RQ-RAG retrieval ranks must be contiguous")
|
|
1349
|
+
if (
|
|
1350
|
+
isinstance(result.score, bool)
|
|
1351
|
+
or not isinstance(result.score, (int, float))
|
|
1352
|
+
or not math.isfinite(result.score)
|
|
1353
|
+
):
|
|
1354
|
+
raise AlgorithmOutputError("RQ-RAG retrieval scores must be finite")
|
|
1355
|
+
if not result.document.text.strip():
|
|
1356
|
+
raise AlgorithmOutputError("RQ-RAG retrieved documents must contain text")
|
|
1357
|
+
if result.document.doc_id in seen:
|
|
1358
|
+
raise AlgorithmOutputError("RQ-RAG retrieval results contain duplicate IDs")
|
|
1359
|
+
seen.add(result.document.doc_id)
|
|
1360
|
+
return results
|
|
1361
|
+
|
|
1362
|
+
|
|
1363
|
+
def _validate_repeated_document(
|
|
1364
|
+
accumulator: _EvidenceAccumulator,
|
|
1365
|
+
result: RetrievalResult,
|
|
1366
|
+
) -> None:
|
|
1367
|
+
document = result.document
|
|
1368
|
+
expected = {
|
|
1369
|
+
"text": accumulator.text,
|
|
1370
|
+
"title": accumulator.metadata.get("title", ""),
|
|
1371
|
+
"corpus_revision": accumulator.metadata.get("corpus_revision", ""),
|
|
1372
|
+
}
|
|
1373
|
+
observed = {
|
|
1374
|
+
"text": document.text,
|
|
1375
|
+
"title": document.title,
|
|
1376
|
+
"corpus_revision": document.corpus_revision,
|
|
1377
|
+
}
|
|
1378
|
+
if content_hash(expected) != content_hash(observed):
|
|
1379
|
+
raise AlgorithmOutputError("RQ-RAG retriever reused a document ID with different content")
|
|
1380
|
+
|
|
1381
|
+
|
|
1382
|
+
def _escape_protocol_tokens(value: str) -> tuple[str, int]:
|
|
1383
|
+
escaped = value
|
|
1384
|
+
replacements = 0
|
|
1385
|
+
for token in _RESERVED_PROTOCOL_TOKENS:
|
|
1386
|
+
count = escaped.count(token)
|
|
1387
|
+
if count:
|
|
1388
|
+
escaped = escaped.replace(
|
|
1389
|
+
token,
|
|
1390
|
+
token.replace("[", "\uff3b").replace("]", "\uff3d"),
|
|
1391
|
+
)
|
|
1392
|
+
replacements += count
|
|
1393
|
+
return escaped, replacements
|
|
1394
|
+
|
|
1395
|
+
|
|
1396
|
+
def _validate_regression_echo(value: object) -> None:
|
|
1397
|
+
if not isinstance(value, Mapping):
|
|
1398
|
+
raise AlgorithmOutputError("RQ-RAG response requires regression_gates")
|
|
1399
|
+
required = {
|
|
1400
|
+
"tree_false": True,
|
|
1401
|
+
"oracle_false": True,
|
|
1402
|
+
"duplicate_stop": True,
|
|
1403
|
+
"semantic_patch_only": True,
|
|
1404
|
+
}
|
|
1405
|
+
if set(value) != set(required) or any(
|
|
1406
|
+
value[key] is not expected for key, expected in required.items()
|
|
1407
|
+
):
|
|
1408
|
+
raise AlgorithmOutputError("RQ-RAG worker did not attest every inference regression gate")
|
|
1409
|
+
|
|
1410
|
+
|
|
1411
|
+
def _validate_tree(
|
|
1412
|
+
nodes: tuple[RQRAGTreeNode, ...],
|
|
1413
|
+
*,
|
|
1414
|
+
selected_node_id: str,
|
|
1415
|
+
selected_path: tuple[str, ...],
|
|
1416
|
+
controls: RQRAGControls,
|
|
1417
|
+
) -> None:
|
|
1418
|
+
if not nodes:
|
|
1419
|
+
raise AlgorithmOutputError("RQ-RAG tree must contain at least one node")
|
|
1420
|
+
if len(nodes) > controls.max_nodes:
|
|
1421
|
+
raise AlgorithmOutputError("RQ-RAG tree exceeded max_nodes")
|
|
1422
|
+
node_by_id: dict[str, RQRAGTreeNode] = {}
|
|
1423
|
+
token_by_type: dict[str, str] = {}
|
|
1424
|
+
type_by_token: dict[str, str] = {}
|
|
1425
|
+
roots: list[str] = []
|
|
1426
|
+
for node in nodes:
|
|
1427
|
+
if node.node_id in node_by_id:
|
|
1428
|
+
raise AlgorithmOutputError("RQ-RAG tree contains duplicate node IDs")
|
|
1429
|
+
if node.query_type == "direct_response" and node.document_ids:
|
|
1430
|
+
raise AlgorithmOutputError("RQ-RAG direct_response nodes cannot retrieve documents")
|
|
1431
|
+
if node.depth > controls.max_depth:
|
|
1432
|
+
raise AlgorithmOutputError("RQ-RAG tree exceeded max_depth")
|
|
1433
|
+
if node.parent_id is None:
|
|
1434
|
+
if node.depth != 0:
|
|
1435
|
+
raise AlgorithmOutputError("RQ-RAG root depth must be zero")
|
|
1436
|
+
roots.append(node.node_id)
|
|
1437
|
+
else:
|
|
1438
|
+
parent = node_by_id.get(node.parent_id)
|
|
1439
|
+
if parent is None:
|
|
1440
|
+
raise AlgorithmOutputError("RQ-RAG parents must precede children")
|
|
1441
|
+
if node.depth != parent.depth + 1:
|
|
1442
|
+
raise AlgorithmOutputError("RQ-RAG child depth is inconsistent with parent")
|
|
1443
|
+
prior_token = token_by_type.setdefault(node.query_type, node.control_token)
|
|
1444
|
+
if prior_token != node.control_token:
|
|
1445
|
+
raise AlgorithmOutputError("RQ-RAG query type changed control tokens")
|
|
1446
|
+
prior_type = type_by_token.setdefault(node.control_token, node.query_type)
|
|
1447
|
+
if prior_type != node.query_type:
|
|
1448
|
+
raise AlgorithmOutputError("RQ-RAG control token maps to multiple query types")
|
|
1449
|
+
node_by_id[node.node_id] = node
|
|
1450
|
+
if len(roots) != 1:
|
|
1451
|
+
raise AlgorithmOutputError("RQ-RAG tree must contain exactly one root")
|
|
1452
|
+
if selected_node_id not in node_by_id:
|
|
1453
|
+
raise AlgorithmOutputError("RQ-RAG selected_node_id is absent from tree")
|
|
1454
|
+
derived: list[str] = []
|
|
1455
|
+
cursor: str | None = selected_node_id
|
|
1456
|
+
while cursor is not None:
|
|
1457
|
+
derived.append(cursor)
|
|
1458
|
+
cursor = node_by_id[cursor].parent_id
|
|
1459
|
+
expected_path = tuple(reversed(derived))
|
|
1460
|
+
if selected_path != expected_path or selected_path[0] != roots[0]:
|
|
1461
|
+
raise AlgorithmOutputError("RQ-RAG selected_path is not the selected node ancestry")
|
|
1462
|
+
|
|
1463
|
+
|
|
1464
|
+
def _validate_document_lineage(
|
|
1465
|
+
nodes: tuple[RQRAGTreeNode, ...],
|
|
1466
|
+
evidence: tuple[_RQRAGEvidence, ...],
|
|
1467
|
+
) -> Mapping[str, _RQRAGEvidence]:
|
|
1468
|
+
evidence_by_id: dict[str, _RQRAGEvidence] = {}
|
|
1469
|
+
expected_nodes: dict[str, list[str]] = {}
|
|
1470
|
+
expected_order: list[str] = []
|
|
1471
|
+
for node in nodes:
|
|
1472
|
+
for document_id in node.document_ids:
|
|
1473
|
+
if document_id not in expected_nodes:
|
|
1474
|
+
expected_nodes[document_id] = []
|
|
1475
|
+
expected_order.append(document_id)
|
|
1476
|
+
expected_nodes[document_id].append(node.node_id)
|
|
1477
|
+
for item in evidence:
|
|
1478
|
+
if item.document_id in evidence_by_id:
|
|
1479
|
+
raise AlgorithmOutputError("RQ-RAG evidence contains duplicate document IDs")
|
|
1480
|
+
evidence_by_id[item.document_id] = item
|
|
1481
|
+
if tuple(evidence_by_id) != tuple(expected_order):
|
|
1482
|
+
raise AlgorithmOutputError("RQ-RAG evidence must follow first-seen tree order")
|
|
1483
|
+
for document_id, node_ids in expected_nodes.items():
|
|
1484
|
+
if evidence_by_id[document_id].retrieved_by_node_ids != tuple(node_ids):
|
|
1485
|
+
raise AlgorithmOutputError("RQ-RAG evidence node lineage mismatch")
|
|
1486
|
+
return MappingProxyType(evidence_by_id)
|
|
1487
|
+
|
|
1488
|
+
|
|
1489
|
+
def _selected_document_order(
|
|
1490
|
+
nodes: tuple[RQRAGTreeNode, ...],
|
|
1491
|
+
selected_path: tuple[str, ...],
|
|
1492
|
+
) -> tuple[str, ...]:
|
|
1493
|
+
node_by_id = {node.node_id: node for node in nodes}
|
|
1494
|
+
ordered: list[str] = []
|
|
1495
|
+
seen: set[str] = set()
|
|
1496
|
+
for node_id in selected_path:
|
|
1497
|
+
for document_id in node_by_id[node_id].document_ids:
|
|
1498
|
+
if document_id not in seen:
|
|
1499
|
+
seen.add(document_id)
|
|
1500
|
+
ordered.append(document_id)
|
|
1501
|
+
return tuple(ordered)
|
|
1502
|
+
|
|
1503
|
+
|
|
1504
|
+
__all__ = [
|
|
1505
|
+
"RQ_RAG_ACTION_TOKENS",
|
|
1506
|
+
"RQ_RAG_METHOD_RELEASE_ID",
|
|
1507
|
+
"RQ_RAG_UPSTREAM_COMMIT",
|
|
1508
|
+
"RQRAGCandidateScore",
|
|
1509
|
+
"RQRAGControls",
|
|
1510
|
+
"RQRAGLicenseClearance",
|
|
1511
|
+
"RQRAGMethodControls",
|
|
1512
|
+
"RQRAGMethodTreeSearch",
|
|
1513
|
+
"RQRAGPatchAttestation",
|
|
1514
|
+
"RQRAGReleased7BExternal",
|
|
1515
|
+
"RQRAGTreeNode",
|
|
1516
|
+
]
|