agentinputlab 0.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentinputlab/__init__.py +28 -0
- agentinputlab/algorithms/__init__.py +6 -0
- agentinputlab/algorithms/acquire/__init__.py +145 -0
- agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
- agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
- agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
- agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
- agentinputlab/algorithms/acquire/hyde.py +296 -0
- agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
- agentinputlab/algorithms/acquire/hyde_external.py +148 -0
- agentinputlab/algorithms/acquire/ircot.py +28 -0
- agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
- agentinputlab/algorithms/acquire/ircot_native.py +355 -0
- agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
- agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
- agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
- agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
- agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
- agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
- agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
- agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
- agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
- agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
- agentinputlab/algorithms/clarify/__init__.py +61 -0
- agentinputlab/algorithms/clarify/_structured.py +64 -0
- agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
- agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
- agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
- agentinputlab/algorithms/clarify/future_turns.py +379 -0
- agentinputlab/algorithms/clarify/udcs.py +255 -0
- agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
- agentinputlab/algorithms/curate/__init__.py +65 -0
- agentinputlab/algorithms/curate/acon/__init__.py +27 -0
- agentinputlab/algorithms/curate/acon/common.py +284 -0
- agentinputlab/algorithms/curate/acon/common_support.py +136 -0
- agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
- agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
- agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
- agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
- agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
- agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
- agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
- agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
- agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
- agentinputlab/algorithms/curate/recomp/external.py +1129 -0
- agentinputlab/algorithms/curate/recomp/selective.py +304 -0
- agentinputlab/algorithms/decompose/__init__.py +107 -0
- agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
- agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
- agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
- agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
- agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
- agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
- agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
- agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
- agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
- agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
- agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
- agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
- agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
- agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
- agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
- agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
- agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
- agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
- agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
- agentinputlab/algorithms/reformulate/__init__.py +85 -0
- agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
- agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
- agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
- agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
- agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
- agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
- agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
- agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
- agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
- agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
- agentinputlab/algorithms/reformulate/rrr.py +696 -0
- agentinputlab/annotation/__init__.py +5 -0
- agentinputlab/annotation/independent.py +428 -0
- agentinputlab/annotation/server.py +233 -0
- agentinputlab/annotation/static/app.js +121 -0
- agentinputlab/annotation/static/index.html +60 -0
- agentinputlab/annotation/static/review.css +2 -0
- agentinputlab/annotation/static/review.html +10 -0
- agentinputlab/annotation/static/review.js +11 -0
- agentinputlab/annotation/static/styles.css +55 -0
- agentinputlab/annotation/store.py +444 -0
- agentinputlab/api.py +368 -0
- agentinputlab/artifacts/__init__.py +15 -0
- agentinputlab/artifacts/json_file.py +104 -0
- agentinputlab/artifacts/manifest.py +98 -0
- agentinputlab/artifacts/store.py +157 -0
- agentinputlab/backends/__init__.py +98 -0
- agentinputlab/backends/embedding/__init__.py +25 -0
- agentinputlab/backends/embedding/base.py +145 -0
- agentinputlab/backends/embedding/callable.py +75 -0
- agentinputlab/backends/embedding/replay.py +129 -0
- agentinputlab/backends/llm/__init__.py +31 -0
- agentinputlab/backends/llm/base.py +149 -0
- agentinputlab/backends/llm/callable.py +108 -0
- agentinputlab/backends/llm/decomp_completion.py +88 -0
- agentinputlab/backends/llm/final_content_stop.py +72 -0
- agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
- agentinputlab/backends/llm/openai_compatible.py +332 -0
- agentinputlab/backends/llm/transformers.py +171 -0
- agentinputlab/backends/retrieval/__init__.py +35 -0
- agentinputlab/backends/retrieval/base.py +50 -0
- agentinputlab/backends/retrieval/dense.py +777 -0
- agentinputlab/backends/retrieval/memory.py +119 -0
- agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
- agentinputlab/backends/user/__init__.py +15 -0
- agentinputlab/backends/user/base.py +66 -0
- agentinputlab/backends/user/callable.py +54 -0
- agentinputlab/backends/user/replay.py +89 -0
- agentinputlab/backends/vector.py +38 -0
- agentinputlab/benchmarks/__init__.py +246 -0
- agentinputlab/benchmarks/acquisition.py +106 -0
- agentinputlab/benchmarks/ale.py +540 -0
- agentinputlab/benchmarks/appworld_ul.py +669 -0
- agentinputlab/benchmarks/assets.py +226 -0
- agentinputlab/benchmarks/base.py +68 -0
- agentinputlab/benchmarks/beir_archive.py +235 -0
- agentinputlab/benchmarks/beir_sample.py +269 -0
- agentinputlab/benchmarks/bfcl_native.py +245 -0
- agentinputlab/benchmarks/bfcl_v3.py +607 -0
- agentinputlab/benchmarks/cache.py +218 -0
- agentinputlab/benchmarks/clariq.py +203 -0
- agentinputlab/benchmarks/data/registry.json +321 -0
- agentinputlab/benchmarks/data/smoke_assets.json +511 -0
- agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
- agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
- agentinputlab/benchmarks/execution.py +1111 -0
- agentinputlab/benchmarks/interactive.py +257 -0
- agentinputlab/benchmarks/ir_query.py +204 -0
- agentinputlab/benchmarks/longbench.py +203 -0
- agentinputlab/benchmarks/paired_replay.py +479 -0
- agentinputlab/benchmarks/preprocess.py +2313 -0
- agentinputlab/benchmarks/production.py +236 -0
- agentinputlab/benchmarks/qa.py +963 -0
- agentinputlab/benchmarks/registry.py +110 -0
- agentinputlab/benchmarks/retrieval.py +459 -0
- agentinputlab/benchmarks/runtime.py +509 -0
- agentinputlab/benchmarks/statebench.py +938 -0
- agentinputlab/benchmarks/tau3.py +725 -0
- agentinputlab/benchmarks/toolsandbox.py +626 -0
- agentinputlab/benchmarks/trec_dl_sample.py +263 -0
- agentinputlab/campaign.py +654 -0
- agentinputlab/campaign_runner.py +953 -0
- agentinputlab/cli/__init__.py +6 -0
- agentinputlab/cli/main.py +933 -0
- agentinputlab/connectors/__init__.py +46 -0
- agentinputlab/connectors/knowledge/__init__.py +4 -0
- agentinputlab/connectors/tools/__init__.py +30 -0
- agentinputlab/connectors/tools/base.py +57 -0
- agentinputlab/connectors/tools/mcp.py +222 -0
- agentinputlab/connectors/tools/memory.py +41 -0
- agentinputlab/connectors/tools/openapi.py +695 -0
- agentinputlab/connectors/tools/schema.py +102 -0
- agentinputlab/contracts/__init__.py +91 -0
- agentinputlab/contracts/availability.py +24 -0
- agentinputlab/contracts/dependency.py +58 -0
- agentinputlab/contracts/enums.py +82 -0
- agentinputlab/contracts/errors.py +63 -0
- agentinputlab/contracts/evidence.py +43 -0
- agentinputlab/contracts/intent.py +110 -0
- agentinputlab/contracts/lifecycle.py +164 -0
- agentinputlab/contracts/patch.py +167 -0
- agentinputlab/contracts/serialization.py +93 -0
- agentinputlab/contracts/state.py +127 -0
- agentinputlab/contracts/trace.py +59 -0
- agentinputlab/contracts/worker.py +42 -0
- agentinputlab/core/__init__.py +7 -0
- agentinputlab/data/__init__.py +32 -0
- agentinputlab/data/checksums.py +9 -0
- agentinputlab/data/downloader.py +235 -0
- agentinputlab/data/http.py +171 -0
- agentinputlab/data/index.py +21 -0
- agentinputlab/data/overlap.py +28 -0
- agentinputlab/data/preprocess.py +32 -0
- agentinputlab/data/storage.py +57 -0
- agentinputlab/data/trec_sample.py +427 -0
- agentinputlab/evaluation/__init__.py +5 -0
- agentinputlab/evaluation/outcomes.py +32 -0
- agentinputlab/experiments/__init__.py +278 -0
- agentinputlab/experiments/analysis.py +981 -0
- agentinputlab/experiments/artifacts.py +687 -0
- agentinputlab/experiments/automatic_measurement.py +678 -0
- agentinputlab/experiments/auxiliary_matrix.py +203 -0
- agentinputlab/experiments/bindings.py +430 -0
- agentinputlab/experiments/blueprints.py +147 -0
- agentinputlab/experiments/builders/__init__.py +45 -0
- agentinputlab/experiments/builders/executable_external_payload.py +423 -0
- agentinputlab/experiments/builders/external_process_arena.py +1018 -0
- agentinputlab/experiments/builders/registry_intervention.py +463 -0
- agentinputlab/experiments/cli.py +158 -0
- agentinputlab/experiments/controls.py +340 -0
- agentinputlab/experiments/driver_guard.py +205 -0
- agentinputlab/experiments/drivers/__init__.py +89 -0
- agentinputlab/experiments/drivers/external_arena.py +2908 -0
- agentinputlab/experiments/drivers/method_process.py +662 -0
- agentinputlab/experiments/drivers/parity_process.py +1484 -0
- agentinputlab/experiments/drivers/static_production.py +1574 -0
- agentinputlab/experiments/executor.py +448 -0
- agentinputlab/experiments/loader.py +605 -0
- agentinputlab/experiments/measurement.py +675 -0
- agentinputlab/experiments/paper_analysis.py +1818 -0
- agentinputlab/experiments/parity_identity.py +605 -0
- agentinputlab/experiments/plan.py +277 -0
- agentinputlab/experiments/production_builders.py +3739 -0
- agentinputlab/experiments/readiness.py +432 -0
- agentinputlab/experiments/schema.py +514 -0
- agentinputlab/experiments/secrets.py +177 -0
- agentinputlab/experiments/stateful_analysis.py +1738 -0
- agentinputlab/experiments/transfer_analysis.py +1316 -0
- agentinputlab/experiments/unseen_analysis.py +1025 -0
- agentinputlab/integrations/__init__.py +1 -0
- agentinputlab/integrations/action_space/__init__.py +1 -0
- agentinputlab/integrations/external_workers/__init__.py +76 -0
- agentinputlab/integrations/external_workers/ace.py +431 -0
- agentinputlab/integrations/external_workers/base.py +181 -0
- agentinputlab/integrations/external_workers/decomp_live.py +249 -0
- agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
- agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
- agentinputlab/integrations/external_workers/ircot.py +576 -0
- agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
- agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
- agentinputlab/integrations/external_workers/method.py +314 -0
- agentinputlab/integrations/external_workers/process.py +678 -0
- agentinputlab/integrations/external_workers/supervisor.py +1071 -0
- agentinputlab/integrations/full_architecture/__init__.py +1 -0
- agentinputlab/integrations/full_architecture/hyde.py +212 -0
- agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
- agentinputlab/integrations/live_history.py +222 -0
- agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
- agentinputlab/integrations/training/__init__.py +1 -0
- agentinputlab/metrics/__init__.py +50 -0
- agentinputlab/metrics/catalog.py +313 -0
- agentinputlab/metrics/harm.py +95 -0
- agentinputlab/metrics/paired.py +40 -0
- agentinputlab/metrics/qa.py +76 -0
- agentinputlab/metrics/text.py +33 -0
- agentinputlab/metrics/third_party.py +161 -0
- agentinputlab/models/__init__.py +4 -0
- agentinputlab/observability/__init__.py +56 -0
- agentinputlab/observability/budget.py +316 -0
- agentinputlab/observability/instrumentation.py +754 -0
- agentinputlab/observability/models.py +403 -0
- agentinputlab/observability/monitor.py +380 -0
- agentinputlab/optimizers/__init__.py +1 -0
- agentinputlab/optimizers/clarify/__init__.py +12 -0
- agentinputlab/optimizers/context/__init__.py +22 -0
- agentinputlab/optimizers/decompose/__init__.py +64 -0
- agentinputlab/optimizers/evidence/__init__.py +22 -0
- agentinputlab/optimizers/rewrite/__init__.py +30 -0
- agentinputlab/parity/__init__.py +5 -0
- agentinputlab/parity/base.py +206 -0
- agentinputlab/production_campaign.py +763 -0
- agentinputlab/py.typed +1 -0
- agentinputlab/recipes/__init__.py +333 -0
- agentinputlab/recipes/ace.py +418 -0
- agentinputlab/recipes/ace_bfcl.py +273 -0
- agentinputlab/recipes/ace_common_support.py +169 -0
- agentinputlab/recipes/ace_native_plan.py +322 -0
- agentinputlab/recipes/ace_online.py +711 -0
- agentinputlab/recipes/ace_retrieval.py +330 -0
- agentinputlab/recipes/ace_statebench.py +382 -0
- agentinputlab/recipes/ace_tau.py +463 -0
- agentinputlab/recipes/ace_toolsandbox.py +432 -0
- agentinputlab/recipes/acon_guideline.py +22 -0
- agentinputlab/recipes/adaptive_rag.py +822 -0
- agentinputlab/recipes/llmlingua2_inference.py +146 -0
- agentinputlab/recipes/llmlingua2_training.py +796 -0
- agentinputlab/recipes/mce.py +1486 -0
- agentinputlab/recipes/mce_final.py +335 -0
- agentinputlab/recipes/mce_text_fit.py +499 -0
- agentinputlab/recipes/mce_text_only.py +298 -0
- agentinputlab/recipes/qrag.py +729 -0
- agentinputlab/recipes/qrag_inference.py +364 -0
- agentinputlab/recipes/r2ag_features.py +236 -0
- agentinputlab/recipes/r2ag_inference.py +503 -0
- agentinputlab/recipes/r2ag_training.py +948 -0
- agentinputlab/recipes/recomp_abstractive.py +592 -0
- agentinputlab/recipes/recomp_extractive.py +525 -0
- agentinputlab/recipes/reformer.py +17 -0
- agentinputlab/recipes/rqrag.py +1516 -0
- agentinputlab/recipes/rqrag_common_support.py +193 -0
- agentinputlab/recipes/rqrag_completion.py +147 -0
- agentinputlab/recipes/skillopt.py +462 -0
- agentinputlab/recipes/skillopt_common_support.py +154 -0
- agentinputlab/recipes/skillopt_environment.py +165 -0
- agentinputlab/recipes/skillopt_environment_cli.py +5 -0
- agentinputlab/recipes/skillopt_retrieval.py +188 -0
- agentinputlab/recipes/smartsearch.py +897 -0
- agentinputlab/recipes/smartsearch_curriculum.py +283 -0
- agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
- agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
- agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
- agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
- agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
- agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
- agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
- agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
- agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
- agentinputlab/recipes/smartsearch_inference.py +524 -0
- agentinputlab/recipes/smartsearch_native_data.py +172 -0
- agentinputlab/recipes/smartsearch_policy.py +616 -0
- agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
- agentinputlab/recipes/smartsearch_sampling.py +232 -0
- agentinputlab/recipes/smartsearch_teacher.py +203 -0
- agentinputlab/recipes/smartsearch_training.py +265 -0
- agentinputlab/registry/__init__.py +46 -0
- agentinputlab/registry/beta.py +727 -0
- agentinputlab/registry/data/algorithms.json +1959 -0
- agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
- agentinputlab/registry/loader.py +100 -0
- agentinputlab/registry/result_manifest.py +184 -0
- agentinputlab/registry/runtime.py +195 -0
- agentinputlab/registry/schema.py +449 -0
- agentinputlab/runner/__init__.py +30 -0
- agentinputlab/runner/balance.py +248 -0
- agentinputlab/runner/cache.py +135 -0
- agentinputlab/runner/composite.py +130 -0
- agentinputlab/runner/engine.py +285 -0
- agentinputlab/runner/runtime_guard.py +268 -0
- agentinputlab/runner/worker.py +80 -0
- agentinputlab/runtime_bootstrap.py +288 -0
- agentinputlab/selectors/__init__.py +196 -0
- agentinputlab/selectors/advanced.py +1514 -0
- agentinputlab/selectors/artifacts.py +1087 -0
- agentinputlab/selectors/base.py +112 -0
- agentinputlab/selectors/campaigns.py +1893 -0
- agentinputlab/selectors/catalog.py +86 -0
- agentinputlab/selectors/discovery.py +1860 -0
- agentinputlab/selectors/evaluation.py +1348 -0
- agentinputlab/selectors/features.py +383 -0
- agentinputlab/selectors/learned.py +502 -0
- agentinputlab/selectors/no_op.py +28 -0
- agentinputlab/selectors/outcome_table.py +475 -0
- agentinputlab/selectors/producer.py +853 -0
- agentinputlab/selectors/rules.py +109 -0
- agentinputlab/selectors/runtime.py +420 -0
- agentinputlab/studies/__init__.py +16 -0
- agentinputlab/studies/design.py +165 -0
- agentinputlab/studies/paired.py +322 -0
- agentinputlab/studio/__init__.py +6 -0
- agentinputlab/studio/campaigns.py +819 -0
- agentinputlab/studio/engine.py +1408 -0
- agentinputlab/studio/model_backends.py +425 -0
- agentinputlab/studio/prepared.py +410 -0
- agentinputlab/studio/server.py +613 -0
- agentinputlab/studio/static/app.js +397 -0
- agentinputlab/studio/static/campaigns.html +175 -0
- agentinputlab/studio/static/campaigns.js +426 -0
- agentinputlab/studio/static/index.html +192 -0
- agentinputlab/studio/static/result.html +103 -0
- agentinputlab/studio/static/result.js +273 -0
- agentinputlab/studio/static/styles.css +467 -0
- agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
- agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
- agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
- agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
- agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
- agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
- agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
- agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
- agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
- workers/README.md +94 -0
- workers/ace/README.md +261 -0
- workers/ace/__init__.py +1 -0
- workers/ace/compatibility_manifest.json +39 -0
- workers/ace/driver.py +1682 -0
- workers/acon/README.md +215 -0
- workers/acon/__init__.py +5 -0
- workers/acon/compatibility_manifest.json +93 -0
- workers/acon/driver.py +1859 -0
- workers/acon/fitted.py +395 -0
- workers/acon/guideline.py +470 -0
- workers/acon/guideline_source_manifest.json +20 -0
- workers/adaptive_rag/README.md +96 -0
- workers/adaptive_rag/__init__.py +15 -0
- workers/adaptive_rag/compatibility_manifest.json +61 -0
- workers/adaptive_rag/driver.py +1470 -0
- workers/adaptive_rag/fixture_runtime.py +163 -0
- workers/adaptive_rag/runtime.py +252 -0
- workers/ask_or_assume/README.md +124 -0
- workers/ask_or_assume/__init__.py +1 -0
- workers/ask_or_assume/compatibility_manifest.json +36 -0
- workers/ask_or_assume/driver.py +1227 -0
- workers/decomp/README.md +133 -0
- workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
- workers/decomp/__init__.py +5 -0
- workers/decomp/compatibility_manifest.json +205 -0
- workers/decomp/driver.py +1015 -0
- workers/decomp/live.py +137 -0
- workers/decomp/runtime.py +231 -0
- workers/infocqr/README.md +106 -0
- workers/infocqr/__init__.py +1 -0
- workers/infocqr/compatibility_manifest.json +84 -0
- workers/infocqr/driver.py +1766 -0
- workers/ircot/README.md +167 -0
- workers/ircot/__init__.py +7 -0
- workers/ircot/compatibility_manifest.json +121 -0
- workers/ircot/controlled.py +457 -0
- workers/ircot/driver.py +1373 -0
- workers/ircot/fixture_bindings.json +39 -0
- workers/ircot/fixture_config.json +48 -0
- workers/ircot/fixture_prompt.txt +2 -0
- workers/ircot/native_qa.py +398 -0
- workers/ircot/qa_compatibility_manifest.json +57 -0
- workers/ircot/runtime.py +97 -0
- workers/llmcompiler/README.md +160 -0
- workers/llmcompiler/__init__.py +1 -0
- workers/llmcompiler/compatibility_manifest.json +45 -0
- workers/llmcompiler/driver.py +1514 -0
- workers/llmcompiler/export_prompts.py +74 -0
- workers/llmcompiler/live.py +174 -0
- workers/llmcompiler/patch_manifest.json +147 -0
- workers/mce/README.md +198 -0
- workers/mce/__init__.py +29 -0
- workers/mce/compatibility_manifest.json +115 -0
- workers/mce/driver.py +2585 -0
- workers/mce/final_eval.py +173 -0
- workers/official_arenas/README.md +25 -0
- workers/official_arenas/__init__.py +1 -0
- workers/official_arenas/capture_records.py +110 -0
- workers/official_arenas/driver.py +619 -0
- workers/official_arenas/protocol.py +334 -0
- workers/official_arenas/providers/__init__.py +1 -0
- workers/official_arenas/providers/_live_history.py +228 -0
- workers/official_arenas/providers/_message_compat.py +77 -0
- workers/official_arenas/providers/appworld_ul.py +196 -0
- workers/official_arenas/providers/fixture.py +112 -0
- workers/official_arenas/providers/statebench.py +960 -0
- workers/official_arenas/providers/tau3.py +1672 -0
- workers/official_arenas/providers/tau_retail.py +586 -0
- workers/official_arenas/providers/toolsandbox.py +1321 -0
- workers/official_arenas/providers/tooltalk.py +699 -0
- workers/official_arenas/retail_capture.py +188 -0
- workers/official_arenas/retail_driver.py +275 -0
- workers/qrag/README.md +172 -0
- workers/qrag/__init__.py +1 -0
- workers/qrag/compatibility_manifest.json +64 -0
- workers/qrag/driver.py +673 -0
- workers/r2ag/README.md +154 -0
- workers/r2ag/__init__.py +5 -0
- workers/r2ag/compatibility_manifest.json +55 -0
- workers/r2ag/driver.py +1287 -0
- workers/r2ag/feature_schema.json +8 -0
- workers/r2ag/runtime.py +589 -0
- workers/recomp/README.md +120 -0
- workers/recomp/__init__.py +1 -0
- workers/recomp/compatibility_manifest.json +43 -0
- workers/recomp/driver.py +1653 -0
- workers/reformer/README.md +125 -0
- workers/reformer/__init__.py +1 -0
- workers/reformer/compatibility_manifest.json +68 -0
- workers/reformer/driver.py +1459 -0
- workers/reformer/induction.py +457 -0
- workers/rrr/README.md +126 -0
- workers/rrr/__init__.py +1 -0
- workers/rrr/compatibility_manifest.json +57 -0
- workers/rrr/driver.py +1133 -0
- workers/smartsearch/README.md +331 -0
- workers/smartsearch/__init__.py +1 -0
- workers/smartsearch/compatibility_manifest.json +41 -0
- workers/smartsearch/driver.py +1643 -0
|
@@ -0,0 +1,330 @@
|
|
|
1
|
+
"""Evaluator-bound retrieval feedback for ACE's unchanged offline training loop.
|
|
2
|
+
|
|
3
|
+
The generator produces query JSON. A caller-owned retriever and benchmark
|
|
4
|
+
adapter compute the feedback; qrels never enter the generated training input.
|
|
5
|
+
ACE's Boolean reflection branch uses a positive native score, while its
|
|
6
|
+
validation selection uses the mean of the declared native metric.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import math
|
|
12
|
+
from collections.abc import Mapping, Sequence
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
from agentinputlab.backends.retrieval import RetrievalResult, Retriever
|
|
16
|
+
from agentinputlab.benchmarks.base import BenchmarkAdapter, BenchmarkExample
|
|
17
|
+
from agentinputlab.benchmarks.ir_query import parse_ir_query
|
|
18
|
+
from agentinputlab.benchmarks.retrieval import RetrievalPrediction
|
|
19
|
+
from agentinputlab.contracts import AlgorithmOutputError, ValidationError
|
|
20
|
+
from agentinputlab.contracts.serialization import canonical_json, content_hash, to_jsonable
|
|
21
|
+
|
|
22
|
+
from .ace import ACEExample
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _bind_feedback(
|
|
26
|
+
processor, *, adapter, retriever, groups, metric, top_k, allow_json_fence, partition_label
|
|
27
|
+
):
|
|
28
|
+
if not isinstance(adapter, BenchmarkAdapter) or not isinstance(retriever, Retriever):
|
|
29
|
+
raise ValidationError("ACE retrieval feedback requires an adapter and retriever")
|
|
30
|
+
if not callable(getattr(adapter, "fingerprint", None)):
|
|
31
|
+
raise ValidationError("ACE retrieval adapter requires an explicit fingerprint")
|
|
32
|
+
if not isinstance(metric, str) or not metric.strip():
|
|
33
|
+
raise ValidationError("ACE retrieval metric must be explicit")
|
|
34
|
+
if type(top_k) is not int or not 1 <= top_k <= 10_000:
|
|
35
|
+
raise ValidationError("ACE retrieval top_k must be between 1 and 10000")
|
|
36
|
+
if type(allow_json_fence) is not bool:
|
|
37
|
+
raise ValidationError("ACE retrieval allow_json_fence must be boolean")
|
|
38
|
+
if not isinstance(partition_label, str) or not partition_label.strip():
|
|
39
|
+
raise ValidationError("ACE retrieval partition label must be explicit")
|
|
40
|
+
cases = tuple(e for group in groups for e in group)
|
|
41
|
+
if any(not group for group in groups):
|
|
42
|
+
raise ValidationError("ACE retrieval groups must be nonempty")
|
|
43
|
+
if any(not isinstance(e, BenchmarkExample) for e in cases):
|
|
44
|
+
raise ValidationError("ACE retrieval groups require BenchmarkExample objects")
|
|
45
|
+
if len({e.example_id for e in cases}) != len(cases):
|
|
46
|
+
raise ValidationError("ACE retrieval group IDs must be disjoint and unique")
|
|
47
|
+
if len(
|
|
48
|
+
{
|
|
49
|
+
content_hash(
|
|
50
|
+
dict(
|
|
51
|
+
query=e.query,
|
|
52
|
+
instruction=e.instruction,
|
|
53
|
+
context=e.visible_context,
|
|
54
|
+
messages=e.visible_messages,
|
|
55
|
+
)
|
|
56
|
+
)
|
|
57
|
+
for e in cases
|
|
58
|
+
}
|
|
59
|
+
) != len(cases):
|
|
60
|
+
raise ValidationError("ACE retrieval group inputs must be disjoint")
|
|
61
|
+
if any(e.visible_tool_names for e in cases):
|
|
62
|
+
raise ValidationError("ACE retrieval feedback does not execute tools")
|
|
63
|
+
processor.adapter, processor.retriever = adapter, retriever
|
|
64
|
+
processor.metric, processor.top_k = metric, top_k
|
|
65
|
+
processor.allow_json_fence, processor.partition_label = allow_json_fence, partition_label
|
|
66
|
+
processor._examples = {e.example_id: e for e in cases}
|
|
67
|
+
processor._dataset = to_jsonable(adapter.fingerprint())
|
|
68
|
+
processor._retriever = to_jsonable(retriever.fingerprint())
|
|
69
|
+
processor._cache = {}
|
|
70
|
+
processor.events = []
|
|
71
|
+
for example in cases:
|
|
72
|
+
processor._metric_value(adapter.score(example, RetrievalPrediction({})))
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class ACERetrievalFeedback:
|
|
76
|
+
"""Bind one explicit IR task and disjoint caller-declared fit partitions.
|
|
77
|
+
|
|
78
|
+
``metric`` must name a scalar in ``adapter.score`` (for example
|
|
79
|
+
``"ndcg_at_10"``). The adapter retains its native exclusions and qrels.
|
|
80
|
+
Invalid query JSON receives zero *training feedback*, not a repaired query
|
|
81
|
+
or an extra model call. Held-out evaluation still uses its normal endpoint.
|
|
82
|
+
"""
|
|
83
|
+
|
|
84
|
+
def __init__(
|
|
85
|
+
self,
|
|
86
|
+
*,
|
|
87
|
+
adapter: BenchmarkAdapter,
|
|
88
|
+
retriever: Retriever,
|
|
89
|
+
train: Sequence[BenchmarkExample],
|
|
90
|
+
validation: Sequence[BenchmarkExample],
|
|
91
|
+
metric: str = "ndcg_at_10",
|
|
92
|
+
top_k: int = 15,
|
|
93
|
+
allow_json_fence: bool = False,
|
|
94
|
+
partition_label: str = "caller_declared_compatibility_partitions",
|
|
95
|
+
) -> None:
|
|
96
|
+
self.train, self.validation = tuple(train), tuple(validation)
|
|
97
|
+
_bind_feedback(
|
|
98
|
+
self,
|
|
99
|
+
adapter=adapter,
|
|
100
|
+
retriever=retriever,
|
|
101
|
+
groups=(self.train, self.validation),
|
|
102
|
+
metric=metric,
|
|
103
|
+
top_k=top_k,
|
|
104
|
+
allow_json_fence=allow_json_fence,
|
|
105
|
+
partition_label=partition_label,
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
def _metric_value(self, metrics: Mapping[str, Any]) -> float:
|
|
109
|
+
value = metrics.get(self.metric)
|
|
110
|
+
if type(value) not in (float, int) or not math.isfinite(value) or not 0 <= value <= 1:
|
|
111
|
+
raise ValidationError("ACE retrieval native metric is absent or outside [0,1]")
|
|
112
|
+
return float(value)
|
|
113
|
+
|
|
114
|
+
def fit_examples(self, split: str) -> tuple[ACEExample, ...]:
|
|
115
|
+
"""Build evaluator-blind query-generation inputs; targets are opaque IDs."""
|
|
116
|
+
if split not in {"train", "validation"}:
|
|
117
|
+
raise ValidationError("ACE retrieval exposes only train and validation groups")
|
|
118
|
+
examples = self.train if split == "train" else self.validation
|
|
119
|
+
return self._examples_for(examples, split)
|
|
120
|
+
|
|
121
|
+
def _examples_for(self, examples, split):
|
|
122
|
+
result = []
|
|
123
|
+
for example in examples:
|
|
124
|
+
state = example.visible_state()
|
|
125
|
+
visible = dict(
|
|
126
|
+
task_instruction=state.instruction,
|
|
127
|
+
original_query=state.query,
|
|
128
|
+
conversation=[dict(role=m.role, content=m.content) for m in state.messages],
|
|
129
|
+
)
|
|
130
|
+
result.append(
|
|
131
|
+
ACEExample(
|
|
132
|
+
example_id=example.example_id,
|
|
133
|
+
input=dict(
|
|
134
|
+
question=(
|
|
135
|
+
"Generate a search query for this retrieval task. Preserve its "
|
|
136
|
+
"information need and task instruction. Your final answer must be "
|
|
137
|
+
'exactly one JSON object: {"query": "nonempty search query"}.\n'
|
|
138
|
+
+ canonical_json(visible)
|
|
139
|
+
),
|
|
140
|
+
context=canonical_json(
|
|
141
|
+
[
|
|
142
|
+
dict(
|
|
143
|
+
document_id=b.block_id,
|
|
144
|
+
text=b.text,
|
|
145
|
+
source=b.source,
|
|
146
|
+
score=b.score,
|
|
147
|
+
)
|
|
148
|
+
for b in state.context
|
|
149
|
+
]
|
|
150
|
+
),
|
|
151
|
+
),
|
|
152
|
+
expected_output=example.example_id,
|
|
153
|
+
split=split,
|
|
154
|
+
metadata=dict(
|
|
155
|
+
task_processor="agentinputlab.ace.native_retrieval_feedback.v1",
|
|
156
|
+
benchmark_id=self.adapter.benchmark_id,
|
|
157
|
+
source_state_hash=state.state_hash,
|
|
158
|
+
partition_label=self.partition_label,
|
|
159
|
+
evaluator_target_is_opaque_id_not_reference_query=True,
|
|
160
|
+
),
|
|
161
|
+
)
|
|
162
|
+
)
|
|
163
|
+
return tuple(result)
|
|
164
|
+
|
|
165
|
+
def validate_examples(self, train: Sequence[Mapping], validation: Sequence[Mapping]) -> None:
|
|
166
|
+
"""Reject unbound IDs, altered inputs or an attempted held-out fit before IO."""
|
|
167
|
+
for supplied, split in ((train, "train"), (validation, "validation")):
|
|
168
|
+
expected = tuple(e.as_dict() for e in self.fit_examples(split))
|
|
169
|
+
if tuple(to_jsonable(e) for e in supplied) != expected:
|
|
170
|
+
raise ValidationError("ACE retrieval examples differ from the frozen fit groups")
|
|
171
|
+
|
|
172
|
+
def fingerprint(self) -> dict[str, Any]:
|
|
173
|
+
return dict(
|
|
174
|
+
processor_id="agentinputlab.ace.native_retrieval_feedback.v1",
|
|
175
|
+
dataset=self._dataset,
|
|
176
|
+
retriever=self._retriever,
|
|
177
|
+
metric=self.metric,
|
|
178
|
+
top_k=self.top_k,
|
|
179
|
+
allow_json_fence=self.allow_json_fence,
|
|
180
|
+
partition_label=self.partition_label,
|
|
181
|
+
training_success_criterion="declared_native_metric_strictly_greater_than_zero",
|
|
182
|
+
validation_selection="mean_declared_native_metric_not_text_exact_match",
|
|
183
|
+
inputs={
|
|
184
|
+
split: [
|
|
185
|
+
dict(
|
|
186
|
+
case_id=e.example_id,
|
|
187
|
+
visible_state_hash=e.visible_state().state_hash,
|
|
188
|
+
evaluator_gold_sha256=content_hash(e.evaluator_gold),
|
|
189
|
+
)
|
|
190
|
+
for e in cases
|
|
191
|
+
]
|
|
192
|
+
for split, cases in (
|
|
193
|
+
("train", self.train),
|
|
194
|
+
("validation", self.validation),
|
|
195
|
+
)
|
|
196
|
+
},
|
|
197
|
+
invalid_query_training_feedback=0.0,
|
|
198
|
+
automatic_retry=False,
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
def score_query(self, predicted: str, target: str) -> float:
|
|
202
|
+
if target not in self._examples:
|
|
203
|
+
raise ValidationError("ACE retrieval target is outside the frozen fit groups")
|
|
204
|
+
if to_jsonable(self.adapter.fingerprint()) != self._dataset:
|
|
205
|
+
raise ValidationError("ACE retrieval dataset fingerprint changed during fitting")
|
|
206
|
+
if to_jsonable(self.retriever.fingerprint()) != self._retriever:
|
|
207
|
+
raise ValidationError("ACE retrieval retriever fingerprint changed during fitting")
|
|
208
|
+
try:
|
|
209
|
+
query = parse_ir_query(predicted, allow_json_fence=self.allow_json_fence)
|
|
210
|
+
except AlgorithmOutputError:
|
|
211
|
+
self.events.append(
|
|
212
|
+
dict(
|
|
213
|
+
target=target,
|
|
214
|
+
status="invalid_query_training_feedback",
|
|
215
|
+
prediction_sha256=content_hash(predicted),
|
|
216
|
+
score=0.0,
|
|
217
|
+
)
|
|
218
|
+
)
|
|
219
|
+
return 0.0
|
|
220
|
+
key = target, query
|
|
221
|
+
if key in self._cache:
|
|
222
|
+
return self._cache[key]
|
|
223
|
+
results = tuple(self.retriever.retrieve(query, top_k=self.top_k))
|
|
224
|
+
if (
|
|
225
|
+
len(results) > self.top_k
|
|
226
|
+
or any(not isinstance(r, RetrievalResult) for r in results)
|
|
227
|
+
or tuple(r.rank for r in results) != tuple(range(1, len(results) + 1))
|
|
228
|
+
or len({r.document.doc_id for r in results}) != len(results)
|
|
229
|
+
):
|
|
230
|
+
raise ValidationError("ACE retrieval received invalid or duplicate native ranks")
|
|
231
|
+
prediction = RetrievalPrediction.from_ranked_ids([r.document.doc_id for r in results])
|
|
232
|
+
metrics = dict(self.adapter.score(self._examples[target], prediction))
|
|
233
|
+
score = self._metric_value(metrics)
|
|
234
|
+
self._cache[key] = score
|
|
235
|
+
self.events.append(
|
|
236
|
+
dict(
|
|
237
|
+
target=target,
|
|
238
|
+
status="native_retrieval_scored",
|
|
239
|
+
prediction_sha256=content_hash(predicted),
|
|
240
|
+
query_sha256=content_hash(query),
|
|
241
|
+
ranked_ids_sha256=content_hash([r.document.doc_id for r in results]),
|
|
242
|
+
metrics=metrics,
|
|
243
|
+
score=score,
|
|
244
|
+
)
|
|
245
|
+
)
|
|
246
|
+
return score
|
|
247
|
+
|
|
248
|
+
def answer_is_correct(self, predicted: str, ground_truth: str) -> bool:
|
|
249
|
+
return self.score_query(predicted, ground_truth) > 0.0
|
|
250
|
+
|
|
251
|
+
def evaluate_accuracy(self, predictions: Sequence[str], ground_truths: Sequence[str]) -> float:
|
|
252
|
+
if len(predictions) != len(ground_truths):
|
|
253
|
+
raise ValidationError("ACE retrieval predictions and targets must have equal length")
|
|
254
|
+
return (
|
|
255
|
+
sum(self.score_query(p, t) for p, t in zip(predictions, ground_truths, strict=True))
|
|
256
|
+
/ len(predictions)
|
|
257
|
+
if predictions
|
|
258
|
+
else 0.0
|
|
259
|
+
)
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
class ACEOnlineRetrievalFeedback(ACERetrievalFeedback):
|
|
263
|
+
"""Native IR feedback for an explicitly transductive ordered online stream.
|
|
264
|
+
|
|
265
|
+
Stream references are opaque case IDs; qrels remain evaluator-owned.
|
|
266
|
+
This processor is not an offline fitting split or held-out evaluation.
|
|
267
|
+
"""
|
|
268
|
+
|
|
269
|
+
def __init__(
|
|
270
|
+
self,
|
|
271
|
+
*,
|
|
272
|
+
adapter,
|
|
273
|
+
retriever,
|
|
274
|
+
stream,
|
|
275
|
+
metric="ndcg_at_10",
|
|
276
|
+
top_k=15,
|
|
277
|
+
allow_json_fence=False,
|
|
278
|
+
):
|
|
279
|
+
self.stream = tuple(stream)
|
|
280
|
+
_bind_feedback(
|
|
281
|
+
self,
|
|
282
|
+
adapter=adapter,
|
|
283
|
+
retriever=retriever,
|
|
284
|
+
groups=(self.stream,),
|
|
285
|
+
metric=metric,
|
|
286
|
+
top_k=top_k,
|
|
287
|
+
allow_json_fence=allow_json_fence,
|
|
288
|
+
partition_label="explicit_native_IR_feedback_transductive_online_stream",
|
|
289
|
+
)
|
|
290
|
+
|
|
291
|
+
def stream_examples(self):
|
|
292
|
+
return self._examples_for(self.stream, "test")
|
|
293
|
+
|
|
294
|
+
def validate_online_examples(self, examples):
|
|
295
|
+
rows = tuple(ACEExample.from_value(row, split="test") for row in examples)
|
|
296
|
+
if [r.as_dict() for r in rows] != [r.as_dict() for r in self.stream_examples()]:
|
|
297
|
+
raise ValidationError("ACE online retrieval inputs/order differ from the frozen stream")
|
|
298
|
+
|
|
299
|
+
def fit_examples(self, split):
|
|
300
|
+
raise ValidationError("online retrieval feedback cannot be used as offline fit groups")
|
|
301
|
+
|
|
302
|
+
def validate_examples(self, train, validation):
|
|
303
|
+
raise ValidationError("online retrieval feedback cannot be used for offline fitting")
|
|
304
|
+
|
|
305
|
+
def fingerprint(self):
|
|
306
|
+
return dict(
|
|
307
|
+
processor_id="agentinputlab.ace.native_online_retrieval_feedback.v1",
|
|
308
|
+
dataset=self._dataset,
|
|
309
|
+
retriever=self._retriever,
|
|
310
|
+
metric=self.metric,
|
|
311
|
+
top_k=self.top_k,
|
|
312
|
+
allow_json_fence=self.allow_json_fence,
|
|
313
|
+
partition_label=self.partition_label,
|
|
314
|
+
stream=[
|
|
315
|
+
dict(
|
|
316
|
+
case_id=e.example_id,
|
|
317
|
+
visible_state_hash=e.visible_state().state_hash,
|
|
318
|
+
evaluator_gold_sha256=content_hash(e.evaluator_gold),
|
|
319
|
+
)
|
|
320
|
+
for e in self.stream
|
|
321
|
+
],
|
|
322
|
+
transductive_feedback_not_offline_or_heldout=True,
|
|
323
|
+
success_criterion="declared_native_metric_strictly_greater_than_zero",
|
|
324
|
+
evaluation_value="mean_declared_native_metric_not_text_exact_match",
|
|
325
|
+
invalid_query_feedback=0.0,
|
|
326
|
+
automatic_retry=False,
|
|
327
|
+
)
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
__all__ = ["ACEOnlineRetrievalFeedback", "ACERetrievalFeedback"]
|