agentinputlab 0.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentinputlab/__init__.py +28 -0
- agentinputlab/algorithms/__init__.py +6 -0
- agentinputlab/algorithms/acquire/__init__.py +145 -0
- agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
- agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
- agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
- agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
- agentinputlab/algorithms/acquire/hyde.py +296 -0
- agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
- agentinputlab/algorithms/acquire/hyde_external.py +148 -0
- agentinputlab/algorithms/acquire/ircot.py +28 -0
- agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
- agentinputlab/algorithms/acquire/ircot_native.py +355 -0
- agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
- agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
- agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
- agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
- agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
- agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
- agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
- agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
- agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
- agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
- agentinputlab/algorithms/clarify/__init__.py +61 -0
- agentinputlab/algorithms/clarify/_structured.py +64 -0
- agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
- agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
- agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
- agentinputlab/algorithms/clarify/future_turns.py +379 -0
- agentinputlab/algorithms/clarify/udcs.py +255 -0
- agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
- agentinputlab/algorithms/curate/__init__.py +65 -0
- agentinputlab/algorithms/curate/acon/__init__.py +27 -0
- agentinputlab/algorithms/curate/acon/common.py +284 -0
- agentinputlab/algorithms/curate/acon/common_support.py +136 -0
- agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
- agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
- agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
- agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
- agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
- agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
- agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
- agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
- agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
- agentinputlab/algorithms/curate/recomp/external.py +1129 -0
- agentinputlab/algorithms/curate/recomp/selective.py +304 -0
- agentinputlab/algorithms/decompose/__init__.py +107 -0
- agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
- agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
- agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
- agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
- agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
- agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
- agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
- agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
- agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
- agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
- agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
- agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
- agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
- agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
- agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
- agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
- agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
- agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
- agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
- agentinputlab/algorithms/reformulate/__init__.py +85 -0
- agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
- agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
- agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
- agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
- agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
- agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
- agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
- agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
- agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
- agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
- agentinputlab/algorithms/reformulate/rrr.py +696 -0
- agentinputlab/annotation/__init__.py +5 -0
- agentinputlab/annotation/independent.py +428 -0
- agentinputlab/annotation/server.py +233 -0
- agentinputlab/annotation/static/app.js +121 -0
- agentinputlab/annotation/static/index.html +60 -0
- agentinputlab/annotation/static/review.css +2 -0
- agentinputlab/annotation/static/review.html +10 -0
- agentinputlab/annotation/static/review.js +11 -0
- agentinputlab/annotation/static/styles.css +55 -0
- agentinputlab/annotation/store.py +444 -0
- agentinputlab/api.py +368 -0
- agentinputlab/artifacts/__init__.py +15 -0
- agentinputlab/artifacts/json_file.py +104 -0
- agentinputlab/artifacts/manifest.py +98 -0
- agentinputlab/artifacts/store.py +157 -0
- agentinputlab/backends/__init__.py +98 -0
- agentinputlab/backends/embedding/__init__.py +25 -0
- agentinputlab/backends/embedding/base.py +145 -0
- agentinputlab/backends/embedding/callable.py +75 -0
- agentinputlab/backends/embedding/replay.py +129 -0
- agentinputlab/backends/llm/__init__.py +31 -0
- agentinputlab/backends/llm/base.py +149 -0
- agentinputlab/backends/llm/callable.py +108 -0
- agentinputlab/backends/llm/decomp_completion.py +88 -0
- agentinputlab/backends/llm/final_content_stop.py +72 -0
- agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
- agentinputlab/backends/llm/openai_compatible.py +332 -0
- agentinputlab/backends/llm/transformers.py +171 -0
- agentinputlab/backends/retrieval/__init__.py +35 -0
- agentinputlab/backends/retrieval/base.py +50 -0
- agentinputlab/backends/retrieval/dense.py +777 -0
- agentinputlab/backends/retrieval/memory.py +119 -0
- agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
- agentinputlab/backends/user/__init__.py +15 -0
- agentinputlab/backends/user/base.py +66 -0
- agentinputlab/backends/user/callable.py +54 -0
- agentinputlab/backends/user/replay.py +89 -0
- agentinputlab/backends/vector.py +38 -0
- agentinputlab/benchmarks/__init__.py +246 -0
- agentinputlab/benchmarks/acquisition.py +106 -0
- agentinputlab/benchmarks/ale.py +540 -0
- agentinputlab/benchmarks/appworld_ul.py +669 -0
- agentinputlab/benchmarks/assets.py +226 -0
- agentinputlab/benchmarks/base.py +68 -0
- agentinputlab/benchmarks/beir_archive.py +235 -0
- agentinputlab/benchmarks/beir_sample.py +269 -0
- agentinputlab/benchmarks/bfcl_native.py +245 -0
- agentinputlab/benchmarks/bfcl_v3.py +607 -0
- agentinputlab/benchmarks/cache.py +218 -0
- agentinputlab/benchmarks/clariq.py +203 -0
- agentinputlab/benchmarks/data/registry.json +321 -0
- agentinputlab/benchmarks/data/smoke_assets.json +511 -0
- agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
- agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
- agentinputlab/benchmarks/execution.py +1111 -0
- agentinputlab/benchmarks/interactive.py +257 -0
- agentinputlab/benchmarks/ir_query.py +204 -0
- agentinputlab/benchmarks/longbench.py +203 -0
- agentinputlab/benchmarks/paired_replay.py +479 -0
- agentinputlab/benchmarks/preprocess.py +2313 -0
- agentinputlab/benchmarks/production.py +236 -0
- agentinputlab/benchmarks/qa.py +963 -0
- agentinputlab/benchmarks/registry.py +110 -0
- agentinputlab/benchmarks/retrieval.py +459 -0
- agentinputlab/benchmarks/runtime.py +509 -0
- agentinputlab/benchmarks/statebench.py +938 -0
- agentinputlab/benchmarks/tau3.py +725 -0
- agentinputlab/benchmarks/toolsandbox.py +626 -0
- agentinputlab/benchmarks/trec_dl_sample.py +263 -0
- agentinputlab/campaign.py +654 -0
- agentinputlab/campaign_runner.py +953 -0
- agentinputlab/cli/__init__.py +6 -0
- agentinputlab/cli/main.py +933 -0
- agentinputlab/connectors/__init__.py +46 -0
- agentinputlab/connectors/knowledge/__init__.py +4 -0
- agentinputlab/connectors/tools/__init__.py +30 -0
- agentinputlab/connectors/tools/base.py +57 -0
- agentinputlab/connectors/tools/mcp.py +222 -0
- agentinputlab/connectors/tools/memory.py +41 -0
- agentinputlab/connectors/tools/openapi.py +695 -0
- agentinputlab/connectors/tools/schema.py +102 -0
- agentinputlab/contracts/__init__.py +91 -0
- agentinputlab/contracts/availability.py +24 -0
- agentinputlab/contracts/dependency.py +58 -0
- agentinputlab/contracts/enums.py +82 -0
- agentinputlab/contracts/errors.py +63 -0
- agentinputlab/contracts/evidence.py +43 -0
- agentinputlab/contracts/intent.py +110 -0
- agentinputlab/contracts/lifecycle.py +164 -0
- agentinputlab/contracts/patch.py +167 -0
- agentinputlab/contracts/serialization.py +93 -0
- agentinputlab/contracts/state.py +127 -0
- agentinputlab/contracts/trace.py +59 -0
- agentinputlab/contracts/worker.py +42 -0
- agentinputlab/core/__init__.py +7 -0
- agentinputlab/data/__init__.py +32 -0
- agentinputlab/data/checksums.py +9 -0
- agentinputlab/data/downloader.py +235 -0
- agentinputlab/data/http.py +171 -0
- agentinputlab/data/index.py +21 -0
- agentinputlab/data/overlap.py +28 -0
- agentinputlab/data/preprocess.py +32 -0
- agentinputlab/data/storage.py +57 -0
- agentinputlab/data/trec_sample.py +427 -0
- agentinputlab/evaluation/__init__.py +5 -0
- agentinputlab/evaluation/outcomes.py +32 -0
- agentinputlab/experiments/__init__.py +278 -0
- agentinputlab/experiments/analysis.py +981 -0
- agentinputlab/experiments/artifacts.py +687 -0
- agentinputlab/experiments/automatic_measurement.py +678 -0
- agentinputlab/experiments/auxiliary_matrix.py +203 -0
- agentinputlab/experiments/bindings.py +430 -0
- agentinputlab/experiments/blueprints.py +147 -0
- agentinputlab/experiments/builders/__init__.py +45 -0
- agentinputlab/experiments/builders/executable_external_payload.py +423 -0
- agentinputlab/experiments/builders/external_process_arena.py +1018 -0
- agentinputlab/experiments/builders/registry_intervention.py +463 -0
- agentinputlab/experiments/cli.py +158 -0
- agentinputlab/experiments/controls.py +340 -0
- agentinputlab/experiments/driver_guard.py +205 -0
- agentinputlab/experiments/drivers/__init__.py +89 -0
- agentinputlab/experiments/drivers/external_arena.py +2908 -0
- agentinputlab/experiments/drivers/method_process.py +662 -0
- agentinputlab/experiments/drivers/parity_process.py +1484 -0
- agentinputlab/experiments/drivers/static_production.py +1574 -0
- agentinputlab/experiments/executor.py +448 -0
- agentinputlab/experiments/loader.py +605 -0
- agentinputlab/experiments/measurement.py +675 -0
- agentinputlab/experiments/paper_analysis.py +1818 -0
- agentinputlab/experiments/parity_identity.py +605 -0
- agentinputlab/experiments/plan.py +277 -0
- agentinputlab/experiments/production_builders.py +3739 -0
- agentinputlab/experiments/readiness.py +432 -0
- agentinputlab/experiments/schema.py +514 -0
- agentinputlab/experiments/secrets.py +177 -0
- agentinputlab/experiments/stateful_analysis.py +1738 -0
- agentinputlab/experiments/transfer_analysis.py +1316 -0
- agentinputlab/experiments/unseen_analysis.py +1025 -0
- agentinputlab/integrations/__init__.py +1 -0
- agentinputlab/integrations/action_space/__init__.py +1 -0
- agentinputlab/integrations/external_workers/__init__.py +76 -0
- agentinputlab/integrations/external_workers/ace.py +431 -0
- agentinputlab/integrations/external_workers/base.py +181 -0
- agentinputlab/integrations/external_workers/decomp_live.py +249 -0
- agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
- agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
- agentinputlab/integrations/external_workers/ircot.py +576 -0
- agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
- agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
- agentinputlab/integrations/external_workers/method.py +314 -0
- agentinputlab/integrations/external_workers/process.py +678 -0
- agentinputlab/integrations/external_workers/supervisor.py +1071 -0
- agentinputlab/integrations/full_architecture/__init__.py +1 -0
- agentinputlab/integrations/full_architecture/hyde.py +212 -0
- agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
- agentinputlab/integrations/live_history.py +222 -0
- agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
- agentinputlab/integrations/training/__init__.py +1 -0
- agentinputlab/metrics/__init__.py +50 -0
- agentinputlab/metrics/catalog.py +313 -0
- agentinputlab/metrics/harm.py +95 -0
- agentinputlab/metrics/paired.py +40 -0
- agentinputlab/metrics/qa.py +76 -0
- agentinputlab/metrics/text.py +33 -0
- agentinputlab/metrics/third_party.py +161 -0
- agentinputlab/models/__init__.py +4 -0
- agentinputlab/observability/__init__.py +56 -0
- agentinputlab/observability/budget.py +316 -0
- agentinputlab/observability/instrumentation.py +754 -0
- agentinputlab/observability/models.py +403 -0
- agentinputlab/observability/monitor.py +380 -0
- agentinputlab/optimizers/__init__.py +1 -0
- agentinputlab/optimizers/clarify/__init__.py +12 -0
- agentinputlab/optimizers/context/__init__.py +22 -0
- agentinputlab/optimizers/decompose/__init__.py +64 -0
- agentinputlab/optimizers/evidence/__init__.py +22 -0
- agentinputlab/optimizers/rewrite/__init__.py +30 -0
- agentinputlab/parity/__init__.py +5 -0
- agentinputlab/parity/base.py +206 -0
- agentinputlab/production_campaign.py +763 -0
- agentinputlab/py.typed +1 -0
- agentinputlab/recipes/__init__.py +333 -0
- agentinputlab/recipes/ace.py +418 -0
- agentinputlab/recipes/ace_bfcl.py +273 -0
- agentinputlab/recipes/ace_common_support.py +169 -0
- agentinputlab/recipes/ace_native_plan.py +322 -0
- agentinputlab/recipes/ace_online.py +711 -0
- agentinputlab/recipes/ace_retrieval.py +330 -0
- agentinputlab/recipes/ace_statebench.py +382 -0
- agentinputlab/recipes/ace_tau.py +463 -0
- agentinputlab/recipes/ace_toolsandbox.py +432 -0
- agentinputlab/recipes/acon_guideline.py +22 -0
- agentinputlab/recipes/adaptive_rag.py +822 -0
- agentinputlab/recipes/llmlingua2_inference.py +146 -0
- agentinputlab/recipes/llmlingua2_training.py +796 -0
- agentinputlab/recipes/mce.py +1486 -0
- agentinputlab/recipes/mce_final.py +335 -0
- agentinputlab/recipes/mce_text_fit.py +499 -0
- agentinputlab/recipes/mce_text_only.py +298 -0
- agentinputlab/recipes/qrag.py +729 -0
- agentinputlab/recipes/qrag_inference.py +364 -0
- agentinputlab/recipes/r2ag_features.py +236 -0
- agentinputlab/recipes/r2ag_inference.py +503 -0
- agentinputlab/recipes/r2ag_training.py +948 -0
- agentinputlab/recipes/recomp_abstractive.py +592 -0
- agentinputlab/recipes/recomp_extractive.py +525 -0
- agentinputlab/recipes/reformer.py +17 -0
- agentinputlab/recipes/rqrag.py +1516 -0
- agentinputlab/recipes/rqrag_common_support.py +193 -0
- agentinputlab/recipes/rqrag_completion.py +147 -0
- agentinputlab/recipes/skillopt.py +462 -0
- agentinputlab/recipes/skillopt_common_support.py +154 -0
- agentinputlab/recipes/skillopt_environment.py +165 -0
- agentinputlab/recipes/skillopt_environment_cli.py +5 -0
- agentinputlab/recipes/skillopt_retrieval.py +188 -0
- agentinputlab/recipes/smartsearch.py +897 -0
- agentinputlab/recipes/smartsearch_curriculum.py +283 -0
- agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
- agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
- agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
- agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
- agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
- agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
- agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
- agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
- agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
- agentinputlab/recipes/smartsearch_inference.py +524 -0
- agentinputlab/recipes/smartsearch_native_data.py +172 -0
- agentinputlab/recipes/smartsearch_policy.py +616 -0
- agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
- agentinputlab/recipes/smartsearch_sampling.py +232 -0
- agentinputlab/recipes/smartsearch_teacher.py +203 -0
- agentinputlab/recipes/smartsearch_training.py +265 -0
- agentinputlab/registry/__init__.py +46 -0
- agentinputlab/registry/beta.py +727 -0
- agentinputlab/registry/data/algorithms.json +1959 -0
- agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
- agentinputlab/registry/loader.py +100 -0
- agentinputlab/registry/result_manifest.py +184 -0
- agentinputlab/registry/runtime.py +195 -0
- agentinputlab/registry/schema.py +449 -0
- agentinputlab/runner/__init__.py +30 -0
- agentinputlab/runner/balance.py +248 -0
- agentinputlab/runner/cache.py +135 -0
- agentinputlab/runner/composite.py +130 -0
- agentinputlab/runner/engine.py +285 -0
- agentinputlab/runner/runtime_guard.py +268 -0
- agentinputlab/runner/worker.py +80 -0
- agentinputlab/runtime_bootstrap.py +288 -0
- agentinputlab/selectors/__init__.py +196 -0
- agentinputlab/selectors/advanced.py +1514 -0
- agentinputlab/selectors/artifacts.py +1087 -0
- agentinputlab/selectors/base.py +112 -0
- agentinputlab/selectors/campaigns.py +1893 -0
- agentinputlab/selectors/catalog.py +86 -0
- agentinputlab/selectors/discovery.py +1860 -0
- agentinputlab/selectors/evaluation.py +1348 -0
- agentinputlab/selectors/features.py +383 -0
- agentinputlab/selectors/learned.py +502 -0
- agentinputlab/selectors/no_op.py +28 -0
- agentinputlab/selectors/outcome_table.py +475 -0
- agentinputlab/selectors/producer.py +853 -0
- agentinputlab/selectors/rules.py +109 -0
- agentinputlab/selectors/runtime.py +420 -0
- agentinputlab/studies/__init__.py +16 -0
- agentinputlab/studies/design.py +165 -0
- agentinputlab/studies/paired.py +322 -0
- agentinputlab/studio/__init__.py +6 -0
- agentinputlab/studio/campaigns.py +819 -0
- agentinputlab/studio/engine.py +1408 -0
- agentinputlab/studio/model_backends.py +425 -0
- agentinputlab/studio/prepared.py +410 -0
- agentinputlab/studio/server.py +613 -0
- agentinputlab/studio/static/app.js +397 -0
- agentinputlab/studio/static/campaigns.html +175 -0
- agentinputlab/studio/static/campaigns.js +426 -0
- agentinputlab/studio/static/index.html +192 -0
- agentinputlab/studio/static/result.html +103 -0
- agentinputlab/studio/static/result.js +273 -0
- agentinputlab/studio/static/styles.css +467 -0
- agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
- agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
- agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
- agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
- agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
- agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
- agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
- agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
- agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
- workers/README.md +94 -0
- workers/ace/README.md +261 -0
- workers/ace/__init__.py +1 -0
- workers/ace/compatibility_manifest.json +39 -0
- workers/ace/driver.py +1682 -0
- workers/acon/README.md +215 -0
- workers/acon/__init__.py +5 -0
- workers/acon/compatibility_manifest.json +93 -0
- workers/acon/driver.py +1859 -0
- workers/acon/fitted.py +395 -0
- workers/acon/guideline.py +470 -0
- workers/acon/guideline_source_manifest.json +20 -0
- workers/adaptive_rag/README.md +96 -0
- workers/adaptive_rag/__init__.py +15 -0
- workers/adaptive_rag/compatibility_manifest.json +61 -0
- workers/adaptive_rag/driver.py +1470 -0
- workers/adaptive_rag/fixture_runtime.py +163 -0
- workers/adaptive_rag/runtime.py +252 -0
- workers/ask_or_assume/README.md +124 -0
- workers/ask_or_assume/__init__.py +1 -0
- workers/ask_or_assume/compatibility_manifest.json +36 -0
- workers/ask_or_assume/driver.py +1227 -0
- workers/decomp/README.md +133 -0
- workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
- workers/decomp/__init__.py +5 -0
- workers/decomp/compatibility_manifest.json +205 -0
- workers/decomp/driver.py +1015 -0
- workers/decomp/live.py +137 -0
- workers/decomp/runtime.py +231 -0
- workers/infocqr/README.md +106 -0
- workers/infocqr/__init__.py +1 -0
- workers/infocqr/compatibility_manifest.json +84 -0
- workers/infocqr/driver.py +1766 -0
- workers/ircot/README.md +167 -0
- workers/ircot/__init__.py +7 -0
- workers/ircot/compatibility_manifest.json +121 -0
- workers/ircot/controlled.py +457 -0
- workers/ircot/driver.py +1373 -0
- workers/ircot/fixture_bindings.json +39 -0
- workers/ircot/fixture_config.json +48 -0
- workers/ircot/fixture_prompt.txt +2 -0
- workers/ircot/native_qa.py +398 -0
- workers/ircot/qa_compatibility_manifest.json +57 -0
- workers/ircot/runtime.py +97 -0
- workers/llmcompiler/README.md +160 -0
- workers/llmcompiler/__init__.py +1 -0
- workers/llmcompiler/compatibility_manifest.json +45 -0
- workers/llmcompiler/driver.py +1514 -0
- workers/llmcompiler/export_prompts.py +74 -0
- workers/llmcompiler/live.py +174 -0
- workers/llmcompiler/patch_manifest.json +147 -0
- workers/mce/README.md +198 -0
- workers/mce/__init__.py +29 -0
- workers/mce/compatibility_manifest.json +115 -0
- workers/mce/driver.py +2585 -0
- workers/mce/final_eval.py +173 -0
- workers/official_arenas/README.md +25 -0
- workers/official_arenas/__init__.py +1 -0
- workers/official_arenas/capture_records.py +110 -0
- workers/official_arenas/driver.py +619 -0
- workers/official_arenas/protocol.py +334 -0
- workers/official_arenas/providers/__init__.py +1 -0
- workers/official_arenas/providers/_live_history.py +228 -0
- workers/official_arenas/providers/_message_compat.py +77 -0
- workers/official_arenas/providers/appworld_ul.py +196 -0
- workers/official_arenas/providers/fixture.py +112 -0
- workers/official_arenas/providers/statebench.py +960 -0
- workers/official_arenas/providers/tau3.py +1672 -0
- workers/official_arenas/providers/tau_retail.py +586 -0
- workers/official_arenas/providers/toolsandbox.py +1321 -0
- workers/official_arenas/providers/tooltalk.py +699 -0
- workers/official_arenas/retail_capture.py +188 -0
- workers/official_arenas/retail_driver.py +275 -0
- workers/qrag/README.md +172 -0
- workers/qrag/__init__.py +1 -0
- workers/qrag/compatibility_manifest.json +64 -0
- workers/qrag/driver.py +673 -0
- workers/r2ag/README.md +154 -0
- workers/r2ag/__init__.py +5 -0
- workers/r2ag/compatibility_manifest.json +55 -0
- workers/r2ag/driver.py +1287 -0
- workers/r2ag/feature_schema.json +8 -0
- workers/r2ag/runtime.py +589 -0
- workers/recomp/README.md +120 -0
- workers/recomp/__init__.py +1 -0
- workers/recomp/compatibility_manifest.json +43 -0
- workers/recomp/driver.py +1653 -0
- workers/reformer/README.md +125 -0
- workers/reformer/__init__.py +1 -0
- workers/reformer/compatibility_manifest.json +68 -0
- workers/reformer/driver.py +1459 -0
- workers/reformer/induction.py +457 -0
- workers/rrr/README.md +126 -0
- workers/rrr/__init__.py +1 -0
- workers/rrr/compatibility_manifest.json +57 -0
- workers/rrr/driver.py +1133 -0
- workers/smartsearch/README.md +331 -0
- workers/smartsearch/__init__.py +1 -0
- workers/smartsearch/compatibility_manifest.json +41 -0
- workers/smartsearch/driver.py +1643 -0
|
@@ -0,0 +1,355 @@
|
|
|
1
|
+
"""Full native IRCoT-QA, with source-locked and explicit dependency substitutions."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
from collections.abc import Mapping
|
|
8
|
+
from dataclasses import asdict, dataclass, field
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from agentinputlab.artifacts import load_pinned_json_mapping
|
|
12
|
+
from agentinputlab.contracts import (
|
|
13
|
+
AlgorithmOutputError,
|
|
14
|
+
ArtifactError,
|
|
15
|
+
BudgetExceededError,
|
|
16
|
+
CapabilityError,
|
|
17
|
+
CompositeResult,
|
|
18
|
+
ContextBlock,
|
|
19
|
+
ExecutionTrace,
|
|
20
|
+
ExternalDependencyError,
|
|
21
|
+
FrozenAgentState,
|
|
22
|
+
InterventionContext,
|
|
23
|
+
SemanticFamily,
|
|
24
|
+
TraceEvent,
|
|
25
|
+
TracePhase,
|
|
26
|
+
)
|
|
27
|
+
from agentinputlab.contracts.serialization import content_hash, freeze_mapping, to_jsonable
|
|
28
|
+
from agentinputlab.integrations.external_workers.ircot import (
|
|
29
|
+
IRCOT_UPSTREAM_COMMIT,
|
|
30
|
+
IRCoTControls,
|
|
31
|
+
IRCoTWorkerRequest,
|
|
32
|
+
IRCoTWorkerResponse,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
IRCOT_NATIVE_QA_RELEASE = "acquire.ircot.native_like_substituted"
|
|
36
|
+
IRCOT_NATIVE_QA_BINDING_MODE = "agentinputlab.ircot.native_like_qa.v1"
|
|
37
|
+
IRCOT_NATIVE_QA_MANIFEST_SCHEMA = "agentinputlab.ircot.native_like_substitutions.v1"
|
|
38
|
+
_ROLES = {
|
|
39
|
+
"retrieve_and_reset_paragraphs",
|
|
40
|
+
"step_by_step_cot_gen",
|
|
41
|
+
"step_by_step_exit_controller",
|
|
42
|
+
"copy_question",
|
|
43
|
+
"llmqa",
|
|
44
|
+
"answer_extractor",
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _manifest(fingerprint):
|
|
49
|
+
if (
|
|
50
|
+
not isinstance(fingerprint, Mapping)
|
|
51
|
+
or fingerprint.get("binding_mode") != IRCOT_NATIVE_QA_BINDING_MODE
|
|
52
|
+
or fingerprint.get("upstream_commit") != IRCOT_UPSTREAM_COMMIT
|
|
53
|
+
or fingerprint.get("full_six_participant_QA_graph") is not True
|
|
54
|
+
or fingerprint.get("native_final_prediction_extraction") is not True
|
|
55
|
+
):
|
|
56
|
+
raise ArtifactError("IRCoT native-like profile requires the complete pinned QA runtime")
|
|
57
|
+
for key in ("generator", "retriever", "tokenizer", "native_dependencies"):
|
|
58
|
+
if not isinstance(fingerprint.get(key), Mapping) or not fingerprint[key]:
|
|
59
|
+
raise ArtifactError("IRCoT native-like runtime dependency fingerprints are incomplete")
|
|
60
|
+
for key in ("corpus_revision", "index_revision", "prompt_sha256"):
|
|
61
|
+
if not isinstance(fingerprint.get(key), str) or not fingerprint[key].strip():
|
|
62
|
+
raise ArtifactError("IRCoT native-like source identities are incomplete")
|
|
63
|
+
for key in (
|
|
64
|
+
"max_model_calls",
|
|
65
|
+
"max_output_tokens",
|
|
66
|
+
"max_retrieval_calls",
|
|
67
|
+
"per_call_output_tokens",
|
|
68
|
+
):
|
|
69
|
+
if type(fingerprint.get(key)) is not int or fingerprint[key] < 1:
|
|
70
|
+
raise ArtifactError("IRCoT native-like runtime budgets must be positive integers")
|
|
71
|
+
if type(fingerprint.get("seed")) is not int or fingerprint["seed"] < 0:
|
|
72
|
+
raise ArtifactError("IRCoT native-like runtime seed must be a nonnegative integer")
|
|
73
|
+
return dict(
|
|
74
|
+
schema=IRCOT_NATIVE_QA_MANIFEST_SCHEMA,
|
|
75
|
+
release_id=IRCOT_NATIVE_QA_RELEASE,
|
|
76
|
+
runtime_fingerprint=to_jsonable(fingerprint),
|
|
77
|
+
runtime_fingerprint_sha256=content_hash(fingerprint),
|
|
78
|
+
substitutions=[
|
|
79
|
+
"generation_backend",
|
|
80
|
+
"retrieval_backend_and_index",
|
|
81
|
+
"gpt2_tokenizer_resolver",
|
|
82
|
+
],
|
|
83
|
+
original_controller_and_six_participants=True,
|
|
84
|
+
native_answer_extraction=True,
|
|
85
|
+
original_native_generator_default_cap=300,
|
|
86
|
+
per_call_limit_is_a_ceiling_not_an_increase=True,
|
|
87
|
+
shared_final_call_and_output_budget=True,
|
|
88
|
+
original_cache_or_retry=False,
|
|
89
|
+
historical_Codex_full_Elasticsearch_or_published_score_parity=False,
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def write_ircot_substitution_manifest(runtime, output, *, accept_substitutions=False):
|
|
94
|
+
"""Create-only operator acceptance of the exact native-like dependency set."""
|
|
95
|
+
if accept_substitutions is not True:
|
|
96
|
+
raise ArtifactError(
|
|
97
|
+
"review the IRCoT model/index substitutions and acknowledge them explicitly"
|
|
98
|
+
)
|
|
99
|
+
value = _manifest(runtime.fingerprint())
|
|
100
|
+
path = Path(output).expanduser().absolute()
|
|
101
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
102
|
+
with path.open("x") as f:
|
|
103
|
+
json.dump(value, f, indent=2, sort_keys=True, allow_nan=False)
|
|
104
|
+
return dict(path=str(path), sha256=hashlib.sha256(path.read_bytes()).hexdigest())
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
@dataclass(frozen=True, slots=True)
|
|
108
|
+
class IRCoTNativeQARequest:
|
|
109
|
+
legacy: IRCoTWorkerRequest
|
|
110
|
+
max_model_calls: int
|
|
111
|
+
per_call_output_tokens: int
|
|
112
|
+
seed: int
|
|
113
|
+
expected_runtime_fingerprint_sha256: str
|
|
114
|
+
|
|
115
|
+
def external_reservation(self):
|
|
116
|
+
return {"model_calls": 1, "retrieval_calls": 1}
|
|
117
|
+
|
|
118
|
+
def raise_for_external_status(self, result):
|
|
119
|
+
if not isinstance(result, Mapping) or result.get("status") != "ok":
|
|
120
|
+
code = (
|
|
121
|
+
result.get("failure_code", "INVALID_RESPONSE")
|
|
122
|
+
if isinstance(result, Mapping)
|
|
123
|
+
else "INVALID_RESPONSE"
|
|
124
|
+
)
|
|
125
|
+
if code == "NATIVE_QA_EMPTY_OR_NON_TEXT":
|
|
126
|
+
raise AlgorithmOutputError(
|
|
127
|
+
"native IRCoT-QA preserved an empty/non-text final prediction"
|
|
128
|
+
)
|
|
129
|
+
raise ExternalDependencyError(f"native IRCoT-QA failed with {code}")
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
@dataclass(frozen=True, slots=True)
|
|
133
|
+
class IRCoTNativeLikeSubstituted:
|
|
134
|
+
manifest_path: str | Path
|
|
135
|
+
expected_manifest_sha256: str
|
|
136
|
+
controls: IRCoTControls | Mapping = field(
|
|
137
|
+
default_factory=lambda: IRCoTControls(
|
|
138
|
+
per_step_top_k=6, max_steps=7, max_total_documents=15, max_output_tokens=4096
|
|
139
|
+
)
|
|
140
|
+
)
|
|
141
|
+
method_id: str = "ircot"
|
|
142
|
+
release_id: str = IRCOT_NATIVE_QA_RELEASE
|
|
143
|
+
family: SemanticFamily = SemanticFamily.ACQUIRE
|
|
144
|
+
_binding: Mapping = field(init=False, repr=False)
|
|
145
|
+
|
|
146
|
+
def __post_init__(self):
|
|
147
|
+
if isinstance(self.controls, Mapping):
|
|
148
|
+
try:
|
|
149
|
+
object.__setattr__(self, "controls", IRCoTControls(**self.controls))
|
|
150
|
+
except (TypeError, ValueError) as exc:
|
|
151
|
+
raise ArtifactError("IRCoT full QA control fields are invalid") from exc
|
|
152
|
+
if not isinstance(self.controls, IRCoTControls):
|
|
153
|
+
raise ArtifactError("IRCoT full QA requires explicit controls")
|
|
154
|
+
for key in ("per_step_top_k", "max_steps", "max_total_documents", "max_output_tokens"):
|
|
155
|
+
if type(getattr(self.controls, key)) is not int or getattr(self.controls, key) < 1:
|
|
156
|
+
raise ArtifactError("IRCoT full QA controls must be positive integers")
|
|
157
|
+
if self.controls.deduplicate is not True:
|
|
158
|
+
raise ArtifactError("the pinned native IRCoT source always deduplicates")
|
|
159
|
+
if (
|
|
160
|
+
self.method_id != "ircot"
|
|
161
|
+
or self.release_id != IRCOT_NATIVE_QA_RELEASE
|
|
162
|
+
or self.family is not SemanticFamily.ACQUIRE
|
|
163
|
+
):
|
|
164
|
+
raise ArtifactError("IRCoT full QA registered profile identity differs")
|
|
165
|
+
value = load_pinned_json_mapping(
|
|
166
|
+
self.manifest_path, expected_sha256=self.expected_manifest_sha256
|
|
167
|
+
)
|
|
168
|
+
if value != _manifest(value.get("runtime_fingerprint")):
|
|
169
|
+
raise ArtifactError(
|
|
170
|
+
"IRCoT full QA substitution manifest differs from the exact supported protocol"
|
|
171
|
+
)
|
|
172
|
+
object.__setattr__(self, "_binding", freeze_mapping(value))
|
|
173
|
+
|
|
174
|
+
def run(self, state: FrozenAgentState, context: InterventionContext) -> CompositeResult:
|
|
175
|
+
if not state.query.strip() or not state.task_id:
|
|
176
|
+
raise CapabilityError("IRCoT full QA requires question text and an explicit task ID")
|
|
177
|
+
if state.context or state.messages or state.tool_names or state.instruction:
|
|
178
|
+
raise CapabilityError(
|
|
179
|
+
"IRCoT full QA accepts question-only state plus an explicitly bound external corpus"
|
|
180
|
+
)
|
|
181
|
+
binding = load_pinned_json_mapping(
|
|
182
|
+
self.manifest_path, expected_sha256=self.expected_manifest_sha256
|
|
183
|
+
)
|
|
184
|
+
if content_hash(binding) != content_hash(self._binding):
|
|
185
|
+
raise ArtifactError("IRCoT full QA substitution manifest changed")
|
|
186
|
+
runtime = context.external_runtime
|
|
187
|
+
if (
|
|
188
|
+
runtime is None
|
|
189
|
+
or not callable(getattr(runtime, "run", None))
|
|
190
|
+
or not callable(getattr(runtime, "fingerprint", None))
|
|
191
|
+
):
|
|
192
|
+
raise ExternalDependencyError(
|
|
193
|
+
"IRCoT full QA requires the startup-bound native QA runtime"
|
|
194
|
+
)
|
|
195
|
+
fingerprint = runtime.fingerprint()
|
|
196
|
+
if content_hash(fingerprint) != binding["runtime_fingerprint_sha256"]:
|
|
197
|
+
raise ArtifactError("IRCoT full QA exact runtime/model/index fingerprint mismatch")
|
|
198
|
+
for component, key in ((context.backend, "generator"), (context.retriever, "retriever")):
|
|
199
|
+
if component is None or not callable(getattr(component, "fingerprint", None)):
|
|
200
|
+
raise ExternalDependencyError(
|
|
201
|
+
"IRCoT full QA requires backend and retriever fingerprints"
|
|
202
|
+
)
|
|
203
|
+
if content_hash(component.fingerprint()) != content_hash(fingerprint[key]):
|
|
204
|
+
raise ArtifactError(
|
|
205
|
+
"IRCoT full QA context is not the acknowledged model/index substitution"
|
|
206
|
+
)
|
|
207
|
+
controls = self.controls
|
|
208
|
+
if (
|
|
209
|
+
context.seed != fingerprint["seed"]
|
|
210
|
+
or controls.max_steps + 1 > state.budget.max_model_calls
|
|
211
|
+
or controls.max_steps + 1 > fingerprint["max_model_calls"]
|
|
212
|
+
or controls.max_output_tokens
|
|
213
|
+
> min(state.budget.max_output_tokens, fingerprint["max_output_tokens"])
|
|
214
|
+
or controls.max_steps + 1
|
|
215
|
+
> min(state.budget.max_retrieval_calls, fingerprint["max_retrieval_calls"])
|
|
216
|
+
):
|
|
217
|
+
raise BudgetExceededError(
|
|
218
|
+
"IRCoT full QA cannot reserve its declared reasoning/final workflow under the bound budget/seed"
|
|
219
|
+
)
|
|
220
|
+
base = dict(
|
|
221
|
+
state_hash=state.state_hash,
|
|
222
|
+
task_id=state.task_id,
|
|
223
|
+
question=state.query,
|
|
224
|
+
controls=asdict(controls),
|
|
225
|
+
generator_fingerprint=fingerprint["generator"],
|
|
226
|
+
retriever_fingerprint=fingerprint["retriever"],
|
|
227
|
+
corpus_revision=fingerprint["corpus_revision"],
|
|
228
|
+
index_revision=fingerprint["index_revision"],
|
|
229
|
+
prompt_hash=fingerprint["prompt_sha256"],
|
|
230
|
+
)
|
|
231
|
+
legacy = IRCoTWorkerRequest(
|
|
232
|
+
request_id=content_hash(
|
|
233
|
+
dict(
|
|
234
|
+
base,
|
|
235
|
+
release_id=self.release_id,
|
|
236
|
+
manifest_sha256=self.expected_manifest_sha256,
|
|
237
|
+
seed=context.seed,
|
|
238
|
+
)
|
|
239
|
+
),
|
|
240
|
+
**{**base, "controls": controls},
|
|
241
|
+
)
|
|
242
|
+
request = IRCoTNativeQARequest(
|
|
243
|
+
legacy,
|
|
244
|
+
min(state.budget.max_model_calls, fingerprint["max_model_calls"]),
|
|
245
|
+
fingerprint["per_call_output_tokens"],
|
|
246
|
+
context.seed,
|
|
247
|
+
content_hash(fingerprint),
|
|
248
|
+
)
|
|
249
|
+
raw = runtime.run(request)
|
|
250
|
+
request.raise_for_external_status(raw)
|
|
251
|
+
if raw.get("identity") != fingerprint or content_hash(
|
|
252
|
+
runtime.fingerprint()
|
|
253
|
+
) != content_hash(fingerprint):
|
|
254
|
+
raise ArtifactError("IRCoT full QA response/runtime identity changed")
|
|
255
|
+
response = IRCoTWorkerResponse.from_mapping(raw, request=legacy)
|
|
256
|
+
roles = [e["role"] for e in raw["native_controller_events"]]
|
|
257
|
+
# The original CopyQuestion participant schedules a task. After answer
|
|
258
|
+
# extraction the source controller visits it again to drain that task;
|
|
259
|
+
# its eoq_after_n_calls=1 guard ends without another model generation.
|
|
260
|
+
tail = ["copy_question", "llmqa", "answer_extractor", "copy_question"]
|
|
261
|
+
prefix = roles[:-4]
|
|
262
|
+
loop = [
|
|
263
|
+
"retrieve_and_reset_paragraphs",
|
|
264
|
+
"step_by_step_cot_gen",
|
|
265
|
+
"step_by_step_exit_controller",
|
|
266
|
+
]
|
|
267
|
+
if (
|
|
268
|
+
not set(roles) >= _ROLES
|
|
269
|
+
or roles[-4:] != tail
|
|
270
|
+
or not prefix
|
|
271
|
+
or len(prefix) % 3
|
|
272
|
+
or any(prefix[i : i + 3] != loop for i in range(0, len(prefix), 3))
|
|
273
|
+
):
|
|
274
|
+
raise AlgorithmOutputError(
|
|
275
|
+
"IRCoT full QA did not execute the complete native final-answer graph"
|
|
276
|
+
)
|
|
277
|
+
if (
|
|
278
|
+
not response.answer.strip()
|
|
279
|
+
or response.usage["model_calls"] > request.max_model_calls
|
|
280
|
+
or response.usage["model_calls"] != roles.count("step_by_step_cot_gen") + 1
|
|
281
|
+
):
|
|
282
|
+
raise AlgorithmOutputError(
|
|
283
|
+
"IRCoT full QA answer/model accounting differs from its native graph"
|
|
284
|
+
)
|
|
285
|
+
by_id = {d.document_id: d for d in response.documents}
|
|
286
|
+
evidence = tuple(
|
|
287
|
+
ContextBlock(
|
|
288
|
+
text=by_id[i].text,
|
|
289
|
+
block_id=i,
|
|
290
|
+
source=response.documents[0].corpus_revision,
|
|
291
|
+
score=by_id[i].score,
|
|
292
|
+
metadata={
|
|
293
|
+
"title": by_id[i].title,
|
|
294
|
+
"corpus_revision": by_id[i].corpus_revision,
|
|
295
|
+
"evidence_kind": "real_corpus_document",
|
|
296
|
+
},
|
|
297
|
+
)
|
|
298
|
+
for i in response.active_document_ids
|
|
299
|
+
)
|
|
300
|
+
return CompositeResult(
|
|
301
|
+
method_id=self.method_id,
|
|
302
|
+
release_id=self.release_id,
|
|
303
|
+
source_state_hash=state.state_hash,
|
|
304
|
+
final_answer=response.answer,
|
|
305
|
+
evidence=evidence,
|
|
306
|
+
trace=ExecutionTrace(
|
|
307
|
+
method_id=self.method_id,
|
|
308
|
+
release_id=self.release_id,
|
|
309
|
+
events=(
|
|
310
|
+
TraceEvent(TracePhase.PREPARE, "native_qa_substitution_binding", binding),
|
|
311
|
+
TraceEvent(
|
|
312
|
+
TracePhase.OBSERVE,
|
|
313
|
+
"native_qa_controller",
|
|
314
|
+
{
|
|
315
|
+
"events": raw["native_controller_events"],
|
|
316
|
+
"model_classes": raw["native_model_classes"],
|
|
317
|
+
"runtime_events": raw["controlled_runtime_events"],
|
|
318
|
+
},
|
|
319
|
+
),
|
|
320
|
+
TraceEvent(
|
|
321
|
+
TracePhase.FINALIZE,
|
|
322
|
+
"native_qa_prediction",
|
|
323
|
+
{
|
|
324
|
+
"answer": response.answer,
|
|
325
|
+
"exit_reason": response.exit_reason,
|
|
326
|
+
"active_document_ids": response.active_document_ids,
|
|
327
|
+
"native_chain": raw["native_chain"],
|
|
328
|
+
},
|
|
329
|
+
),
|
|
330
|
+
),
|
|
331
|
+
counters=dict(response.usage),
|
|
332
|
+
),
|
|
333
|
+
implementation_identity=IRCOT_NATIVE_QA_BINDING_MODE,
|
|
334
|
+
profile_kind="controlled_adapter",
|
|
335
|
+
metadata={
|
|
336
|
+
"substitution_manifest_sha256": self.expected_manifest_sha256,
|
|
337
|
+
"runtime_fingerprint": fingerprint,
|
|
338
|
+
"native_full_QA_graph_executed": True,
|
|
339
|
+
"native_final_prediction_extraction": True,
|
|
340
|
+
"independent_downstream_answer_call_added": False,
|
|
341
|
+
"native_generator_default_output_cap": 300,
|
|
342
|
+
"current_provider_usage_is_measured": raw["usage_is_complete"],
|
|
343
|
+
"original_historical_backbone_index_or_published_score_parity": False,
|
|
344
|
+
"automatic_retry": False,
|
|
345
|
+
"official_scoring": "caller_owned_independent_scorer",
|
|
346
|
+
},
|
|
347
|
+
)
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
__all__ = [
|
|
351
|
+
"IRCOT_NATIVE_QA_RELEASE",
|
|
352
|
+
"IRCoTNativeLikeSubstituted",
|
|
353
|
+
"IRCoTNativeQARequest",
|
|
354
|
+
"write_ircot_substitution_manifest",
|
|
355
|
+
]
|
|
@@ -0,0 +1,327 @@
|
|
|
1
|
+
"""Checksum-bound replay of original IRCoT ID-to-answer prediction files.
|
|
2
|
+
|
|
3
|
+
The archive remains operator-owned. Loading verifies bytes and declared input/
|
|
4
|
+
model bindings, not authorship or published-score parity. No prediction is
|
|
5
|
+
passed off as corpus evidence and no model request is made during replay.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from collections.abc import Mapping
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
from pathlib import Path, PurePosixPath
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
from agentinputlab.artifacts import load_pinned_json_mapping
|
|
16
|
+
from agentinputlab.contracts import (
|
|
17
|
+
AlgorithmOutputError,
|
|
18
|
+
ArtifactError,
|
|
19
|
+
CompositeResult,
|
|
20
|
+
ExecutionTrace,
|
|
21
|
+
FrozenAgentState,
|
|
22
|
+
InterventionContext,
|
|
23
|
+
SemanticFamily,
|
|
24
|
+
TraceEvent,
|
|
25
|
+
TracePhase,
|
|
26
|
+
ValidationError,
|
|
27
|
+
)
|
|
28
|
+
from agentinputlab.contracts.serialization import content_hash, freeze_mapping
|
|
29
|
+
|
|
30
|
+
IRCOT_REPLAY_RELEASE = "acquire.ircot.prediction_replay"
|
|
31
|
+
IRCOT_REPLAY_SCHEMA = "agentinputlab.ircot.prediction_replay.v1"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _sha(value, label):
|
|
35
|
+
if (
|
|
36
|
+
not isinstance(value, str)
|
|
37
|
+
or len(value) != 64
|
|
38
|
+
or any(c not in "0123456789abcdef" for c in value)
|
|
39
|
+
):
|
|
40
|
+
raise ArtifactError(f"IRCoT replay {label} must be a lowercase SHA-256")
|
|
41
|
+
return value
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _mounted_json(root, spec):
|
|
45
|
+
if not isinstance(spec, Mapping) or set(spec) != {"path", "sha256"}:
|
|
46
|
+
raise ArtifactError("IRCoT replay files require a relative path and trusted checksum")
|
|
47
|
+
name = spec["path"]
|
|
48
|
+
if not isinstance(name, str) or not name or "\\" in name:
|
|
49
|
+
raise ArtifactError("IRCoT replay file path is invalid")
|
|
50
|
+
path = PurePosixPath(name)
|
|
51
|
+
if path.is_absolute() or ".." in path.parts or not path.parts:
|
|
52
|
+
raise ArtifactError("IRCoT replay file path must remain under its manifest directory")
|
|
53
|
+
target = root.joinpath(*path.parts)
|
|
54
|
+
if not target.parent.resolve().is_relative_to(root):
|
|
55
|
+
raise ArtifactError("IRCoT replay file parent escapes its manifest directory")
|
|
56
|
+
return load_pinned_json_mapping(
|
|
57
|
+
target, expected_sha256=_sha(spec["sha256"], "file checksum"), max_bytes=8 * 1024 * 1024
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@dataclass(frozen=True, slots=True)
|
|
62
|
+
class IRCoTPredictionArchive:
|
|
63
|
+
"""Immutable selected scope, including empty predictions and historical cost."""
|
|
64
|
+
|
|
65
|
+
manifest_sha256: str
|
|
66
|
+
manifest: Mapping[str, Any]
|
|
67
|
+
records: Mapping[str, Any]
|
|
68
|
+
source_prediction_count: int
|
|
69
|
+
|
|
70
|
+
def __post_init__(self):
|
|
71
|
+
object.__setattr__(self, "manifest", freeze_mapping(self.manifest))
|
|
72
|
+
object.__setattr__(self, "records", freeze_mapping(self.records))
|
|
73
|
+
|
|
74
|
+
@property
|
|
75
|
+
def generator_fingerprint_sha256(self):
|
|
76
|
+
return content_hash(self.manifest["generator_fingerprint"])
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def load_ircot_prediction_archive(manifest_path, *, expected_sha256):
|
|
80
|
+
"""Load a bounded operator manifest plus an unchanged upstream-format file.
|
|
81
|
+
|
|
82
|
+
``inputs`` is a label-free binding map, never a ground-truth answer file.
|
|
83
|
+
It may select a documented ID-based scope from a larger prediction file.
|
|
84
|
+
Unselected predictions remain counted in ``source_prediction_count``.
|
|
85
|
+
"""
|
|
86
|
+
pin = _sha(expected_sha256, "manifest checksum")
|
|
87
|
+
path = Path(manifest_path).expanduser().absolute()
|
|
88
|
+
manifest = load_pinned_json_mapping(path, expected_sha256=pin, max_bytes=1024 * 1024)
|
|
89
|
+
required = {
|
|
90
|
+
"schema",
|
|
91
|
+
"origin_kind",
|
|
92
|
+
"run_revision",
|
|
93
|
+
"benchmark",
|
|
94
|
+
"split",
|
|
95
|
+
"generator_fingerprint",
|
|
96
|
+
"dependency_fingerprint",
|
|
97
|
+
"provenance",
|
|
98
|
+
"selection_rule",
|
|
99
|
+
"predictions",
|
|
100
|
+
"inputs",
|
|
101
|
+
}
|
|
102
|
+
if set(manifest) != required or manifest["schema"] != IRCOT_REPLAY_SCHEMA:
|
|
103
|
+
raise ArtifactError("IRCoT replay manifest schema/fields differ")
|
|
104
|
+
if manifest["origin_kind"] not in ("author_release", "controlled_capture"):
|
|
105
|
+
raise ArtifactError(
|
|
106
|
+
"IRCoT replay origin must be an author release or separately labelled controlled capture"
|
|
107
|
+
)
|
|
108
|
+
revision = manifest["run_revision"]
|
|
109
|
+
if (
|
|
110
|
+
not isinstance(revision, str)
|
|
111
|
+
or len(revision) != 40
|
|
112
|
+
or any(c not in "0123456789abcdef" for c in revision)
|
|
113
|
+
):
|
|
114
|
+
raise ArtifactError(
|
|
115
|
+
"IRCoT replay run_revision must be the actual full immutable Git revision"
|
|
116
|
+
)
|
|
117
|
+
for key in ("benchmark", "split", "selection_rule"):
|
|
118
|
+
if not isinstance(manifest[key], str) or not manifest[key].strip():
|
|
119
|
+
raise ArtifactError(f"IRCoT replay requires {key}")
|
|
120
|
+
for key in ("generator_fingerprint", "dependency_fingerprint"):
|
|
121
|
+
if not isinstance(manifest[key], Mapping) or not manifest[key]:
|
|
122
|
+
raise ArtifactError(f"IRCoT replay requires {key}")
|
|
123
|
+
content_hash(manifest[key])
|
|
124
|
+
provenance = manifest["provenance"]
|
|
125
|
+
if not isinstance(provenance, Mapping) or set(provenance) != {
|
|
126
|
+
"source_uri",
|
|
127
|
+
"source_artifact_sha256",
|
|
128
|
+
"run_config_sha256",
|
|
129
|
+
"query_source_sha256",
|
|
130
|
+
}:
|
|
131
|
+
raise ArtifactError(
|
|
132
|
+
"IRCoT replay requires explicit source, configuration and query provenance"
|
|
133
|
+
)
|
|
134
|
+
if not isinstance(provenance["source_uri"], str) or not provenance["source_uri"].strip():
|
|
135
|
+
raise ArtifactError("IRCoT replay source_uri is required")
|
|
136
|
+
for key in ("source_artifact_sha256", "run_config_sha256", "query_source_sha256"):
|
|
137
|
+
_sha(provenance[key], key)
|
|
138
|
+
root = path.parent.resolve()
|
|
139
|
+
predictions = _mounted_json(root, manifest["predictions"])
|
|
140
|
+
bindings = _mounted_json(root, manifest["inputs"])
|
|
141
|
+
if not predictions or not bindings or not set(bindings) <= set(predictions):
|
|
142
|
+
raise ArtifactError("IRCoT replay selected IDs must exist in the original prediction file")
|
|
143
|
+
if any(
|
|
144
|
+
not isinstance(k, str) or not k or not isinstance(v, str) for k, v in predictions.items()
|
|
145
|
+
):
|
|
146
|
+
raise ArtifactError(
|
|
147
|
+
"IRCoT answer replay requires the original ID-to-text format, not retrieval IDs or gold answer lists"
|
|
148
|
+
)
|
|
149
|
+
records = {}
|
|
150
|
+
for key, binding in bindings.items():
|
|
151
|
+
if not isinstance(binding, Mapping) or set(binding) != {
|
|
152
|
+
"query",
|
|
153
|
+
"input_state_hash",
|
|
154
|
+
"historical_usage",
|
|
155
|
+
"origin_receipt_sha256",
|
|
156
|
+
}:
|
|
157
|
+
raise ArtifactError("IRCoT replay input binding fields differ; labels are not allowed")
|
|
158
|
+
if not isinstance(binding["query"], str) or not binding["query"].strip():
|
|
159
|
+
raise ArtifactError("IRCoT replay input binding requires its exact source query")
|
|
160
|
+
for name in ("input_state_hash", "origin_receipt_sha256"):
|
|
161
|
+
if binding[name] is not None:
|
|
162
|
+
_sha(binding[name], name)
|
|
163
|
+
historical = binding["historical_usage"]
|
|
164
|
+
if historical is not None:
|
|
165
|
+
if not isinstance(historical, Mapping) or not historical:
|
|
166
|
+
raise ArtifactError(
|
|
167
|
+
"IRCoT replay historical_usage must be measured data or null, never a guessed zero"
|
|
168
|
+
)
|
|
169
|
+
from agentinputlab.observability import normalize_external_usage
|
|
170
|
+
|
|
171
|
+
normalize_external_usage(historical)
|
|
172
|
+
if (
|
|
173
|
+
manifest["origin_kind"] == "controlled_capture"
|
|
174
|
+
and binding["origin_receipt_sha256"] is None
|
|
175
|
+
):
|
|
176
|
+
raise ArtifactError(
|
|
177
|
+
"IRCoT controlled capture requires the original inference receipt checksum"
|
|
178
|
+
)
|
|
179
|
+
records[key] = {
|
|
180
|
+
**dict(binding),
|
|
181
|
+
"answer": predictions[key],
|
|
182
|
+
"status": "archived_prediction"
|
|
183
|
+
if predictions[key].strip()
|
|
184
|
+
else "archived_empty_prediction",
|
|
185
|
+
}
|
|
186
|
+
return IRCoTPredictionArchive(pin, manifest, records, len(predictions))
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
@dataclass(frozen=True, slots=True)
|
|
190
|
+
class IRCoTPredictionReplay:
|
|
191
|
+
manifest_path: str | Path
|
|
192
|
+
expected_manifest_sha256: str
|
|
193
|
+
expected_generator_fingerprint_sha256: str
|
|
194
|
+
benchmark: str
|
|
195
|
+
split: str
|
|
196
|
+
method_id: str = "ircot"
|
|
197
|
+
release_id: str = IRCOT_REPLAY_RELEASE
|
|
198
|
+
family: SemanticFamily = SemanticFamily.ACQUIRE
|
|
199
|
+
_archive: IRCoTPredictionArchive = field(init=False, repr=False)
|
|
200
|
+
|
|
201
|
+
def __post_init__(self):
|
|
202
|
+
if (
|
|
203
|
+
self.method_id != "ircot"
|
|
204
|
+
or self.release_id != IRCOT_REPLAY_RELEASE
|
|
205
|
+
or self.family is not SemanticFamily.ACQUIRE
|
|
206
|
+
):
|
|
207
|
+
raise ValidationError("IRCoT replay identity differs from the registered profile")
|
|
208
|
+
archive = load_ircot_prediction_archive(
|
|
209
|
+
self.manifest_path, expected_sha256=self.expected_manifest_sha256
|
|
210
|
+
)
|
|
211
|
+
if archive.generator_fingerprint_sha256 != _sha(
|
|
212
|
+
self.expected_generator_fingerprint_sha256, "expected generator"
|
|
213
|
+
):
|
|
214
|
+
raise ArtifactError("IRCoT replay belongs to a different model/deployment")
|
|
215
|
+
if (
|
|
216
|
+
archive.manifest["benchmark"] != self.benchmark
|
|
217
|
+
or archive.manifest["split"] != self.split
|
|
218
|
+
):
|
|
219
|
+
raise ArtifactError("IRCoT replay benchmark/split does not match the requested scope")
|
|
220
|
+
object.__setattr__(self, "_archive", archive)
|
|
221
|
+
|
|
222
|
+
@property
|
|
223
|
+
def archive(self):
|
|
224
|
+
"""Inspect original empty outcomes/provenance without fabricating answers."""
|
|
225
|
+
return self._archive
|
|
226
|
+
|
|
227
|
+
def run(self, state: FrozenAgentState, context: InterventionContext) -> CompositeResult:
|
|
228
|
+
# Recheck trusted files before reuse; there is never a live fallback.
|
|
229
|
+
archive = load_ircot_prediction_archive(
|
|
230
|
+
self.manifest_path, expected_sha256=self.expected_manifest_sha256
|
|
231
|
+
)
|
|
232
|
+
if archive != self._archive:
|
|
233
|
+
raise ArtifactError("IRCoT replay archive changed after construction")
|
|
234
|
+
record = archive.records.get(state.task_id)
|
|
235
|
+
if record is None or record["query"] != state.query:
|
|
236
|
+
raise ArtifactError("IRCoT replay exact task ID/query mismatch; no generation fallback")
|
|
237
|
+
if (
|
|
238
|
+
record["input_state_hash"] is not None
|
|
239
|
+
and record["input_state_hash"] != state.state_hash
|
|
240
|
+
):
|
|
241
|
+
raise ArtifactError("IRCoT replay original input-state hash differs")
|
|
242
|
+
for key, value in (("benchmark_cell", self.benchmark), ("source_split", self.split)):
|
|
243
|
+
if key in state.metadata and state.metadata[key] != value:
|
|
244
|
+
raise ArtifactError("IRCoT replay visible dataset scope differs")
|
|
245
|
+
if (
|
|
246
|
+
context.backend is not None
|
|
247
|
+
and content_hash(context.backend.fingerprint()) != archive.generator_fingerprint_sha256
|
|
248
|
+
):
|
|
249
|
+
raise ArtifactError(
|
|
250
|
+
"IRCoT replay cannot relabel an archived answer as another live backbone"
|
|
251
|
+
)
|
|
252
|
+
if not record["answer"].strip():
|
|
253
|
+
raise AlgorithmOutputError(
|
|
254
|
+
"IRCoT archived empty prediction preserved; no retry or replacement answer"
|
|
255
|
+
)
|
|
256
|
+
manifest = archive.manifest
|
|
257
|
+
return CompositeResult(
|
|
258
|
+
method_id=self.method_id,
|
|
259
|
+
release_id=self.release_id,
|
|
260
|
+
source_state_hash=state.state_hash,
|
|
261
|
+
final_answer=record["answer"],
|
|
262
|
+
evidence=(),
|
|
263
|
+
trace=ExecutionTrace(
|
|
264
|
+
method_id=self.method_id,
|
|
265
|
+
release_id=self.release_id,
|
|
266
|
+
events=(
|
|
267
|
+
TraceEvent(
|
|
268
|
+
TracePhase.PREPARE,
|
|
269
|
+
"prediction_archive_binding",
|
|
270
|
+
{
|
|
271
|
+
"manifest_sha256": archive.manifest_sha256,
|
|
272
|
+
"source_prediction_count": archive.source_prediction_count,
|
|
273
|
+
"selected_record_count": len(archive.records),
|
|
274
|
+
"task_id": state.task_id,
|
|
275
|
+
"generator_fingerprint_sha256": archive.generator_fingerprint_sha256,
|
|
276
|
+
},
|
|
277
|
+
),
|
|
278
|
+
TraceEvent(
|
|
279
|
+
TracePhase.FINALIZE,
|
|
280
|
+
"archived_prediction",
|
|
281
|
+
{
|
|
282
|
+
"answer": record["answer"],
|
|
283
|
+
"origin_receipt_sha256": record["origin_receipt_sha256"],
|
|
284
|
+
"historical_usage": record["historical_usage"],
|
|
285
|
+
},
|
|
286
|
+
),
|
|
287
|
+
),
|
|
288
|
+
counters={
|
|
289
|
+
"model_calls": 0,
|
|
290
|
+
"retrieval_calls": 0,
|
|
291
|
+
"input_tokens": 0,
|
|
292
|
+
"output_tokens": 0,
|
|
293
|
+
},
|
|
294
|
+
),
|
|
295
|
+
implementation_identity=IRCOT_REPLAY_SCHEMA,
|
|
296
|
+
profile_kind="official_native"
|
|
297
|
+
if manifest["origin_kind"] == "author_release"
|
|
298
|
+
else "controlled_adapter",
|
|
299
|
+
metadata={
|
|
300
|
+
"artifact_replay": True,
|
|
301
|
+
"origin_kind": manifest["origin_kind"],
|
|
302
|
+
"run_revision": manifest["run_revision"],
|
|
303
|
+
"manifest_sha256": archive.manifest_sha256,
|
|
304
|
+
"generator_fingerprint": manifest["generator_fingerprint"],
|
|
305
|
+
"dependency_fingerprint": manifest["dependency_fingerprint"],
|
|
306
|
+
"provenance": manifest["provenance"],
|
|
307
|
+
"historical_usage": record["historical_usage"],
|
|
308
|
+
"historical_cost_recharged": False,
|
|
309
|
+
"historical_usage_unknown": record["historical_usage"] is None,
|
|
310
|
+
"new_provider_requests": 0,
|
|
311
|
+
"current_seed_or_budget_rerun_claimed": False,
|
|
312
|
+
"input_state_hash_verified": record["input_state_hash"] is not None,
|
|
313
|
+
"answers_are_not_corpus_evidence": True,
|
|
314
|
+
"checksum_proves_authorship": False,
|
|
315
|
+
"benchmark_scoring": "caller_owned_independent_scorer",
|
|
316
|
+
"automatic_retry": False,
|
|
317
|
+
},
|
|
318
|
+
)
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
__all__ = [
|
|
322
|
+
"IRCOT_REPLAY_RELEASE",
|
|
323
|
+
"IRCOT_REPLAY_SCHEMA",
|
|
324
|
+
"IRCoTPredictionArchive",
|
|
325
|
+
"IRCoTPredictionReplay",
|
|
326
|
+
"load_ircot_prediction_archive",
|
|
327
|
+
]
|