agentinputlab 0.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentinputlab/__init__.py +28 -0
- agentinputlab/algorithms/__init__.py +6 -0
- agentinputlab/algorithms/acquire/__init__.py +145 -0
- agentinputlab/algorithms/acquire/adaptive_rag/__init__.py +43 -0
- agentinputlab/algorithms/acquire/adaptive_rag/replay.py +367 -0
- agentinputlab/algorithms/acquire/adaptive_rag/runtime.py +1202 -0
- agentinputlab/algorithms/acquire/adaptive_rag_common_support.py +542 -0
- agentinputlab/algorithms/acquire/hyde.py +296 -0
- agentinputlab/algorithms/acquire/hyde_controlled.py +200 -0
- agentinputlab/algorithms/acquire/hyde_external.py +148 -0
- agentinputlab/algorithms/acquire/ircot.py +28 -0
- agentinputlab/algorithms/acquire/ircot_common_support.py +478 -0
- agentinputlab/algorithms/acquire/ircot_native.py +355 -0
- agentinputlab/algorithms/acquire/ircot_replay.py +327 -0
- agentinputlab/algorithms/acquire/qrag/__init__.py +79 -0
- agentinputlab/algorithms/acquire/qrag/bridge.py +828 -0
- agentinputlab/algorithms/acquire/qrag/common_support.py +891 -0
- agentinputlab/algorithms/acquire/qrag/native_axis.py +71 -0
- agentinputlab/algorithms/acquire/qrag/reader_adapter.py +139 -0
- agentinputlab/algorithms/acquire/r2ag/__init__.py +57 -0
- agentinputlab/algorithms/acquire/r2ag/bridge.py +1312 -0
- agentinputlab/algorithms/acquire/r2ag/common_support.py +1029 -0
- agentinputlab/algorithms/acquire/r2ag/socket_runtime.py +255 -0
- agentinputlab/algorithms/clarify/__init__.py +61 -0
- agentinputlab/algorithms/clarify/_structured.py +64 -0
- agentinputlab/algorithms/clarify/active_task_disambiguation.py +371 -0
- agentinputlab/algorithms/clarify/ask_or_assume/__init__.py +17 -0
- agentinputlab/algorithms/clarify/ask_or_assume/monitor.py +410 -0
- agentinputlab/algorithms/clarify/future_turns.py +379 -0
- agentinputlab/algorithms/clarify/udcs.py +255 -0
- agentinputlab/algorithms/clarify/udcs_deterministic.py +196 -0
- agentinputlab/algorithms/curate/__init__.py +65 -0
- agentinputlab/algorithms/curate/acon/__init__.py +27 -0
- agentinputlab/algorithms/curate/acon/common.py +284 -0
- agentinputlab/algorithms/curate/acon/common_support.py +136 -0
- agentinputlab/algorithms/curate/acon/history_bridge.py +309 -0
- agentinputlab/algorithms/curate/acon/observation_bridge.py +290 -0
- agentinputlab/algorithms/curate/llmlingua/__init__.py +30 -0
- agentinputlab/algorithms/curate/llmlingua/adapters.py +537 -0
- agentinputlab/algorithms/curate/llmlingua/budget.py +48 -0
- agentinputlab/algorithms/curate/llmlingua/controlled.py +175 -0
- agentinputlab/algorithms/curate/llmlingua/multilingual.py +105 -0
- agentinputlab/algorithms/curate/llmlingua/span_map.py +167 -0
- agentinputlab/algorithms/curate/recomp/__init__.py +47 -0
- agentinputlab/algorithms/curate/recomp/external.py +1129 -0
- agentinputlab/algorithms/curate/recomp/selective.py +304 -0
- agentinputlab/algorithms/decompose/__init__.py +107 -0
- agentinputlab/algorithms/decompose/decomp/__init__.py +53 -0
- agentinputlab/algorithms/decompose/decomp/common_support.py +478 -0
- agentinputlab/algorithms/decompose/decomp/planner_component.py +199 -0
- agentinputlab/algorithms/decompose/decomp/worker_bridge.py +416 -0
- agentinputlab/algorithms/decompose/llm_compiler/__init__.py +35 -0
- agentinputlab/algorithms/decompose/llm_compiler/bridge.py +520 -0
- agentinputlab/algorithms/decompose/llm_compiler/common_support.py +504 -0
- agentinputlab/algorithms/decompose/plan_and_budget/__init__.py +43 -0
- agentinputlab/algorithms/decompose/plan_and_budget/cached_soft_budget.py +362 -0
- agentinputlab/algorithms/decompose/plan_and_budget/decomposer_component.py +107 -0
- agentinputlab/algorithms/decompose/plan_and_budget/full.py +205 -0
- agentinputlab/algorithms/decompose/plan_and_budget/hard_cap.py +246 -0
- agentinputlab/algorithms/decompose/plan_and_budget/regenerated_soft_budget.py +448 -0
- agentinputlab/algorithms/decompose/plan_and_solve/__init__.py +33 -0
- agentinputlab/algorithms/decompose/plan_and_solve/plan_component.py +192 -0
- agentinputlab/algorithms/decompose/plan_and_solve/ps_prompt.py +127 -0
- agentinputlab/algorithms/decompose/self_ask/__init__.py +27 -0
- agentinputlab/algorithms/decompose/self_ask/loop.py +362 -0
- agentinputlab/algorithms/decompose/self_ask/planner_component.py +117 -0
- agentinputlab/algorithms/reformulate/__init__.py +85 -0
- agentinputlab/algorithms/reformulate/infocqr/__init__.py +40 -0
- agentinputlab/algorithms/reformulate/infocqr/cache_replay.py +362 -0
- agentinputlab/algorithms/reformulate/infocqr/external.py +675 -0
- agentinputlab/algorithms/reformulate/infocqr/student.py +189 -0
- agentinputlab/algorithms/reformulate/query2doc/__init__.py +42 -0
- agentinputlab/algorithms/reformulate/query2doc/controlled_live.py +190 -0
- agentinputlab/algorithms/reformulate/query2doc/models.py +95 -0
- agentinputlab/algorithms/reformulate/query2doc/official_cached.py +335 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_bridge.py +223 -0
- agentinputlab/algorithms/reformulate/query2doc/querygym_runtime.py +326 -0
- agentinputlab/algorithms/reformulate/reformer/__init__.py +23 -0
- agentinputlab/algorithms/reformulate/reformer/external.py +417 -0
- agentinputlab/algorithms/reformulate/rrr.py +696 -0
- agentinputlab/annotation/__init__.py +5 -0
- agentinputlab/annotation/independent.py +428 -0
- agentinputlab/annotation/server.py +233 -0
- agentinputlab/annotation/static/app.js +121 -0
- agentinputlab/annotation/static/index.html +60 -0
- agentinputlab/annotation/static/review.css +2 -0
- agentinputlab/annotation/static/review.html +10 -0
- agentinputlab/annotation/static/review.js +11 -0
- agentinputlab/annotation/static/styles.css +55 -0
- agentinputlab/annotation/store.py +444 -0
- agentinputlab/api.py +368 -0
- agentinputlab/artifacts/__init__.py +15 -0
- agentinputlab/artifacts/json_file.py +104 -0
- agentinputlab/artifacts/manifest.py +98 -0
- agentinputlab/artifacts/store.py +157 -0
- agentinputlab/backends/__init__.py +98 -0
- agentinputlab/backends/embedding/__init__.py +25 -0
- agentinputlab/backends/embedding/base.py +145 -0
- agentinputlab/backends/embedding/callable.py +75 -0
- agentinputlab/backends/embedding/replay.py +129 -0
- agentinputlab/backends/llm/__init__.py +31 -0
- agentinputlab/backends/llm/base.py +149 -0
- agentinputlab/backends/llm/callable.py +108 -0
- agentinputlab/backends/llm/decomp_completion.py +88 -0
- agentinputlab/backends/llm/final_content_stop.py +72 -0
- agentinputlab/backends/llm/gpt_oss_final_channel.py +84 -0
- agentinputlab/backends/llm/openai_compatible.py +332 -0
- agentinputlab/backends/llm/transformers.py +171 -0
- agentinputlab/backends/retrieval/__init__.py +35 -0
- agentinputlab/backends/retrieval/base.py +50 -0
- agentinputlab/backends/retrieval/dense.py +777 -0
- agentinputlab/backends/retrieval/memory.py +119 -0
- agentinputlab/backends/retrieval/sqlite_fts.py +155 -0
- agentinputlab/backends/user/__init__.py +15 -0
- agentinputlab/backends/user/base.py +66 -0
- agentinputlab/backends/user/callable.py +54 -0
- agentinputlab/backends/user/replay.py +89 -0
- agentinputlab/backends/vector.py +38 -0
- agentinputlab/benchmarks/__init__.py +246 -0
- agentinputlab/benchmarks/acquisition.py +106 -0
- agentinputlab/benchmarks/ale.py +540 -0
- agentinputlab/benchmarks/appworld_ul.py +669 -0
- agentinputlab/benchmarks/assets.py +226 -0
- agentinputlab/benchmarks/base.py +68 -0
- agentinputlab/benchmarks/beir_archive.py +235 -0
- agentinputlab/benchmarks/beir_sample.py +269 -0
- agentinputlab/benchmarks/bfcl_native.py +245 -0
- agentinputlab/benchmarks/bfcl_v3.py +607 -0
- agentinputlab/benchmarks/cache.py +218 -0
- agentinputlab/benchmarks/clariq.py +203 -0
- agentinputlab/benchmarks/data/registry.json +321 -0
- agentinputlab/benchmarks/data/smoke_assets.json +511 -0
- agentinputlab/benchmarks/data/statebench_v0_8_1_task_ids.json +475 -0
- agentinputlab/benchmarks/data/toolsandbox_165848b_scenario_names.json +1 -0
- agentinputlab/benchmarks/execution.py +1111 -0
- agentinputlab/benchmarks/interactive.py +257 -0
- agentinputlab/benchmarks/ir_query.py +204 -0
- agentinputlab/benchmarks/longbench.py +203 -0
- agentinputlab/benchmarks/paired_replay.py +479 -0
- agentinputlab/benchmarks/preprocess.py +2313 -0
- agentinputlab/benchmarks/production.py +236 -0
- agentinputlab/benchmarks/qa.py +963 -0
- agentinputlab/benchmarks/registry.py +110 -0
- agentinputlab/benchmarks/retrieval.py +459 -0
- agentinputlab/benchmarks/runtime.py +509 -0
- agentinputlab/benchmarks/statebench.py +938 -0
- agentinputlab/benchmarks/tau3.py +725 -0
- agentinputlab/benchmarks/toolsandbox.py +626 -0
- agentinputlab/benchmarks/trec_dl_sample.py +263 -0
- agentinputlab/campaign.py +654 -0
- agentinputlab/campaign_runner.py +953 -0
- agentinputlab/cli/__init__.py +6 -0
- agentinputlab/cli/main.py +933 -0
- agentinputlab/connectors/__init__.py +46 -0
- agentinputlab/connectors/knowledge/__init__.py +4 -0
- agentinputlab/connectors/tools/__init__.py +30 -0
- agentinputlab/connectors/tools/base.py +57 -0
- agentinputlab/connectors/tools/mcp.py +222 -0
- agentinputlab/connectors/tools/memory.py +41 -0
- agentinputlab/connectors/tools/openapi.py +695 -0
- agentinputlab/connectors/tools/schema.py +102 -0
- agentinputlab/contracts/__init__.py +91 -0
- agentinputlab/contracts/availability.py +24 -0
- agentinputlab/contracts/dependency.py +58 -0
- agentinputlab/contracts/enums.py +82 -0
- agentinputlab/contracts/errors.py +63 -0
- agentinputlab/contracts/evidence.py +43 -0
- agentinputlab/contracts/intent.py +110 -0
- agentinputlab/contracts/lifecycle.py +164 -0
- agentinputlab/contracts/patch.py +167 -0
- agentinputlab/contracts/serialization.py +93 -0
- agentinputlab/contracts/state.py +127 -0
- agentinputlab/contracts/trace.py +59 -0
- agentinputlab/contracts/worker.py +42 -0
- agentinputlab/core/__init__.py +7 -0
- agentinputlab/data/__init__.py +32 -0
- agentinputlab/data/checksums.py +9 -0
- agentinputlab/data/downloader.py +235 -0
- agentinputlab/data/http.py +171 -0
- agentinputlab/data/index.py +21 -0
- agentinputlab/data/overlap.py +28 -0
- agentinputlab/data/preprocess.py +32 -0
- agentinputlab/data/storage.py +57 -0
- agentinputlab/data/trec_sample.py +427 -0
- agentinputlab/evaluation/__init__.py +5 -0
- agentinputlab/evaluation/outcomes.py +32 -0
- agentinputlab/experiments/__init__.py +278 -0
- agentinputlab/experiments/analysis.py +981 -0
- agentinputlab/experiments/artifacts.py +687 -0
- agentinputlab/experiments/automatic_measurement.py +678 -0
- agentinputlab/experiments/auxiliary_matrix.py +203 -0
- agentinputlab/experiments/bindings.py +430 -0
- agentinputlab/experiments/blueprints.py +147 -0
- agentinputlab/experiments/builders/__init__.py +45 -0
- agentinputlab/experiments/builders/executable_external_payload.py +423 -0
- agentinputlab/experiments/builders/external_process_arena.py +1018 -0
- agentinputlab/experiments/builders/registry_intervention.py +463 -0
- agentinputlab/experiments/cli.py +158 -0
- agentinputlab/experiments/controls.py +340 -0
- agentinputlab/experiments/driver_guard.py +205 -0
- agentinputlab/experiments/drivers/__init__.py +89 -0
- agentinputlab/experiments/drivers/external_arena.py +2908 -0
- agentinputlab/experiments/drivers/method_process.py +662 -0
- agentinputlab/experiments/drivers/parity_process.py +1484 -0
- agentinputlab/experiments/drivers/static_production.py +1574 -0
- agentinputlab/experiments/executor.py +448 -0
- agentinputlab/experiments/loader.py +605 -0
- agentinputlab/experiments/measurement.py +675 -0
- agentinputlab/experiments/paper_analysis.py +1818 -0
- agentinputlab/experiments/parity_identity.py +605 -0
- agentinputlab/experiments/plan.py +277 -0
- agentinputlab/experiments/production_builders.py +3739 -0
- agentinputlab/experiments/readiness.py +432 -0
- agentinputlab/experiments/schema.py +514 -0
- agentinputlab/experiments/secrets.py +177 -0
- agentinputlab/experiments/stateful_analysis.py +1738 -0
- agentinputlab/experiments/transfer_analysis.py +1316 -0
- agentinputlab/experiments/unseen_analysis.py +1025 -0
- agentinputlab/integrations/__init__.py +1 -0
- agentinputlab/integrations/action_space/__init__.py +1 -0
- agentinputlab/integrations/external_workers/__init__.py +76 -0
- agentinputlab/integrations/external_workers/ace.py +431 -0
- agentinputlab/integrations/external_workers/base.py +181 -0
- agentinputlab/integrations/external_workers/decomp_live.py +249 -0
- agentinputlab/integrations/external_workers/decomp_planner.py +30 -0
- agentinputlab/integrations/external_workers/infocqr_student.py +280 -0
- agentinputlab/integrations/external_workers/ircot.py +576 -0
- agentinputlab/integrations/external_workers/llmcompiler_live.py +360 -0
- agentinputlab/integrations/external_workers/longllmlingua.py +162 -0
- agentinputlab/integrations/external_workers/method.py +314 -0
- agentinputlab/integrations/external_workers/process.py +678 -0
- agentinputlab/integrations/external_workers/supervisor.py +1071 -0
- agentinputlab/integrations/full_architecture/__init__.py +1 -0
- agentinputlab/integrations/full_architecture/hyde.py +212 -0
- agentinputlab/integrations/full_architecture/plan_and_budget.py +188 -0
- agentinputlab/integrations/live_history.py +222 -0
- agentinputlab/integrations/prompt_optimization/__init__.py +1 -0
- agentinputlab/integrations/training/__init__.py +1 -0
- agentinputlab/metrics/__init__.py +50 -0
- agentinputlab/metrics/catalog.py +313 -0
- agentinputlab/metrics/harm.py +95 -0
- agentinputlab/metrics/paired.py +40 -0
- agentinputlab/metrics/qa.py +76 -0
- agentinputlab/metrics/text.py +33 -0
- agentinputlab/metrics/third_party.py +161 -0
- agentinputlab/models/__init__.py +4 -0
- agentinputlab/observability/__init__.py +56 -0
- agentinputlab/observability/budget.py +316 -0
- agentinputlab/observability/instrumentation.py +754 -0
- agentinputlab/observability/models.py +403 -0
- agentinputlab/observability/monitor.py +380 -0
- agentinputlab/optimizers/__init__.py +1 -0
- agentinputlab/optimizers/clarify/__init__.py +12 -0
- agentinputlab/optimizers/context/__init__.py +22 -0
- agentinputlab/optimizers/decompose/__init__.py +64 -0
- agentinputlab/optimizers/evidence/__init__.py +22 -0
- agentinputlab/optimizers/rewrite/__init__.py +30 -0
- agentinputlab/parity/__init__.py +5 -0
- agentinputlab/parity/base.py +206 -0
- agentinputlab/production_campaign.py +763 -0
- agentinputlab/py.typed +1 -0
- agentinputlab/recipes/__init__.py +333 -0
- agentinputlab/recipes/ace.py +418 -0
- agentinputlab/recipes/ace_bfcl.py +273 -0
- agentinputlab/recipes/ace_common_support.py +169 -0
- agentinputlab/recipes/ace_native_plan.py +322 -0
- agentinputlab/recipes/ace_online.py +711 -0
- agentinputlab/recipes/ace_retrieval.py +330 -0
- agentinputlab/recipes/ace_statebench.py +382 -0
- agentinputlab/recipes/ace_tau.py +463 -0
- agentinputlab/recipes/ace_toolsandbox.py +432 -0
- agentinputlab/recipes/acon_guideline.py +22 -0
- agentinputlab/recipes/adaptive_rag.py +822 -0
- agentinputlab/recipes/llmlingua2_inference.py +146 -0
- agentinputlab/recipes/llmlingua2_training.py +796 -0
- agentinputlab/recipes/mce.py +1486 -0
- agentinputlab/recipes/mce_final.py +335 -0
- agentinputlab/recipes/mce_text_fit.py +499 -0
- agentinputlab/recipes/mce_text_only.py +298 -0
- agentinputlab/recipes/qrag.py +729 -0
- agentinputlab/recipes/qrag_inference.py +364 -0
- agentinputlab/recipes/r2ag_features.py +236 -0
- agentinputlab/recipes/r2ag_inference.py +503 -0
- agentinputlab/recipes/r2ag_training.py +948 -0
- agentinputlab/recipes/recomp_abstractive.py +592 -0
- agentinputlab/recipes/recomp_extractive.py +525 -0
- agentinputlab/recipes/reformer.py +17 -0
- agentinputlab/recipes/rqrag.py +1516 -0
- agentinputlab/recipes/rqrag_common_support.py +193 -0
- agentinputlab/recipes/rqrag_completion.py +147 -0
- agentinputlab/recipes/skillopt.py +462 -0
- agentinputlab/recipes/skillopt_common_support.py +154 -0
- agentinputlab/recipes/skillopt_environment.py +165 -0
- agentinputlab/recipes/skillopt_environment_cli.py +5 -0
- agentinputlab/recipes/skillopt_retrieval.py +188 -0
- agentinputlab/recipes/smartsearch.py +897 -0
- agentinputlab/recipes/smartsearch_curriculum.py +283 -0
- agentinputlab/recipes/smartsearch_grpo_contracts.py +58 -0
- agentinputlab/recipes/smartsearch_grpo_data.py +314 -0
- agentinputlab/recipes/smartsearch_grpo_meter.py +324 -0
- agentinputlab/recipes/smartsearch_grpo_prefix.py +144 -0
- agentinputlab/recipes/smartsearch_grpo_rollout.py +119 -0
- agentinputlab/recipes/smartsearch_grpo_tools.py +364 -0
- agentinputlab/recipes/smartsearch_grpo_training.py +305 -0
- agentinputlab/recipes/smartsearch_grpo_transport.py +248 -0
- agentinputlab/recipes/smartsearch_grpo_worker.py +364 -0
- agentinputlab/recipes/smartsearch_inference.py +524 -0
- agentinputlab/recipes/smartsearch_native_data.py +172 -0
- agentinputlab/recipes/smartsearch_policy.py +616 -0
- agentinputlab/recipes/smartsearch_policy_worker.py +9 -0
- agentinputlab/recipes/smartsearch_sampling.py +232 -0
- agentinputlab/recipes/smartsearch_teacher.py +203 -0
- agentinputlab/recipes/smartsearch_training.py +265 -0
- agentinputlab/registry/__init__.py +46 -0
- agentinputlab/registry/beta.py +727 -0
- agentinputlab/registry/data/algorithms.json +1959 -0
- agentinputlab/registry/data/auxiliary_method_profiles.json +34 -0
- agentinputlab/registry/loader.py +100 -0
- agentinputlab/registry/result_manifest.py +184 -0
- agentinputlab/registry/runtime.py +195 -0
- agentinputlab/registry/schema.py +449 -0
- agentinputlab/runner/__init__.py +30 -0
- agentinputlab/runner/balance.py +248 -0
- agentinputlab/runner/cache.py +135 -0
- agentinputlab/runner/composite.py +130 -0
- agentinputlab/runner/engine.py +285 -0
- agentinputlab/runner/runtime_guard.py +268 -0
- agentinputlab/runner/worker.py +80 -0
- agentinputlab/runtime_bootstrap.py +288 -0
- agentinputlab/selectors/__init__.py +196 -0
- agentinputlab/selectors/advanced.py +1514 -0
- agentinputlab/selectors/artifacts.py +1087 -0
- agentinputlab/selectors/base.py +112 -0
- agentinputlab/selectors/campaigns.py +1893 -0
- agentinputlab/selectors/catalog.py +86 -0
- agentinputlab/selectors/discovery.py +1860 -0
- agentinputlab/selectors/evaluation.py +1348 -0
- agentinputlab/selectors/features.py +383 -0
- agentinputlab/selectors/learned.py +502 -0
- agentinputlab/selectors/no_op.py +28 -0
- agentinputlab/selectors/outcome_table.py +475 -0
- agentinputlab/selectors/producer.py +853 -0
- agentinputlab/selectors/rules.py +109 -0
- agentinputlab/selectors/runtime.py +420 -0
- agentinputlab/studies/__init__.py +16 -0
- agentinputlab/studies/design.py +165 -0
- agentinputlab/studies/paired.py +322 -0
- agentinputlab/studio/__init__.py +6 -0
- agentinputlab/studio/campaigns.py +819 -0
- agentinputlab/studio/engine.py +1408 -0
- agentinputlab/studio/model_backends.py +425 -0
- agentinputlab/studio/prepared.py +410 -0
- agentinputlab/studio/server.py +613 -0
- agentinputlab/studio/static/app.js +397 -0
- agentinputlab/studio/static/campaigns.html +175 -0
- agentinputlab/studio/static/campaigns.js +426 -0
- agentinputlab/studio/static/index.html +192 -0
- agentinputlab/studio/static/result.html +103 -0
- agentinputlab/studio/static/result.js +273 -0
- agentinputlab/studio/static/styles.css +467 -0
- agentinputlab-0.1.0a1.data/data/share/agentinputlab/CITATION.cff +22 -0
- agentinputlab-0.1.0a1.dist-info/METADATA +392 -0
- agentinputlab-0.1.0a1.dist-info/RECORD +469 -0
- agentinputlab-0.1.0a1.dist-info/WHEEL +5 -0
- agentinputlab-0.1.0a1.dist-info/entry_points.txt +6 -0
- agentinputlab-0.1.0a1.dist-info/licenses/LICENSE +21 -0
- agentinputlab-0.1.0a1.dist-info/licenses/NOTICE +31 -0
- agentinputlab-0.1.0a1.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
- agentinputlab-0.1.0a1.dist-info/top_level.txt +2 -0
- workers/README.md +94 -0
- workers/ace/README.md +261 -0
- workers/ace/__init__.py +1 -0
- workers/ace/compatibility_manifest.json +39 -0
- workers/ace/driver.py +1682 -0
- workers/acon/README.md +215 -0
- workers/acon/__init__.py +5 -0
- workers/acon/compatibility_manifest.json +93 -0
- workers/acon/driver.py +1859 -0
- workers/acon/fitted.py +395 -0
- workers/acon/guideline.py +470 -0
- workers/acon/guideline_source_manifest.json +20 -0
- workers/adaptive_rag/README.md +96 -0
- workers/adaptive_rag/__init__.py +15 -0
- workers/adaptive_rag/compatibility_manifest.json +61 -0
- workers/adaptive_rag/driver.py +1470 -0
- workers/adaptive_rag/fixture_runtime.py +163 -0
- workers/adaptive_rag/runtime.py +252 -0
- workers/ask_or_assume/README.md +124 -0
- workers/ask_or_assume/__init__.py +1 -0
- workers/ask_or_assume/compatibility_manifest.json +36 -0
- workers/ask_or_assume/driver.py +1227 -0
- workers/decomp/README.md +133 -0
- workers/decomp/UPSTREAM_LICENSE.Apache-2.0.txt +201 -0
- workers/decomp/__init__.py +5 -0
- workers/decomp/compatibility_manifest.json +205 -0
- workers/decomp/driver.py +1015 -0
- workers/decomp/live.py +137 -0
- workers/decomp/runtime.py +231 -0
- workers/infocqr/README.md +106 -0
- workers/infocqr/__init__.py +1 -0
- workers/infocqr/compatibility_manifest.json +84 -0
- workers/infocqr/driver.py +1766 -0
- workers/ircot/README.md +167 -0
- workers/ircot/__init__.py +7 -0
- workers/ircot/compatibility_manifest.json +121 -0
- workers/ircot/controlled.py +457 -0
- workers/ircot/driver.py +1373 -0
- workers/ircot/fixture_bindings.json +39 -0
- workers/ircot/fixture_config.json +48 -0
- workers/ircot/fixture_prompt.txt +2 -0
- workers/ircot/native_qa.py +398 -0
- workers/ircot/qa_compatibility_manifest.json +57 -0
- workers/ircot/runtime.py +97 -0
- workers/llmcompiler/README.md +160 -0
- workers/llmcompiler/__init__.py +1 -0
- workers/llmcompiler/compatibility_manifest.json +45 -0
- workers/llmcompiler/driver.py +1514 -0
- workers/llmcompiler/export_prompts.py +74 -0
- workers/llmcompiler/live.py +174 -0
- workers/llmcompiler/patch_manifest.json +147 -0
- workers/mce/README.md +198 -0
- workers/mce/__init__.py +29 -0
- workers/mce/compatibility_manifest.json +115 -0
- workers/mce/driver.py +2585 -0
- workers/mce/final_eval.py +173 -0
- workers/official_arenas/README.md +25 -0
- workers/official_arenas/__init__.py +1 -0
- workers/official_arenas/capture_records.py +110 -0
- workers/official_arenas/driver.py +619 -0
- workers/official_arenas/protocol.py +334 -0
- workers/official_arenas/providers/__init__.py +1 -0
- workers/official_arenas/providers/_live_history.py +228 -0
- workers/official_arenas/providers/_message_compat.py +77 -0
- workers/official_arenas/providers/appworld_ul.py +196 -0
- workers/official_arenas/providers/fixture.py +112 -0
- workers/official_arenas/providers/statebench.py +960 -0
- workers/official_arenas/providers/tau3.py +1672 -0
- workers/official_arenas/providers/tau_retail.py +586 -0
- workers/official_arenas/providers/toolsandbox.py +1321 -0
- workers/official_arenas/providers/tooltalk.py +699 -0
- workers/official_arenas/retail_capture.py +188 -0
- workers/official_arenas/retail_driver.py +275 -0
- workers/qrag/README.md +172 -0
- workers/qrag/__init__.py +1 -0
- workers/qrag/compatibility_manifest.json +64 -0
- workers/qrag/driver.py +673 -0
- workers/r2ag/README.md +154 -0
- workers/r2ag/__init__.py +5 -0
- workers/r2ag/compatibility_manifest.json +55 -0
- workers/r2ag/driver.py +1287 -0
- workers/r2ag/feature_schema.json +8 -0
- workers/r2ag/runtime.py +589 -0
- workers/recomp/README.md +120 -0
- workers/recomp/__init__.py +1 -0
- workers/recomp/compatibility_manifest.json +43 -0
- workers/recomp/driver.py +1653 -0
- workers/reformer/README.md +125 -0
- workers/reformer/__init__.py +1 -0
- workers/reformer/compatibility_manifest.json +68 -0
- workers/reformer/driver.py +1459 -0
- workers/reformer/induction.py +457 -0
- workers/rrr/README.md +126 -0
- workers/rrr/__init__.py +1 -0
- workers/rrr/compatibility_manifest.json +57 -0
- workers/rrr/driver.py +1133 -0
- workers/smartsearch/README.md +331 -0
- workers/smartsearch/__init__.py +1 -0
- workers/smartsearch/compatibility_manifest.json +41 -0
- workers/smartsearch/driver.py +1643 -0
workers/r2ag/driver.py
ADDED
|
@@ -0,0 +1,1287 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Fail-closed R²AG R²-Former/projection/reader sidecar.
|
|
3
|
+
|
|
4
|
+
The worker never downloads or redistributes source or weights. Production uses
|
|
5
|
+
the bundled runtime with read-only user mounts; the injectable runtime seam is
|
|
6
|
+
available only when ``fixture_mode=True`` so tests cannot self-attest as
|
|
7
|
+
production execution.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import argparse
|
|
13
|
+
import hashlib
|
|
14
|
+
import json
|
|
15
|
+
import math
|
|
16
|
+
import re
|
|
17
|
+
import subprocess
|
|
18
|
+
import sys
|
|
19
|
+
from collections.abc import Callable, Mapping, Sequence
|
|
20
|
+
from contextlib import redirect_stdout
|
|
21
|
+
from dataclasses import dataclass
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from typing import Any, TextIO
|
|
24
|
+
|
|
25
|
+
UPSTREAM_COMMIT = "61cfd1027c0101172fda701ec816bc18360b5bab"
|
|
26
|
+
UPSTREAM_TREE = "f1757a51fc1c3d64fe9d1b8788178293c8d5ef3b"
|
|
27
|
+
PROTOCOL = "agentinputlab.r2ag.v4"
|
|
28
|
+
RELEASE = "acquire.r2ag.controlled_same_candidates"
|
|
29
|
+
DEFAULT_MANIFEST = Path(__file__).with_name("compatibility_manifest.json")
|
|
30
|
+
DEFAULT_FEATURE_SCHEMA = Path(__file__).with_name("feature_schema.json")
|
|
31
|
+
FEATURE_NAMES = ("rerank", "neighbor", "precedent")
|
|
32
|
+
RUNTIME_DEPENDENCIES = {
|
|
33
|
+
"torch": "2.1.2",
|
|
34
|
+
"transformers": "4.40.2",
|
|
35
|
+
"sentence-transformers": "2.6.1",
|
|
36
|
+
}
|
|
37
|
+
_HEX64 = re.compile(r"^[0-9a-f]{64}$")
|
|
38
|
+
_IMMUTABLE_REVISION_ALIASES = frozenset({"default", "head", "latest", "main", "master"})
|
|
39
|
+
_ARTIFACT_MANIFEST_SCHEMA = "agentinputlab.artifact-closure.v1"
|
|
40
|
+
_ARTIFACT_MANIFEST_FIELDS = frozenset({"schema", "repository", "revision", "files"})
|
|
41
|
+
_ARTIFACT_FILE_FIELDS = frozenset({"size", "sha256"})
|
|
42
|
+
_FORBIDDEN_FINGERPRINT_TERMS = frozenset(
|
|
43
|
+
{
|
|
44
|
+
"answer",
|
|
45
|
+
"api",
|
|
46
|
+
"credential",
|
|
47
|
+
"expected",
|
|
48
|
+
"gold",
|
|
49
|
+
"groundtruth",
|
|
50
|
+
"label",
|
|
51
|
+
"oracle",
|
|
52
|
+
"password",
|
|
53
|
+
"reference",
|
|
54
|
+
"reward",
|
|
55
|
+
"secret",
|
|
56
|
+
"solution",
|
|
57
|
+
"supporting",
|
|
58
|
+
"target",
|
|
59
|
+
}
|
|
60
|
+
)
|
|
61
|
+
_REPOSITORY_IDENTITY_REQUIRED = frozenset({"repository", "revision"})
|
|
62
|
+
_REPOSITORY_IDENTITY_OPTIONAL = frozenset({"manifest_sha256"})
|
|
63
|
+
_INDEX_IDENTITY_REQUIRED = frozenset({"algorithm", "revision"})
|
|
64
|
+
_INDEX_IDENTITY_OPTIONAL = frozenset({"manifest_sha256"})
|
|
65
|
+
_FEATURE_ARTIFACT_FIELDS = frozenset({"repository", "revision", "sha256"})
|
|
66
|
+
_FEATURE_EMBEDDER_FIELDS = frozenset({"kind", "fingerprint_sha256"})
|
|
67
|
+
_ARTIFACT_FIELDS = frozenset(
|
|
68
|
+
{
|
|
69
|
+
"checkpoint_repository",
|
|
70
|
+
"checkpoint_revision",
|
|
71
|
+
"checkpoint_config_sha256",
|
|
72
|
+
"rformer_weights_sha256",
|
|
73
|
+
"projection_weights_sha256",
|
|
74
|
+
"checkpoint_num_documents",
|
|
75
|
+
"base_model_repository",
|
|
76
|
+
"base_model_revision",
|
|
77
|
+
"base_model_manifest_sha256",
|
|
78
|
+
"tokenizer_repository",
|
|
79
|
+
"tokenizer_revision",
|
|
80
|
+
"tokenizer_manifest_sha256",
|
|
81
|
+
"feature_schema_revision",
|
|
82
|
+
"feature_schema_sha256",
|
|
83
|
+
"source_manifest_sha256",
|
|
84
|
+
"compatibility_manifest_sha256",
|
|
85
|
+
"upstream_commit",
|
|
86
|
+
}
|
|
87
|
+
)
|
|
88
|
+
_REQUEST_ID_FIELDS = (
|
|
89
|
+
"method_id",
|
|
90
|
+
"release_id",
|
|
91
|
+
"upstream_commit",
|
|
92
|
+
"state_hash",
|
|
93
|
+
"payload",
|
|
94
|
+
"dependency_fingerprints",
|
|
95
|
+
"minimum_reservation",
|
|
96
|
+
"protocol_version",
|
|
97
|
+
)
|
|
98
|
+
_REQUEST_FIELDS = frozenset((*_REQUEST_ID_FIELDS, "request_id"))
|
|
99
|
+
_CANDIDATE_FIELDS = frozenset(
|
|
100
|
+
{"document_id", "text", "title", "source", "score", "corpus_revision"}
|
|
101
|
+
)
|
|
102
|
+
_POOL_FIELDS = frozenset({"retriever", "corpus", "index", "top_k", "order", "content_sha256"})
|
|
103
|
+
_PAYLOAD_FIELDS = frozenset(
|
|
104
|
+
{
|
|
105
|
+
"task_id",
|
|
106
|
+
"query",
|
|
107
|
+
"candidate_documents",
|
|
108
|
+
"candidate_document_ids",
|
|
109
|
+
"candidate_documents_sha256",
|
|
110
|
+
"candidate_order_sha256",
|
|
111
|
+
"candidate_pool_fingerprint",
|
|
112
|
+
"candidate_pool_fingerprint_sha256",
|
|
113
|
+
"artifacts",
|
|
114
|
+
"artifact_identity_sha256",
|
|
115
|
+
"features",
|
|
116
|
+
"feature_tensor_sha256",
|
|
117
|
+
"feature_source",
|
|
118
|
+
"feature_names",
|
|
119
|
+
"feature_embedder_fingerprint",
|
|
120
|
+
"feature_embedder_fingerprint_sha256",
|
|
121
|
+
"feature_artifact_fingerprint",
|
|
122
|
+
"feature_artifact_fingerprint_sha256",
|
|
123
|
+
"reader_instruction",
|
|
124
|
+
"reader_instruction_sha256",
|
|
125
|
+
"reader_input",
|
|
126
|
+
"reader_input_sha256",
|
|
127
|
+
"controls",
|
|
128
|
+
"remaining_budget",
|
|
129
|
+
"seed",
|
|
130
|
+
"execution_contract",
|
|
131
|
+
}
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
class DriverError(RuntimeError):
|
|
136
|
+
"""Stable, secret-free worker failure."""
|
|
137
|
+
|
|
138
|
+
def __init__(
|
|
139
|
+
self,
|
|
140
|
+
code: str,
|
|
141
|
+
message: str,
|
|
142
|
+
*,
|
|
143
|
+
consumed: bool = False,
|
|
144
|
+
usage: Mapping[str, Any] | None = None,
|
|
145
|
+
) -> None:
|
|
146
|
+
super().__init__(message)
|
|
147
|
+
self.code = code
|
|
148
|
+
self.public_message = message
|
|
149
|
+
self.consumed = consumed
|
|
150
|
+
self.usage = dict(usage or {})
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
@dataclass(frozen=True, slots=True)
|
|
154
|
+
class PreparedReader:
|
|
155
|
+
value: Any
|
|
156
|
+
marker_positions: tuple[int, ...]
|
|
157
|
+
input_tokens: int
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
@dataclass(frozen=True, slots=True)
|
|
161
|
+
class TensorStage:
|
|
162
|
+
value: Any
|
|
163
|
+
sha256: str
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
@dataclass(frozen=True, slots=True)
|
|
167
|
+
class ConditionedReader:
|
|
168
|
+
value: Any
|
|
169
|
+
marker_positions: tuple[int, ...]
|
|
170
|
+
only_marker_positions_modified: bool
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
@dataclass(frozen=True, slots=True)
|
|
174
|
+
class GenerationResult:
|
|
175
|
+
answer: str
|
|
176
|
+
output_tokens: int
|
|
177
|
+
accelerator_seconds: float = 0.0
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
class R2AGDriver:
|
|
181
|
+
"""Validate identity and budgets, then execute the five ordered stages."""
|
|
182
|
+
|
|
183
|
+
worker_id = "agentinputlab.r2ag.pinned.v4"
|
|
184
|
+
|
|
185
|
+
def __init__(
|
|
186
|
+
self,
|
|
187
|
+
*,
|
|
188
|
+
source_root: Path,
|
|
189
|
+
checkpoint_root: Path,
|
|
190
|
+
base_model_root: Path,
|
|
191
|
+
tokenizer_root: Path,
|
|
192
|
+
manifest_path: Path = DEFAULT_MANIFEST,
|
|
193
|
+
feature_schema_path: Path = DEFAULT_FEATURE_SCHEMA,
|
|
194
|
+
runtime_factory: Callable[[Path, Path, Path, Path], Any] | None = None,
|
|
195
|
+
fixture_mode: bool = False,
|
|
196
|
+
) -> None:
|
|
197
|
+
if runtime_factory is not None and not fixture_mode:
|
|
198
|
+
raise ValueError("custom R²AG runtimes are fixture-only")
|
|
199
|
+
for label, path in (
|
|
200
|
+
("source root", source_root),
|
|
201
|
+
("checkpoint root", checkpoint_root),
|
|
202
|
+
("base-model root", base_model_root),
|
|
203
|
+
("tokenizer root", tokenizer_root),
|
|
204
|
+
("compatibility manifest", manifest_path),
|
|
205
|
+
("feature schema", feature_schema_path),
|
|
206
|
+
):
|
|
207
|
+
if path.is_symlink():
|
|
208
|
+
raise DriverError("PATH_REJECTED", f"R²AG {label} must not be a symlink")
|
|
209
|
+
self.source_root = source_root.resolve()
|
|
210
|
+
self.checkpoint_root = checkpoint_root.resolve()
|
|
211
|
+
self.base_model_root = base_model_root.resolve()
|
|
212
|
+
self.tokenizer_root = tokenizer_root.resolve()
|
|
213
|
+
self.manifest_path = manifest_path.resolve()
|
|
214
|
+
self.feature_schema_path = feature_schema_path.resolve()
|
|
215
|
+
self.runtime_factory = runtime_factory
|
|
216
|
+
self.fixture_mode = fixture_mode
|
|
217
|
+
self.manifest = _load_json(self.manifest_path, label="compatibility manifest")
|
|
218
|
+
self.compatibility_manifest_sha256 = _content_hash(self.manifest)
|
|
219
|
+
runtime_manifest = _mapping(self.manifest, "runtime")
|
|
220
|
+
dependencies = _mapping(runtime_manifest, "dependencies")
|
|
221
|
+
if dict(dependencies) != RUNTIME_DEPENDENCIES:
|
|
222
|
+
raise DriverError(
|
|
223
|
+
"MANIFEST_INVALID",
|
|
224
|
+
"R²AG runtime dependency versions drifted",
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
def fingerprint(self) -> Mapping[str, Any]:
|
|
228
|
+
return {
|
|
229
|
+
"worker_id": self.worker_id,
|
|
230
|
+
"upstream_commit": UPSTREAM_COMMIT,
|
|
231
|
+
"protocol": PROTOCOL,
|
|
232
|
+
"compatibility_manifest_sha256": self.compatibility_manifest_sha256,
|
|
233
|
+
"runtime_dependencies": dict(RUNTIME_DEPENDENCIES),
|
|
234
|
+
"fixture_mode": self.fixture_mode,
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
def call(self, request: Mapping[str, Any]) -> Mapping[str, Any]:
|
|
238
|
+
"""Implement the standard in-process ``WorkerClient`` boundary."""
|
|
239
|
+
|
|
240
|
+
return self.handle(request)
|
|
241
|
+
|
|
242
|
+
def call_controlled(self, request: Any) -> Mapping[str, Any]:
|
|
243
|
+
"""Accept a typed host request without weakening envelope validation."""
|
|
244
|
+
|
|
245
|
+
serializer = getattr(request, "as_dict", None)
|
|
246
|
+
if not callable(serializer):
|
|
247
|
+
return self.handle({})
|
|
248
|
+
value = serializer()
|
|
249
|
+
if not isinstance(value, Mapping):
|
|
250
|
+
return self.handle({})
|
|
251
|
+
return self.handle(value)
|
|
252
|
+
|
|
253
|
+
def handle(self, request: Mapping[str, Any]) -> Mapping[str, Any]:
|
|
254
|
+
try:
|
|
255
|
+
return self._handle(request)
|
|
256
|
+
except DriverError as exc:
|
|
257
|
+
response: dict[str, Any] = {
|
|
258
|
+
"status": "error",
|
|
259
|
+
"failure_code": exc.code,
|
|
260
|
+
"message": exc.public_message,
|
|
261
|
+
"consumed": exc.consumed,
|
|
262
|
+
}
|
|
263
|
+
if exc.usage:
|
|
264
|
+
response["usage"] = exc.usage
|
|
265
|
+
return response
|
|
266
|
+
|
|
267
|
+
def verify_source_only(self) -> Mapping[str, Any]:
|
|
268
|
+
return self._verify_source()
|
|
269
|
+
|
|
270
|
+
def _handle(self, request: Mapping[str, Any]) -> Mapping[str, Any]:
|
|
271
|
+
payload = _validate_envelope(request)
|
|
272
|
+
if set(payload) != _PAYLOAD_FIELDS:
|
|
273
|
+
raise DriverError("REQUEST_INVALID", "R²AG payload fields drifted")
|
|
274
|
+
dependencies = _mapping(request, "dependency_fingerprints")
|
|
275
|
+
contract = _mapping(payload, "execution_contract")
|
|
276
|
+
_validate_contract(contract)
|
|
277
|
+
controls = _mapping(payload, "controls")
|
|
278
|
+
remaining = _mapping(payload, "remaining_budget")
|
|
279
|
+
artifacts = _mapping(payload, "artifacts")
|
|
280
|
+
_validate_artifact_identity(artifacts)
|
|
281
|
+
if payload.get("artifact_identity_sha256") != _content_hash(artifacts):
|
|
282
|
+
raise DriverError("IDENTITY_MISMATCH", "artifact identity hash mismatch")
|
|
283
|
+
if artifacts.get("upstream_commit") != UPSTREAM_COMMIT:
|
|
284
|
+
raise DriverError("IDENTITY_MISMATCH", "artifact source commit mismatch")
|
|
285
|
+
if artifacts.get("compatibility_manifest_sha256") != (self.compatibility_manifest_sha256):
|
|
286
|
+
raise DriverError("IDENTITY_MISMATCH", "compatibility manifest mismatch")
|
|
287
|
+
|
|
288
|
+
candidates = _candidate_documents(payload.get("candidate_documents"))
|
|
289
|
+
candidate_ids = _string_sequence(
|
|
290
|
+
payload.get("candidate_document_ids"), label="candidate_document_ids"
|
|
291
|
+
)
|
|
292
|
+
if candidate_ids != tuple(row["document_id"] for row in candidates):
|
|
293
|
+
raise DriverError("IDENTITY_MISMATCH", "candidate ID/order mismatch")
|
|
294
|
+
if len(candidate_ids) != len(set(candidate_ids)):
|
|
295
|
+
raise DriverError("REQUEST_INVALID", "candidate IDs must be unique")
|
|
296
|
+
if payload.get("candidate_documents_sha256") != _content_hash(candidates):
|
|
297
|
+
raise DriverError("IDENTITY_MISMATCH", "candidate document hash mismatch")
|
|
298
|
+
if payload.get("candidate_order_sha256") != _content_hash(candidate_ids):
|
|
299
|
+
raise DriverError("IDENTITY_MISMATCH", "candidate order hash mismatch")
|
|
300
|
+
pool = _validate_pool(payload, candidates)
|
|
301
|
+
feature_source = payload.get("feature_source")
|
|
302
|
+
feature_embedder_identity, feature_artifact_identity = _validate_feature_identities(
|
|
303
|
+
payload,
|
|
304
|
+
feature_source=feature_source,
|
|
305
|
+
)
|
|
306
|
+
_validate_dependency_fingerprints(
|
|
307
|
+
dependencies,
|
|
308
|
+
pool=pool,
|
|
309
|
+
artifacts=artifacts,
|
|
310
|
+
feature_source=feature_source,
|
|
311
|
+
feature_embedder_identity=feature_embedder_identity,
|
|
312
|
+
feature_artifact_identity=feature_artifact_identity,
|
|
313
|
+
)
|
|
314
|
+
|
|
315
|
+
num_k = _positive_int(artifacts, "checkpoint_num_documents")
|
|
316
|
+
if _positive_int(controls, "checkpoint_num_documents") != num_k:
|
|
317
|
+
raise DriverError("PROFILE_DRIFT", "controls/checkpoint num_k mismatch")
|
|
318
|
+
if len(candidates) != num_k:
|
|
319
|
+
raise DriverError("PROFILE_DRIFT", "candidate count must equal config.num_k")
|
|
320
|
+
if num_k > 30 or num_k > _positive_int(controls, "max_documents"):
|
|
321
|
+
raise DriverError("INPUT_TOO_LARGE", "R²AG num_k exceeds its controlled limit")
|
|
322
|
+
if tuple(payload.get("feature_names", ())) != FEATURE_NAMES:
|
|
323
|
+
raise DriverError("PROFILE_DRIFT", "retrieval feature schema drifted")
|
|
324
|
+
if controls.get("retrieval_token") != "<R>" or controls.get("unknown_token") != "<unk>":
|
|
325
|
+
raise DriverError("PROFILE_DRIFT", "R²AG marker controls drifted")
|
|
326
|
+
|
|
327
|
+
if _nonnegative_int(remaining, "model_calls") < 1:
|
|
328
|
+
raise DriverError("MODEL_BUDGET_EXHAUSTED", "reader model budget exhausted")
|
|
329
|
+
output_remaining = _nonnegative_int(remaining, "output_tokens")
|
|
330
|
+
if output_remaining < 1:
|
|
331
|
+
raise DriverError("OUTPUT_BUDGET_EXHAUSTED", "reader output budget exhausted")
|
|
332
|
+
max_new_tokens = min(
|
|
333
|
+
output_remaining,
|
|
334
|
+
_positive_int(controls, "max_output_tokens"),
|
|
335
|
+
)
|
|
336
|
+
|
|
337
|
+
source_attestation = self._verify_source()
|
|
338
|
+
self._verify_static_artifacts(artifacts)
|
|
339
|
+
if feature_source == "worker":
|
|
340
|
+
self._verify_feature_embedder(pool)
|
|
341
|
+
try:
|
|
342
|
+
runtime = self._runtime()
|
|
343
|
+
verified = runtime.verify_artifacts(artifacts)
|
|
344
|
+
except DriverError:
|
|
345
|
+
raise
|
|
346
|
+
except Exception as exc:
|
|
347
|
+
raise DriverError(
|
|
348
|
+
"CHECKPOINT_INVALID",
|
|
349
|
+
"R²AG runtime artifact verification failed",
|
|
350
|
+
) from exc
|
|
351
|
+
if not isinstance(verified, Mapping):
|
|
352
|
+
raise DriverError("CHECKPOINT_INVALID", "runtime artifact verification failed")
|
|
353
|
+
expected_verified = {
|
|
354
|
+
"checkpoint_num_documents": num_k,
|
|
355
|
+
"checkpoint_config_sha256": artifacts["checkpoint_config_sha256"],
|
|
356
|
+
"rformer_weights_sha256": artifacts["rformer_weights_sha256"],
|
|
357
|
+
"projection_weights_sha256": artifacts["projection_weights_sha256"],
|
|
358
|
+
}
|
|
359
|
+
if any(verified.get(key) != value for key, value in expected_verified.items()):
|
|
360
|
+
raise DriverError("CHECKPOINT_MISMATCH", "runtime component identity mismatch")
|
|
361
|
+
|
|
362
|
+
embedding_calls = 0
|
|
363
|
+
if feature_source == "worker":
|
|
364
|
+
try:
|
|
365
|
+
raw_features = runtime.compute_features(
|
|
366
|
+
_text(payload, "query"),
|
|
367
|
+
candidates,
|
|
368
|
+
_mapping(pool, "retriever"),
|
|
369
|
+
)
|
|
370
|
+
except Exception as exc:
|
|
371
|
+
raise DriverError(
|
|
372
|
+
"FEATURE_EXTRACTION_FAILED",
|
|
373
|
+
"R²AG feature extraction failed",
|
|
374
|
+
consumed=True,
|
|
375
|
+
) from exc
|
|
376
|
+
embedding_calls = 1
|
|
377
|
+
elif feature_source in {"embedder", "candidate_metadata"}:
|
|
378
|
+
raw_features = payload.get("features")
|
|
379
|
+
else:
|
|
380
|
+
raise DriverError("PROFILE_DRIFT", "unsupported R²AG feature source")
|
|
381
|
+
features = _features(raw_features, expected_count=num_k)
|
|
382
|
+
feature_hash = _content_hash(
|
|
383
|
+
tuple(tuple(row[name] for name in FEATURE_NAMES) for row in features)
|
|
384
|
+
)
|
|
385
|
+
sent_feature_hash = payload.get("feature_tensor_sha256")
|
|
386
|
+
if feature_source != "worker" and sent_feature_hash != feature_hash:
|
|
387
|
+
raise DriverError("IDENTITY_MISMATCH", "feature tensor hash mismatch")
|
|
388
|
+
_validate_descending(features)
|
|
389
|
+
|
|
390
|
+
reader_input = _text(payload, "reader_input")
|
|
391
|
+
if payload.get("reader_input_sha256") != _content_hash(reader_input):
|
|
392
|
+
raise DriverError("IDENTITY_MISMATCH", "reader input hash mismatch")
|
|
393
|
+
reader_instruction = _text(payload, "reader_instruction")
|
|
394
|
+
if payload.get("reader_instruction_sha256") != _content_hash(reader_instruction):
|
|
395
|
+
raise DriverError("IDENTITY_MISMATCH", "reader instruction hash mismatch")
|
|
396
|
+
|
|
397
|
+
try:
|
|
398
|
+
prepared: PreparedReader = runtime.prepare_reader(reader_input, num_k)
|
|
399
|
+
except Exception as exc:
|
|
400
|
+
raise DriverError("TOKENIZATION_FAILED", "R²AG reader tokenization failed") from exc
|
|
401
|
+
_validate_prepared(prepared, num_k)
|
|
402
|
+
context_limit = remaining.get("context_tokens")
|
|
403
|
+
if context_limit is not None and prepared.input_tokens > _nonnegative_int(
|
|
404
|
+
remaining, "context_tokens"
|
|
405
|
+
):
|
|
406
|
+
raise DriverError("CONTEXT_BUDGET_EXHAUSTED", "reader context budget exhausted")
|
|
407
|
+
|
|
408
|
+
try:
|
|
409
|
+
rformer: TensorStage = runtime.run_rformer(features)
|
|
410
|
+
_validate_tensor_stage(rformer, label="R²-Former")
|
|
411
|
+
projection: TensorStage = runtime.project(rformer.value)
|
|
412
|
+
_validate_tensor_stage(projection, label="projection")
|
|
413
|
+
conditioned: ConditionedReader = runtime.replace_markers(
|
|
414
|
+
prepared,
|
|
415
|
+
projection.value,
|
|
416
|
+
)
|
|
417
|
+
except Exception as exc:
|
|
418
|
+
raise DriverError(
|
|
419
|
+
"R2AG_STAGE_FAILED",
|
|
420
|
+
"R²AG neural conditioning stage failed",
|
|
421
|
+
consumed=True,
|
|
422
|
+
) from exc
|
|
423
|
+
_validate_conditioned(conditioned, prepared)
|
|
424
|
+
|
|
425
|
+
try:
|
|
426
|
+
generated: GenerationResult = runtime.generate(
|
|
427
|
+
conditioned.value,
|
|
428
|
+
max_new_tokens=max_new_tokens,
|
|
429
|
+
seed=_nonnegative_int(payload, "seed"),
|
|
430
|
+
)
|
|
431
|
+
except Exception as exc:
|
|
432
|
+
raise DriverError(
|
|
433
|
+
"READER_FAILED",
|
|
434
|
+
"R²AG reader generation failed",
|
|
435
|
+
consumed=True,
|
|
436
|
+
) from exc
|
|
437
|
+
_validate_generation(generated, max_new_tokens)
|
|
438
|
+
|
|
439
|
+
attestation = {
|
|
440
|
+
"fixture_mode": self.fixture_mode,
|
|
441
|
+
"source_commit": UPSTREAM_COMMIT,
|
|
442
|
+
"source_manifest_sha256": artifacts["source_manifest_sha256"],
|
|
443
|
+
"compatibility_manifest_sha256": self.compatibility_manifest_sha256,
|
|
444
|
+
"checkpoint_config_sha256": artifacts["checkpoint_config_sha256"],
|
|
445
|
+
"rformer_weights_sha256": artifacts["rformer_weights_sha256"],
|
|
446
|
+
"projection_weights_sha256": artifacts["projection_weights_sha256"],
|
|
447
|
+
"base_model_manifest_sha256": artifacts["base_model_manifest_sha256"],
|
|
448
|
+
"tokenizer_manifest_sha256": artifacts["tokenizer_manifest_sha256"],
|
|
449
|
+
"feature_schema_sha256": artifacts["feature_schema_sha256"],
|
|
450
|
+
"worker_config_num_k": num_k,
|
|
451
|
+
"source_commit_verified": True,
|
|
452
|
+
"source_files_verified": True,
|
|
453
|
+
"source_checkout_clean": True,
|
|
454
|
+
"checkpoint_hashes_verified": True,
|
|
455
|
+
"rformer_called": True,
|
|
456
|
+
"projection_called": True,
|
|
457
|
+
"marker_replacement_called": True,
|
|
458
|
+
"reader_generate_called": True,
|
|
459
|
+
"only_marker_positions_modified": True,
|
|
460
|
+
"gold_metadata_received": False,
|
|
461
|
+
"training_called": False,
|
|
462
|
+
"budget_checked_before_reader_call": True,
|
|
463
|
+
"runtime_dependencies": dict(RUNTIME_DEPENDENCIES),
|
|
464
|
+
"runtime_dependencies_verified": not self.fixture_mode,
|
|
465
|
+
"execution_evidence": (
|
|
466
|
+
"fixture_source_execution"
|
|
467
|
+
if self.fixture_mode
|
|
468
|
+
else "pinned_source_checkpoint_execution"
|
|
469
|
+
),
|
|
470
|
+
"source_attestation": source_attestation,
|
|
471
|
+
}
|
|
472
|
+
response_payload = {
|
|
473
|
+
"candidate_document_ids": candidate_ids,
|
|
474
|
+
"candidate_documents_sha256": payload["candidate_documents_sha256"],
|
|
475
|
+
"candidate_order_sha256": payload["candidate_order_sha256"],
|
|
476
|
+
"candidate_pool_fingerprint_sha256": payload["candidate_pool_fingerprint_sha256"],
|
|
477
|
+
"artifacts": artifacts,
|
|
478
|
+
"artifact_identity_sha256": payload["artifact_identity_sha256"],
|
|
479
|
+
"worker_config_num_k": num_k,
|
|
480
|
+
"features": features,
|
|
481
|
+
"feature_tensor_sha256": feature_hash,
|
|
482
|
+
"feature_embedder_fingerprint_sha256": payload["feature_embedder_fingerprint_sha256"],
|
|
483
|
+
"feature_artifact_fingerprint_sha256": payload["feature_artifact_fingerprint_sha256"],
|
|
484
|
+
"reader_instruction_sha256": payload["reader_instruction_sha256"],
|
|
485
|
+
"reader_input_sha256": payload["reader_input_sha256"],
|
|
486
|
+
"r2former_output_sha256": rformer.sha256,
|
|
487
|
+
"projection_output_sha256": projection.sha256,
|
|
488
|
+
"r_token_count": num_k,
|
|
489
|
+
"r_token_positions": prepared.marker_positions,
|
|
490
|
+
"non_retrieval_unk_positions_modified": False,
|
|
491
|
+
"runtime_attestation": attestation,
|
|
492
|
+
"answer": generated.answer,
|
|
493
|
+
}
|
|
494
|
+
return {
|
|
495
|
+
"status": "ok",
|
|
496
|
+
"request_id": request["request_id"],
|
|
497
|
+
"upstream_commit": UPSTREAM_COMMIT,
|
|
498
|
+
"protocol_version": PROTOCOL,
|
|
499
|
+
"payload": response_payload,
|
|
500
|
+
"usage": {
|
|
501
|
+
"model_calls": 1,
|
|
502
|
+
"retrieval_calls": 0,
|
|
503
|
+
"tool_calls": 0,
|
|
504
|
+
"embedding_calls": embedding_calls,
|
|
505
|
+
"input_tokens": prepared.input_tokens,
|
|
506
|
+
"output_tokens": generated.output_tokens,
|
|
507
|
+
"accelerator_seconds": float(generated.accelerator_seconds),
|
|
508
|
+
},
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
def _runtime(self) -> Any:
|
|
512
|
+
if self.runtime_factory is not None:
|
|
513
|
+
return self.runtime_factory(
|
|
514
|
+
self.source_root,
|
|
515
|
+
self.checkpoint_root,
|
|
516
|
+
self.base_model_root,
|
|
517
|
+
self.tokenizer_root,
|
|
518
|
+
)
|
|
519
|
+
from workers.r2ag.runtime import OfficialR2AGRuntime
|
|
520
|
+
|
|
521
|
+
return OfficialR2AGRuntime(
|
|
522
|
+
source_root=self.source_root,
|
|
523
|
+
checkpoint_root=self.checkpoint_root,
|
|
524
|
+
base_model_root=self.base_model_root,
|
|
525
|
+
tokenizer_root=self.tokenizer_root,
|
|
526
|
+
expected_runtime_versions=RUNTIME_DEPENDENCIES,
|
|
527
|
+
)
|
|
528
|
+
|
|
529
|
+
def _verify_source(self) -> Mapping[str, Any]:
|
|
530
|
+
source = _mapping(self.manifest, "source")
|
|
531
|
+
required = _mapping(source, "required_files")
|
|
532
|
+
for relative, expected in required.items():
|
|
533
|
+
if _sha256_file(_safe_file(self.source_root, relative)) != expected:
|
|
534
|
+
raise DriverError("SOURCE_MISMATCH", f"source hash mismatch: {relative}")
|
|
535
|
+
source_manifest = _content_hash(required)
|
|
536
|
+
if self.fixture_mode:
|
|
537
|
+
actual_files = {
|
|
538
|
+
path.relative_to(self.source_root).as_posix()
|
|
539
|
+
for path in self.source_root.rglob("*")
|
|
540
|
+
if path.is_file()
|
|
541
|
+
}
|
|
542
|
+
if actual_files != set(required):
|
|
543
|
+
raise DriverError("SOURCE_NOT_CLOSED", "fixture source tree is not closed")
|
|
544
|
+
return {
|
|
545
|
+
"kind": "closed_fixture_source",
|
|
546
|
+
"tree": source.get("tree"),
|
|
547
|
+
"content_manifest_sha256": source_manifest,
|
|
548
|
+
}
|
|
549
|
+
if source.get("commit") != UPSTREAM_COMMIT or source.get("tree") != UPSTREAM_TREE:
|
|
550
|
+
raise DriverError("SOURCE_MISMATCH", "compatibility source identity drifted")
|
|
551
|
+
head = _git(self.source_root, "rev-parse", "HEAD")
|
|
552
|
+
tree = _git(self.source_root, "rev-parse", "HEAD^{tree}")
|
|
553
|
+
status = _git(self.source_root, "status", "--porcelain", "--untracked-files=all")
|
|
554
|
+
if head != UPSTREAM_COMMIT or tree != UPSTREAM_TREE or status:
|
|
555
|
+
raise DriverError("SOURCE_MISMATCH", "source checkout is not pinned and clean")
|
|
556
|
+
return {
|
|
557
|
+
"kind": "clean_git_checkout",
|
|
558
|
+
"tree": tree,
|
|
559
|
+
"content_manifest_sha256": source_manifest,
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
def _verify_static_artifacts(self, artifacts: Mapping[str, Any]) -> None:
|
|
563
|
+
source_required = _mapping(_mapping(self.manifest, "source"), "required_files")
|
|
564
|
+
if artifacts.get("source_manifest_sha256") != _content_hash(source_required):
|
|
565
|
+
raise DriverError("SOURCE_MISMATCH", "source manifest identity mismatch")
|
|
566
|
+
checks = (
|
|
567
|
+
(
|
|
568
|
+
_safe_file(self.checkpoint_root, "config.json"),
|
|
569
|
+
artifacts["checkpoint_config_sha256"],
|
|
570
|
+
"checkpoint config",
|
|
571
|
+
),
|
|
572
|
+
(
|
|
573
|
+
self.feature_schema_path,
|
|
574
|
+
artifacts["feature_schema_sha256"],
|
|
575
|
+
"feature schema",
|
|
576
|
+
),
|
|
577
|
+
)
|
|
578
|
+
for path, expected, label in checks:
|
|
579
|
+
if _sha256_file(path) != expected:
|
|
580
|
+
raise DriverError("ARTIFACT_MISMATCH", f"{label} hash mismatch")
|
|
581
|
+
_verify_artifact_closure(
|
|
582
|
+
self.base_model_root,
|
|
583
|
+
repository=artifacts.get("base_model_repository"),
|
|
584
|
+
revision=artifacts.get("base_model_revision"),
|
|
585
|
+
manifest_sha256=artifacts.get("base_model_manifest_sha256"),
|
|
586
|
+
label="base model",
|
|
587
|
+
)
|
|
588
|
+
_verify_artifact_closure(
|
|
589
|
+
self.tokenizer_root,
|
|
590
|
+
repository=artifacts.get("tokenizer_repository"),
|
|
591
|
+
revision=artifacts.get("tokenizer_revision"),
|
|
592
|
+
manifest_sha256=artifacts.get("tokenizer_manifest_sha256"),
|
|
593
|
+
label="tokenizer",
|
|
594
|
+
)
|
|
595
|
+
|
|
596
|
+
def _verify_feature_embedder(self, pool: Mapping[str, Any]) -> None:
|
|
597
|
+
identity = _mapping(pool, "retriever")
|
|
598
|
+
manifest_sha256 = identity.get("manifest_sha256")
|
|
599
|
+
if not isinstance(manifest_sha256, str) or not _HEX64.fullmatch(manifest_sha256):
|
|
600
|
+
raise DriverError(
|
|
601
|
+
"ARTIFACT_MISMATCH",
|
|
602
|
+
"worker feature source requires retriever manifest_sha256",
|
|
603
|
+
)
|
|
604
|
+
_verify_artifact_closure(
|
|
605
|
+
self.checkpoint_root / "feature-embedder",
|
|
606
|
+
repository=identity.get("repository"),
|
|
607
|
+
revision=identity.get("revision"),
|
|
608
|
+
manifest_sha256=manifest_sha256,
|
|
609
|
+
label="feature embedder",
|
|
610
|
+
)
|
|
611
|
+
|
|
612
|
+
|
|
613
|
+
def _validate_envelope(request: Mapping[str, Any]) -> Mapping[str, Any]:
|
|
614
|
+
if not isinstance(request, Mapping):
|
|
615
|
+
raise DriverError("REQUEST_INVALID", "request must be a mapping")
|
|
616
|
+
if set(request) != _REQUEST_FIELDS:
|
|
617
|
+
raise DriverError("REQUEST_INVALID", "request envelope fields drifted")
|
|
618
|
+
expected = {
|
|
619
|
+
"method_id": "r2ag",
|
|
620
|
+
"release_id": RELEASE,
|
|
621
|
+
"upstream_commit": UPSTREAM_COMMIT,
|
|
622
|
+
"protocol_version": PROTOCOL,
|
|
623
|
+
}
|
|
624
|
+
if any(request.get(key) != value for key, value in expected.items()):
|
|
625
|
+
raise DriverError("IDENTITY_MISMATCH", "method/release/source/protocol mismatch")
|
|
626
|
+
state_hash = request.get("state_hash")
|
|
627
|
+
if not isinstance(state_hash, str) or not _HEX64.fullmatch(state_hash):
|
|
628
|
+
raise DriverError("REQUEST_INVALID", "state_hash must be a SHA-256 digest")
|
|
629
|
+
reservation = _mapping(request, "minimum_reservation")
|
|
630
|
+
if dict(reservation) != {"model_calls": 1}:
|
|
631
|
+
raise DriverError("REQUEST_INVALID", "minimum reservation must be one model call")
|
|
632
|
+
request_id = request.get("request_id")
|
|
633
|
+
if not isinstance(request_id, str) or not _HEX64.fullmatch(request_id):
|
|
634
|
+
raise DriverError("REQUEST_INVALID", "request_id must be a SHA-256 digest")
|
|
635
|
+
identity = {key: request[key] for key in _REQUEST_ID_FIELDS}
|
|
636
|
+
if request_id != _content_hash(identity):
|
|
637
|
+
raise DriverError("IDENTITY_MISMATCH", "request_id content hash mismatch")
|
|
638
|
+
return _mapping(request, "payload")
|
|
639
|
+
|
|
640
|
+
|
|
641
|
+
def _validate_contract(value: Mapping[str, Any]) -> None:
|
|
642
|
+
expected = {
|
|
643
|
+
"allow_training": False,
|
|
644
|
+
"allow_gold_metadata": False,
|
|
645
|
+
"require_source_verification": True,
|
|
646
|
+
"require_checkpoint_hash_verification": True,
|
|
647
|
+
"require_rformer_then_projection": True,
|
|
648
|
+
"replace_only_retrieval_marker_embeddings": True,
|
|
649
|
+
"check_budget_before_reader_call": True,
|
|
650
|
+
}
|
|
651
|
+
if dict(value) != expected:
|
|
652
|
+
raise DriverError("EXECUTION_CONTRACT_REJECTED", "R²AG execution contract drifted")
|
|
653
|
+
|
|
654
|
+
|
|
655
|
+
def _candidate_documents(value: Any) -> tuple[Mapping[str, Any], ...]:
|
|
656
|
+
if isinstance(value, (str, bytes)) or not isinstance(value, Sequence):
|
|
657
|
+
raise DriverError("REQUEST_INVALID", "candidate_documents must be a sequence")
|
|
658
|
+
rows = []
|
|
659
|
+
for raw in value:
|
|
660
|
+
if not isinstance(raw, Mapping) or set(raw) != _CANDIDATE_FIELDS:
|
|
661
|
+
raise DriverError("ORACLE_FIELD_REJECTED", "candidate DTO fields are not allowlisted")
|
|
662
|
+
for key in ("document_id", "text", "title", "source", "corpus_revision"):
|
|
663
|
+
if not isinstance(raw.get(key), str):
|
|
664
|
+
raise DriverError("REQUEST_INVALID", f"candidate {key} must be text")
|
|
665
|
+
if not raw["document_id"] or not raw["text"]:
|
|
666
|
+
raise DriverError("REQUEST_INVALID", "candidate ID/text must not be empty")
|
|
667
|
+
score = raw.get("score")
|
|
668
|
+
if score is not None and (
|
|
669
|
+
isinstance(score, bool)
|
|
670
|
+
or not isinstance(score, (int, float))
|
|
671
|
+
or not math.isfinite(score)
|
|
672
|
+
):
|
|
673
|
+
raise DriverError("REQUEST_INVALID", "candidate score must be finite or null")
|
|
674
|
+
rows.append(dict(raw))
|
|
675
|
+
return tuple(rows)
|
|
676
|
+
|
|
677
|
+
|
|
678
|
+
def _validate_pool(
|
|
679
|
+
payload: Mapping[str, Any],
|
|
680
|
+
candidates: tuple[Mapping[str, Any], ...],
|
|
681
|
+
) -> Mapping[str, Any]:
|
|
682
|
+
pool = _mapping(payload, "candidate_pool_fingerprint")
|
|
683
|
+
if set(pool) != _POOL_FIELDS:
|
|
684
|
+
raise DriverError("IDENTITY_MISMATCH", "candidate pool fingerprint fields drifted")
|
|
685
|
+
_reject_forbidden_fingerprint_keys(pool, label="candidate pool fingerprint")
|
|
686
|
+
if pool.get("top_k") != len(candidates) or pool.get("order") != "rerank_desc":
|
|
687
|
+
raise DriverError("IDENTITY_MISMATCH", "candidate pool count/order mismatch")
|
|
688
|
+
retriever = _repository_identity(
|
|
689
|
+
pool.get("retriever"),
|
|
690
|
+
label="candidate pool retriever",
|
|
691
|
+
)
|
|
692
|
+
corpus = _repository_identity(
|
|
693
|
+
pool.get("corpus"),
|
|
694
|
+
label="candidate pool corpus",
|
|
695
|
+
)
|
|
696
|
+
index = _index_identity(pool.get("index"), label="candidate pool index")
|
|
697
|
+
if pool.get("content_sha256") != _content_hash(candidates):
|
|
698
|
+
raise DriverError("IDENTITY_MISMATCH", "candidate pool content hash mismatch")
|
|
699
|
+
normalized = {
|
|
700
|
+
"retriever": retriever,
|
|
701
|
+
"corpus": corpus,
|
|
702
|
+
"index": index,
|
|
703
|
+
"top_k": pool["top_k"],
|
|
704
|
+
"order": pool["order"],
|
|
705
|
+
"content_sha256": pool["content_sha256"],
|
|
706
|
+
}
|
|
707
|
+
if dict(pool) != normalized:
|
|
708
|
+
raise DriverError("IDENTITY_MISMATCH", "candidate pool fingerprint is not canonical")
|
|
709
|
+
if payload.get("candidate_pool_fingerprint_sha256") != _content_hash(normalized):
|
|
710
|
+
raise DriverError("IDENTITY_MISMATCH", "candidate pool identity hash mismatch")
|
|
711
|
+
return normalized
|
|
712
|
+
|
|
713
|
+
|
|
714
|
+
def _validate_feature_identities(
|
|
715
|
+
payload: Mapping[str, Any],
|
|
716
|
+
*,
|
|
717
|
+
feature_source: Any,
|
|
718
|
+
) -> tuple[Mapping[str, Any] | None, Mapping[str, Any] | None]:
|
|
719
|
+
embedder_raw = payload.get("feature_embedder_fingerprint")
|
|
720
|
+
artifact_raw = payload.get("feature_artifact_fingerprint")
|
|
721
|
+
if feature_source == "embedder":
|
|
722
|
+
embedder = _feature_embedder_identity(
|
|
723
|
+
embedder_raw,
|
|
724
|
+
label="feature embedder fingerprint",
|
|
725
|
+
)
|
|
726
|
+
artifact = None
|
|
727
|
+
elif feature_source == "candidate_metadata":
|
|
728
|
+
embedder = None
|
|
729
|
+
artifact = _feature_artifact_identity(
|
|
730
|
+
artifact_raw,
|
|
731
|
+
label="feature artifact fingerprint",
|
|
732
|
+
)
|
|
733
|
+
elif feature_source == "worker":
|
|
734
|
+
embedder = None
|
|
735
|
+
artifact = None
|
|
736
|
+
else:
|
|
737
|
+
raise DriverError("PROFILE_DRIFT", "unsupported R²AG feature source")
|
|
738
|
+
if embedder_raw is not None and embedder is None:
|
|
739
|
+
raise DriverError("PROFILE_DRIFT", "unexpected feature embedder fingerprint")
|
|
740
|
+
if artifact_raw is not None and artifact is None:
|
|
741
|
+
raise DriverError("PROFILE_DRIFT", "unexpected feature artifact fingerprint")
|
|
742
|
+
if payload.get("feature_embedder_fingerprint_sha256") != _content_hash(embedder):
|
|
743
|
+
raise DriverError("IDENTITY_MISMATCH", "feature embedder fingerprint hash mismatch")
|
|
744
|
+
if payload.get("feature_artifact_fingerprint_sha256") != _content_hash(artifact):
|
|
745
|
+
raise DriverError("IDENTITY_MISMATCH", "feature artifact fingerprint hash mismatch")
|
|
746
|
+
return embedder, artifact
|
|
747
|
+
|
|
748
|
+
|
|
749
|
+
def _validate_dependency_fingerprints(
|
|
750
|
+
value: Mapping[str, Any],
|
|
751
|
+
*,
|
|
752
|
+
pool: Mapping[str, Any],
|
|
753
|
+
artifacts: Mapping[str, Any],
|
|
754
|
+
feature_source: Any,
|
|
755
|
+
feature_embedder_identity: Mapping[str, Any] | None,
|
|
756
|
+
feature_artifact_identity: Mapping[str, Any] | None,
|
|
757
|
+
) -> None:
|
|
758
|
+
expected_keys = {"candidate_pool", "worker", "artifacts"}
|
|
759
|
+
if feature_source == "embedder":
|
|
760
|
+
expected_keys.add("feature_embedder")
|
|
761
|
+
elif feature_source == "candidate_metadata":
|
|
762
|
+
expected_keys.add("feature_artifact")
|
|
763
|
+
if set(value) != expected_keys:
|
|
764
|
+
raise DriverError("IDENTITY_MISMATCH", "dependency fingerprint fields drifted")
|
|
765
|
+
_reject_forbidden_fingerprint_keys(value, label="dependency fingerprints")
|
|
766
|
+
if value.get("candidate_pool") != pool or value.get("artifacts") != artifacts:
|
|
767
|
+
raise DriverError("IDENTITY_MISMATCH", "dependency fingerprint identity mismatch")
|
|
768
|
+
worker = value.get("worker")
|
|
769
|
+
if not isinstance(worker, Mapping) or not worker:
|
|
770
|
+
raise DriverError("IDENTITY_MISMATCH", "worker fingerprint must be a mapping")
|
|
771
|
+
if feature_source == "embedder" and value.get("feature_embedder") != (
|
|
772
|
+
feature_embedder_identity
|
|
773
|
+
):
|
|
774
|
+
raise DriverError("IDENTITY_MISMATCH", "feature embedder dependency mismatch")
|
|
775
|
+
if feature_source == "candidate_metadata" and value.get("feature_artifact") != (
|
|
776
|
+
feature_artifact_identity
|
|
777
|
+
):
|
|
778
|
+
raise DriverError("IDENTITY_MISMATCH", "feature artifact dependency mismatch")
|
|
779
|
+
|
|
780
|
+
|
|
781
|
+
def _features(value: Any, *, expected_count: int) -> tuple[Mapping[str, float], ...]:
|
|
782
|
+
if isinstance(value, (str, bytes)) or not isinstance(value, Sequence):
|
|
783
|
+
raise DriverError("REQUEST_INVALID", "features must be a sequence")
|
|
784
|
+
rows = []
|
|
785
|
+
for raw in value:
|
|
786
|
+
if not isinstance(raw, Mapping) or set(raw) != set(FEATURE_NAMES):
|
|
787
|
+
raise DriverError("REQUEST_INVALID", "feature row schema mismatch")
|
|
788
|
+
row: dict[str, float] = {}
|
|
789
|
+
for name in FEATURE_NAMES:
|
|
790
|
+
number = raw[name]
|
|
791
|
+
if (
|
|
792
|
+
isinstance(number, bool)
|
|
793
|
+
or not isinstance(number, (int, float))
|
|
794
|
+
or not math.isfinite(number)
|
|
795
|
+
or number < -1.000001
|
|
796
|
+
or number > 1.000001
|
|
797
|
+
):
|
|
798
|
+
raise DriverError("REQUEST_INVALID", "feature values must be finite cosine scores")
|
|
799
|
+
row[name] = float(number)
|
|
800
|
+
rows.append(row)
|
|
801
|
+
if len(rows) != expected_count:
|
|
802
|
+
raise DriverError("REQUEST_INVALID", "feature count must equal config.num_k")
|
|
803
|
+
return tuple(rows)
|
|
804
|
+
|
|
805
|
+
|
|
806
|
+
def _repository_identity(value: Any, *, label: str) -> Mapping[str, str]:
|
|
807
|
+
if not isinstance(value, Mapping):
|
|
808
|
+
raise DriverError("IDENTITY_MISMATCH", f"{label} must be a mapping")
|
|
809
|
+
_reject_forbidden_fingerprint_keys(value, label=label)
|
|
810
|
+
keys = set(value)
|
|
811
|
+
allowed = _REPOSITORY_IDENTITY_REQUIRED | _REPOSITORY_IDENTITY_OPTIONAL
|
|
812
|
+
if not _REPOSITORY_IDENTITY_REQUIRED.issubset(keys) or not keys.issubset(allowed):
|
|
813
|
+
raise DriverError(
|
|
814
|
+
"IDENTITY_MISMATCH",
|
|
815
|
+
f"{label} must contain repository/revision and optional manifest_sha256 only",
|
|
816
|
+
)
|
|
817
|
+
repository = value.get("repository")
|
|
818
|
+
revision = _pinned_revision(value.get("revision"), label=f"{label}.revision")
|
|
819
|
+
if not isinstance(repository, str) or not repository.strip():
|
|
820
|
+
raise DriverError("IDENTITY_MISMATCH", f"{label} values must be non-empty text")
|
|
821
|
+
identity: dict[str, str] = {
|
|
822
|
+
"repository": repository.strip(),
|
|
823
|
+
"revision": revision,
|
|
824
|
+
}
|
|
825
|
+
if "manifest_sha256" in value:
|
|
826
|
+
digest = value.get("manifest_sha256")
|
|
827
|
+
if not isinstance(digest, str) or not _HEX64.fullmatch(digest):
|
|
828
|
+
raise DriverError("IDENTITY_MISMATCH", f"{label} manifest hash is invalid")
|
|
829
|
+
identity["manifest_sha256"] = digest
|
|
830
|
+
return identity
|
|
831
|
+
|
|
832
|
+
|
|
833
|
+
def _index_identity(value: Any, *, label: str) -> Mapping[str, str]:
|
|
834
|
+
if not isinstance(value, Mapping):
|
|
835
|
+
raise DriverError("IDENTITY_MISMATCH", f"{label} must be a mapping")
|
|
836
|
+
_reject_forbidden_fingerprint_keys(value, label=label)
|
|
837
|
+
keys = set(value)
|
|
838
|
+
allowed = _INDEX_IDENTITY_REQUIRED | _INDEX_IDENTITY_OPTIONAL
|
|
839
|
+
if not _INDEX_IDENTITY_REQUIRED.issubset(keys) or not keys.issubset(allowed):
|
|
840
|
+
raise DriverError(
|
|
841
|
+
"IDENTITY_MISMATCH",
|
|
842
|
+
f"{label} must contain algorithm/revision and optional manifest_sha256 only",
|
|
843
|
+
)
|
|
844
|
+
algorithm = value.get("algorithm")
|
|
845
|
+
revision = _pinned_revision(value.get("revision"), label=f"{label}.revision")
|
|
846
|
+
if not isinstance(algorithm, str) or not algorithm.strip():
|
|
847
|
+
raise DriverError("IDENTITY_MISMATCH", f"{label} values must be non-empty text")
|
|
848
|
+
identity: dict[str, str] = {
|
|
849
|
+
"algorithm": algorithm.strip(),
|
|
850
|
+
"revision": revision,
|
|
851
|
+
}
|
|
852
|
+
if "manifest_sha256" in value:
|
|
853
|
+
digest = value.get("manifest_sha256")
|
|
854
|
+
if not isinstance(digest, str) or not _HEX64.fullmatch(digest):
|
|
855
|
+
raise DriverError("IDENTITY_MISMATCH", f"{label} manifest hash is invalid")
|
|
856
|
+
identity["manifest_sha256"] = digest
|
|
857
|
+
return identity
|
|
858
|
+
|
|
859
|
+
|
|
860
|
+
def _feature_embedder_identity(value: Any, *, label: str) -> Mapping[str, str]:
|
|
861
|
+
if not isinstance(value, Mapping) or set(value) != _FEATURE_EMBEDDER_FIELDS:
|
|
862
|
+
raise DriverError(
|
|
863
|
+
"IDENTITY_MISMATCH",
|
|
864
|
+
f"{label} must contain exactly kind and fingerprint_sha256",
|
|
865
|
+
)
|
|
866
|
+
_reject_forbidden_fingerprint_keys(value, label=label)
|
|
867
|
+
if value.get("kind") != "dense_embedder":
|
|
868
|
+
raise DriverError("IDENTITY_MISMATCH", f"{label} kind is invalid")
|
|
869
|
+
digest = value.get("fingerprint_sha256")
|
|
870
|
+
if not isinstance(digest, str) or not _HEX64.fullmatch(digest):
|
|
871
|
+
raise DriverError("IDENTITY_MISMATCH", f"{label} digest is invalid")
|
|
872
|
+
return {"kind": "dense_embedder", "fingerprint_sha256": digest}
|
|
873
|
+
|
|
874
|
+
|
|
875
|
+
def _feature_artifact_identity(value: Any, *, label: str) -> Mapping[str, str]:
|
|
876
|
+
if not isinstance(value, Mapping):
|
|
877
|
+
raise DriverError(
|
|
878
|
+
"IDENTITY_MISMATCH",
|
|
879
|
+
f"{label} must contain exactly repository, revision, and sha256",
|
|
880
|
+
)
|
|
881
|
+
_reject_forbidden_fingerprint_keys(value, label=label)
|
|
882
|
+
if set(value) != _FEATURE_ARTIFACT_FIELDS:
|
|
883
|
+
raise DriverError(
|
|
884
|
+
"IDENTITY_MISMATCH",
|
|
885
|
+
f"{label} must contain exactly repository, revision, and sha256",
|
|
886
|
+
)
|
|
887
|
+
repository = value.get("repository")
|
|
888
|
+
revision = _pinned_revision(value.get("revision"), label=f"{label}.revision")
|
|
889
|
+
digest = value.get("sha256")
|
|
890
|
+
if (
|
|
891
|
+
not isinstance(repository, str)
|
|
892
|
+
or not repository.strip()
|
|
893
|
+
or not isinstance(digest, str)
|
|
894
|
+
or not _HEX64.fullmatch(digest)
|
|
895
|
+
):
|
|
896
|
+
raise DriverError("IDENTITY_MISMATCH", f"{label} values are invalid")
|
|
897
|
+
return {
|
|
898
|
+
"repository": repository.strip(),
|
|
899
|
+
"revision": revision,
|
|
900
|
+
"sha256": digest,
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
|
|
904
|
+
def _reject_forbidden_fingerprint_keys(value: Any, *, label: str) -> None:
|
|
905
|
+
if isinstance(value, Mapping):
|
|
906
|
+
for key, nested in value.items():
|
|
907
|
+
if not isinstance(key, str):
|
|
908
|
+
raise DriverError("REQUEST_INVALID", f"{label} keys must be strings")
|
|
909
|
+
terms = tuple(re.findall(r"[a-z0-9]+", key.lower()))
|
|
910
|
+
compact = "".join(terms)
|
|
911
|
+
if any(term in _FORBIDDEN_FINGERPRINT_TERMS for term in (*terms, compact)):
|
|
912
|
+
raise DriverError(
|
|
913
|
+
"ORACLE_FIELD_REJECTED",
|
|
914
|
+
f"{label} contains a forbidden metadata key",
|
|
915
|
+
)
|
|
916
|
+
_reject_forbidden_fingerprint_keys(nested, label=label)
|
|
917
|
+
elif isinstance(value, Sequence) and not isinstance(value, (str, bytes)):
|
|
918
|
+
for nested in value:
|
|
919
|
+
_reject_forbidden_fingerprint_keys(nested, label=label)
|
|
920
|
+
|
|
921
|
+
|
|
922
|
+
def _pinned_revision(value: Any, *, label: str) -> str:
|
|
923
|
+
if not isinstance(value, str) or not value.strip():
|
|
924
|
+
raise DriverError("IDENTITY_MISMATCH", f"{label} must be non-empty text")
|
|
925
|
+
revision = value.strip()
|
|
926
|
+
if revision.lower() in _IMMUTABLE_REVISION_ALIASES:
|
|
927
|
+
raise DriverError("IDENTITY_MISMATCH", f"{label} must be immutable")
|
|
928
|
+
return revision
|
|
929
|
+
|
|
930
|
+
|
|
931
|
+
def _validate_descending(features: Sequence[Mapping[str, float]]) -> None:
|
|
932
|
+
if any(
|
|
933
|
+
features[index]["rerank"] + 1e-6 < features[index + 1]["rerank"]
|
|
934
|
+
for index in range(len(features) - 1)
|
|
935
|
+
):
|
|
936
|
+
raise DriverError("REQUEST_INVALID", "features are not in descending rerank order")
|
|
937
|
+
|
|
938
|
+
|
|
939
|
+
def _validate_prepared(value: Any, num_k: int) -> None:
|
|
940
|
+
if not isinstance(value, PreparedReader):
|
|
941
|
+
raise DriverError("TOKENIZATION_FAILED", "runtime returned invalid prepared reader")
|
|
942
|
+
if (
|
|
943
|
+
value.marker_positions != tuple(sorted(value.marker_positions))
|
|
944
|
+
or len(value.marker_positions) != num_k
|
|
945
|
+
or len(set(value.marker_positions)) != num_k
|
|
946
|
+
or isinstance(value.input_tokens, bool)
|
|
947
|
+
or not isinstance(value.input_tokens, int)
|
|
948
|
+
or value.input_tokens < 1
|
|
949
|
+
):
|
|
950
|
+
raise DriverError("MARKER_MISMATCH", "tokenized marker positions do not match num_k")
|
|
951
|
+
|
|
952
|
+
|
|
953
|
+
def _validate_tensor_stage(value: Any, *, label: str) -> None:
|
|
954
|
+
if not isinstance(value, TensorStage) or not _HEX64.fullmatch(value.sha256):
|
|
955
|
+
raise DriverError("R2AG_STAGE_FAILED", f"{label} returned invalid evidence", consumed=True)
|
|
956
|
+
|
|
957
|
+
|
|
958
|
+
def _validate_conditioned(value: Any, prepared: PreparedReader) -> None:
|
|
959
|
+
if (
|
|
960
|
+
not isinstance(value, ConditionedReader)
|
|
961
|
+
or value.marker_positions != prepared.marker_positions
|
|
962
|
+
or value.only_marker_positions_modified is not True
|
|
963
|
+
):
|
|
964
|
+
raise DriverError(
|
|
965
|
+
"MARKER_MISMATCH",
|
|
966
|
+
"runtime did not replace only the retrieval markers",
|
|
967
|
+
consumed=True,
|
|
968
|
+
)
|
|
969
|
+
|
|
970
|
+
|
|
971
|
+
def _validate_generation(value: Any, max_new_tokens: int) -> None:
|
|
972
|
+
if (
|
|
973
|
+
not isinstance(value, GenerationResult)
|
|
974
|
+
or not isinstance(value.answer, str)
|
|
975
|
+
or not value.answer.strip()
|
|
976
|
+
or isinstance(value.output_tokens, bool)
|
|
977
|
+
or not isinstance(value.output_tokens, int)
|
|
978
|
+
or value.output_tokens < 1
|
|
979
|
+
or value.output_tokens > max_new_tokens
|
|
980
|
+
or isinstance(value.accelerator_seconds, bool)
|
|
981
|
+
or not isinstance(value.accelerator_seconds, (int, float))
|
|
982
|
+
or not math.isfinite(value.accelerator_seconds)
|
|
983
|
+
or value.accelerator_seconds < 0
|
|
984
|
+
):
|
|
985
|
+
raise DriverError("READER_FAILED", "runtime returned invalid reader output", consumed=True)
|
|
986
|
+
|
|
987
|
+
|
|
988
|
+
def _validate_artifact_identity(artifacts: Mapping[str, Any]) -> None:
|
|
989
|
+
if set(artifacts) != _ARTIFACT_FIELDS:
|
|
990
|
+
raise DriverError("REQUEST_INVALID", "artifact identity fields drifted")
|
|
991
|
+
required = {
|
|
992
|
+
"checkpoint_config_sha256",
|
|
993
|
+
"rformer_weights_sha256",
|
|
994
|
+
"projection_weights_sha256",
|
|
995
|
+
"base_model_manifest_sha256",
|
|
996
|
+
"tokenizer_manifest_sha256",
|
|
997
|
+
"feature_schema_sha256",
|
|
998
|
+
"source_manifest_sha256",
|
|
999
|
+
"compatibility_manifest_sha256",
|
|
1000
|
+
}
|
|
1001
|
+
if any(
|
|
1002
|
+
not isinstance(artifacts.get(key), str) or not _HEX64.fullmatch(artifacts[key])
|
|
1003
|
+
for key in required
|
|
1004
|
+
):
|
|
1005
|
+
raise DriverError("REQUEST_INVALID", "artifact SHA-256 identity is invalid")
|
|
1006
|
+
for key in (
|
|
1007
|
+
"checkpoint_repository",
|
|
1008
|
+
"base_model_repository",
|
|
1009
|
+
"tokenizer_repository",
|
|
1010
|
+
):
|
|
1011
|
+
value = artifacts.get(key)
|
|
1012
|
+
if not isinstance(value, str) or not value.strip():
|
|
1013
|
+
raise DriverError("IDENTITY_MISMATCH", f"{key} must be non-empty text")
|
|
1014
|
+
for key in (
|
|
1015
|
+
"checkpoint_revision",
|
|
1016
|
+
"base_model_revision",
|
|
1017
|
+
"tokenizer_revision",
|
|
1018
|
+
"feature_schema_revision",
|
|
1019
|
+
):
|
|
1020
|
+
_pinned_revision(artifacts.get(key), label=key)
|
|
1021
|
+
|
|
1022
|
+
|
|
1023
|
+
def _verify_artifact_closure(
|
|
1024
|
+
root: Path,
|
|
1025
|
+
*,
|
|
1026
|
+
repository: Any,
|
|
1027
|
+
revision: Any,
|
|
1028
|
+
manifest_sha256: Any,
|
|
1029
|
+
label: str,
|
|
1030
|
+
) -> Mapping[str, Any]:
|
|
1031
|
+
"""Verify one exact, symlink-free local snapshot closure."""
|
|
1032
|
+
|
|
1033
|
+
if root.is_symlink():
|
|
1034
|
+
raise DriverError("PATH_REJECTED", f"{label} root must not be a symlink")
|
|
1035
|
+
try:
|
|
1036
|
+
resolved_root = root.resolve(strict=True)
|
|
1037
|
+
except FileNotFoundError as exc:
|
|
1038
|
+
raise DriverError("PATH_REJECTED", f"{label} root is missing") from exc
|
|
1039
|
+
if not resolved_root.is_dir():
|
|
1040
|
+
raise DriverError("PATH_REJECTED", f"{label} root must be a directory")
|
|
1041
|
+
manifest_path = _safe_file(resolved_root, "agentinputlab-manifest.json")
|
|
1042
|
+
if not isinstance(manifest_sha256, str) or not _HEX64.fullmatch(manifest_sha256):
|
|
1043
|
+
raise DriverError("ARTIFACT_MISMATCH", f"{label} manifest hash is invalid")
|
|
1044
|
+
if _sha256_file(manifest_path) != manifest_sha256:
|
|
1045
|
+
raise DriverError("ARTIFACT_MISMATCH", f"{label} manifest hash mismatch")
|
|
1046
|
+
manifest = _load_json(manifest_path, label=f"{label} manifest")
|
|
1047
|
+
if set(manifest) != _ARTIFACT_MANIFEST_FIELDS:
|
|
1048
|
+
raise DriverError("MANIFEST_INVALID", f"{label} manifest fields drifted")
|
|
1049
|
+
if (
|
|
1050
|
+
manifest.get("schema") != _ARTIFACT_MANIFEST_SCHEMA
|
|
1051
|
+
or manifest.get("repository") != repository
|
|
1052
|
+
or manifest.get("revision") != revision
|
|
1053
|
+
):
|
|
1054
|
+
raise DriverError("ARTIFACT_MISMATCH", f"{label} manifest identity mismatch")
|
|
1055
|
+
raw_files = manifest.get("files")
|
|
1056
|
+
if not isinstance(raw_files, Mapping) or not raw_files:
|
|
1057
|
+
raise DriverError("MANIFEST_INVALID", f"{label} files closure must be non-empty")
|
|
1058
|
+
|
|
1059
|
+
expected_files: dict[str, Mapping[str, Any]] = {}
|
|
1060
|
+
for relative, descriptor in raw_files.items():
|
|
1061
|
+
if (
|
|
1062
|
+
not isinstance(relative, str)
|
|
1063
|
+
or not relative
|
|
1064
|
+
or "\\" in relative
|
|
1065
|
+
or relative == "agentinputlab-manifest.json"
|
|
1066
|
+
or Path(relative).is_absolute()
|
|
1067
|
+
or Path(relative).as_posix() != relative
|
|
1068
|
+
or any(part in {"", ".", ".."} for part in Path(relative).parts)
|
|
1069
|
+
):
|
|
1070
|
+
raise DriverError("MANIFEST_INVALID", f"{label} manifest path is invalid")
|
|
1071
|
+
if not isinstance(descriptor, Mapping) or set(descriptor) != _ARTIFACT_FILE_FIELDS:
|
|
1072
|
+
raise DriverError("MANIFEST_INVALID", f"{label} file descriptor is invalid")
|
|
1073
|
+
size = descriptor.get("size")
|
|
1074
|
+
digest = descriptor.get("sha256")
|
|
1075
|
+
if (
|
|
1076
|
+
isinstance(size, bool)
|
|
1077
|
+
or not isinstance(size, int)
|
|
1078
|
+
or size < 0
|
|
1079
|
+
or not isinstance(digest, str)
|
|
1080
|
+
or not _HEX64.fullmatch(digest)
|
|
1081
|
+
):
|
|
1082
|
+
raise DriverError("MANIFEST_INVALID", f"{label} file descriptor is invalid")
|
|
1083
|
+
expected_files[relative] = {"size": size, "sha256": digest}
|
|
1084
|
+
|
|
1085
|
+
entries = tuple(resolved_root.rglob("*"))
|
|
1086
|
+
if any(path.is_symlink() for path in entries):
|
|
1087
|
+
raise DriverError("PATH_REJECTED", f"{label} closure contains a symlink")
|
|
1088
|
+
actual_files = {
|
|
1089
|
+
path.relative_to(resolved_root).as_posix() for path in entries if path.is_file()
|
|
1090
|
+
}
|
|
1091
|
+
if actual_files != {*expected_files, "agentinputlab-manifest.json"}:
|
|
1092
|
+
raise DriverError("ARTIFACT_MISMATCH", f"{label} file closure mismatch")
|
|
1093
|
+
for relative, descriptor in expected_files.items():
|
|
1094
|
+
path = _safe_file(resolved_root, relative)
|
|
1095
|
+
if path.stat().st_size != descriptor["size"]:
|
|
1096
|
+
raise DriverError("ARTIFACT_MISMATCH", f"{label} file size mismatch")
|
|
1097
|
+
if _sha256_file(path) != descriptor["sha256"]:
|
|
1098
|
+
raise DriverError("ARTIFACT_MISMATCH", f"{label} file hash mismatch")
|
|
1099
|
+
return manifest
|
|
1100
|
+
|
|
1101
|
+
|
|
1102
|
+
def _safe_file(root: Path, relative: Any) -> Path:
|
|
1103
|
+
if not isinstance(relative, str) or not relative or Path(relative).is_absolute():
|
|
1104
|
+
raise DriverError("PATH_REJECTED", "artifact path is invalid")
|
|
1105
|
+
candidate = root / relative
|
|
1106
|
+
if candidate.is_symlink():
|
|
1107
|
+
raise DriverError("PATH_REJECTED", "symlinks are forbidden")
|
|
1108
|
+
try:
|
|
1109
|
+
resolved = candidate.resolve(strict=True)
|
|
1110
|
+
resolved.relative_to(root)
|
|
1111
|
+
except (FileNotFoundError, ValueError) as exc:
|
|
1112
|
+
raise DriverError("PATH_REJECTED", "artifact path escapes its mount or is missing") from exc
|
|
1113
|
+
if not resolved.is_file():
|
|
1114
|
+
raise DriverError("PATH_REJECTED", "artifact path must be a regular file")
|
|
1115
|
+
return resolved
|
|
1116
|
+
|
|
1117
|
+
|
|
1118
|
+
def _load_json(path: Path, *, label: str) -> Mapping[str, Any]:
|
|
1119
|
+
try:
|
|
1120
|
+
value = json.loads(path.read_text(encoding="utf-8"))
|
|
1121
|
+
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
1122
|
+
raise DriverError("MANIFEST_INVALID", f"{label} is unreadable") from exc
|
|
1123
|
+
if not isinstance(value, Mapping):
|
|
1124
|
+
raise DriverError("MANIFEST_INVALID", f"{label} must be a JSON object")
|
|
1125
|
+
return value
|
|
1126
|
+
|
|
1127
|
+
|
|
1128
|
+
def _mapping(value: Mapping[str, Any], key: str) -> Mapping[str, Any]:
|
|
1129
|
+
result = value.get(key)
|
|
1130
|
+
if not isinstance(result, Mapping):
|
|
1131
|
+
raise DriverError("REQUEST_INVALID", f"{key} must be a mapping")
|
|
1132
|
+
return result
|
|
1133
|
+
|
|
1134
|
+
|
|
1135
|
+
def _text(value: Mapping[str, Any], key: str) -> str:
|
|
1136
|
+
result = value.get(key)
|
|
1137
|
+
if not isinstance(result, str) or not result.strip():
|
|
1138
|
+
raise DriverError("REQUEST_INVALID", f"{key} must be non-empty text")
|
|
1139
|
+
return result
|
|
1140
|
+
|
|
1141
|
+
|
|
1142
|
+
def _string_sequence(value: Any, *, label: str) -> tuple[str, ...]:
|
|
1143
|
+
if isinstance(value, (str, bytes)) or not isinstance(value, Sequence):
|
|
1144
|
+
raise DriverError("REQUEST_INVALID", f"{label} must be a sequence")
|
|
1145
|
+
result = tuple(value)
|
|
1146
|
+
if not all(isinstance(item, str) and item for item in result):
|
|
1147
|
+
raise DriverError("REQUEST_INVALID", f"{label} must contain non-empty text")
|
|
1148
|
+
return result
|
|
1149
|
+
|
|
1150
|
+
|
|
1151
|
+
def _positive_int(value: Mapping[str, Any], key: str) -> int:
|
|
1152
|
+
result = value.get(key)
|
|
1153
|
+
if isinstance(result, bool) or not isinstance(result, int) or result < 1:
|
|
1154
|
+
raise DriverError("REQUEST_INVALID", f"{key} must be a positive integer")
|
|
1155
|
+
return result
|
|
1156
|
+
|
|
1157
|
+
|
|
1158
|
+
def _nonnegative_int(value: Mapping[str, Any], key: str) -> int:
|
|
1159
|
+
result = value.get(key)
|
|
1160
|
+
if isinstance(result, bool) or not isinstance(result, int) or result < 0:
|
|
1161
|
+
raise DriverError("REQUEST_INVALID", f"{key} must be a non-negative integer")
|
|
1162
|
+
return result
|
|
1163
|
+
|
|
1164
|
+
|
|
1165
|
+
def _git(root: Path, *args: str) -> str:
|
|
1166
|
+
completed = subprocess.run(
|
|
1167
|
+
["git", "-C", str(root), *args],
|
|
1168
|
+
check=False,
|
|
1169
|
+
capture_output=True,
|
|
1170
|
+
text=True,
|
|
1171
|
+
timeout=10,
|
|
1172
|
+
)
|
|
1173
|
+
if completed.returncode != 0:
|
|
1174
|
+
raise DriverError("SOURCE_MISMATCH", "source Git verification failed")
|
|
1175
|
+
return completed.stdout.strip()
|
|
1176
|
+
|
|
1177
|
+
|
|
1178
|
+
def _sha256_file(path: Path) -> str:
|
|
1179
|
+
digest = hashlib.sha256()
|
|
1180
|
+
try:
|
|
1181
|
+
with path.open("rb") as handle:
|
|
1182
|
+
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
1183
|
+
digest.update(chunk)
|
|
1184
|
+
except OSError as exc:
|
|
1185
|
+
raise DriverError("ARTIFACT_INVALID", "artifact is unreadable") from exc
|
|
1186
|
+
return digest.hexdigest()
|
|
1187
|
+
|
|
1188
|
+
|
|
1189
|
+
def _content_hash(value: Any) -> str:
|
|
1190
|
+
encoded = json.dumps(
|
|
1191
|
+
value,
|
|
1192
|
+
ensure_ascii=False,
|
|
1193
|
+
sort_keys=True,
|
|
1194
|
+
separators=(",", ":"),
|
|
1195
|
+
allow_nan=False,
|
|
1196
|
+
).encode("utf-8")
|
|
1197
|
+
return hashlib.sha256(encoded).hexdigest()
|
|
1198
|
+
|
|
1199
|
+
|
|
1200
|
+
def serve_jsonl(
|
|
1201
|
+
driver: R2AGDriver,
|
|
1202
|
+
source: TextIO = sys.stdin,
|
|
1203
|
+
sink: TextIO = sys.stdout,
|
|
1204
|
+
) -> int:
|
|
1205
|
+
"""Serve one strict JSON response for every non-empty JSON input line."""
|
|
1206
|
+
|
|
1207
|
+
for line in source:
|
|
1208
|
+
if not line.strip():
|
|
1209
|
+
continue
|
|
1210
|
+
try:
|
|
1211
|
+
request = _loads_json_object(line)
|
|
1212
|
+
with redirect_stdout(sys.stderr):
|
|
1213
|
+
response = driver.handle(request)
|
|
1214
|
+
except (json.JSONDecodeError, DriverError, TypeError, ValueError):
|
|
1215
|
+
response = {
|
|
1216
|
+
"status": "error",
|
|
1217
|
+
"failure_code": "REQUEST_INVALID",
|
|
1218
|
+
"message": "request is not valid canonical JSON",
|
|
1219
|
+
"consumed": False,
|
|
1220
|
+
}
|
|
1221
|
+
sink.write(
|
|
1222
|
+
json.dumps(
|
|
1223
|
+
response,
|
|
1224
|
+
ensure_ascii=False,
|
|
1225
|
+
sort_keys=True,
|
|
1226
|
+
separators=(",", ":"),
|
|
1227
|
+
allow_nan=False,
|
|
1228
|
+
)
|
|
1229
|
+
+ "\n"
|
|
1230
|
+
)
|
|
1231
|
+
sink.flush()
|
|
1232
|
+
return 0
|
|
1233
|
+
|
|
1234
|
+
|
|
1235
|
+
def _loads_json_object(value: str) -> Mapping[str, Any]:
|
|
1236
|
+
def reject_duplicates(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
|
|
1237
|
+
result: dict[str, Any] = {}
|
|
1238
|
+
for key, item in pairs:
|
|
1239
|
+
if key in result:
|
|
1240
|
+
raise ValueError("duplicate JSON key")
|
|
1241
|
+
result[key] = item
|
|
1242
|
+
return result
|
|
1243
|
+
|
|
1244
|
+
decoded = json.loads(value, object_pairs_hook=reject_duplicates)
|
|
1245
|
+
if not isinstance(decoded, Mapping):
|
|
1246
|
+
raise ValueError("JSON request must be an object")
|
|
1247
|
+
return decoded
|
|
1248
|
+
|
|
1249
|
+
|
|
1250
|
+
def main(argv: Sequence[str] | None = None) -> int:
|
|
1251
|
+
parser = argparse.ArgumentParser(description="Run the pinned R²AG JSONL worker")
|
|
1252
|
+
parser.add_argument("--source-root", required=True, type=Path)
|
|
1253
|
+
parser.add_argument("--checkpoint-root", required=True, type=Path)
|
|
1254
|
+
parser.add_argument("--base-model-root", required=True, type=Path)
|
|
1255
|
+
parser.add_argument("--tokenizer-root", required=True, type=Path)
|
|
1256
|
+
parser.add_argument("--manifest-path", type=Path, default=DEFAULT_MANIFEST)
|
|
1257
|
+
parser.add_argument("--feature-schema-path", type=Path, default=DEFAULT_FEATURE_SCHEMA)
|
|
1258
|
+
arguments = parser.parse_args(argv)
|
|
1259
|
+
driver = R2AGDriver(
|
|
1260
|
+
source_root=arguments.source_root,
|
|
1261
|
+
checkpoint_root=arguments.checkpoint_root,
|
|
1262
|
+
base_model_root=arguments.base_model_root,
|
|
1263
|
+
tokenizer_root=arguments.tokenizer_root,
|
|
1264
|
+
manifest_path=arguments.manifest_path,
|
|
1265
|
+
feature_schema_path=arguments.feature_schema_path,
|
|
1266
|
+
)
|
|
1267
|
+
return serve_jsonl(driver)
|
|
1268
|
+
|
|
1269
|
+
|
|
1270
|
+
if __name__ == "__main__": # pragma: no cover
|
|
1271
|
+
raise SystemExit(main())
|
|
1272
|
+
|
|
1273
|
+
|
|
1274
|
+
__all__ = [
|
|
1275
|
+
"FEATURE_NAMES",
|
|
1276
|
+
"PROTOCOL",
|
|
1277
|
+
"RELEASE",
|
|
1278
|
+
"RUNTIME_DEPENDENCIES",
|
|
1279
|
+
"UPSTREAM_COMMIT",
|
|
1280
|
+
"ConditionedReader",
|
|
1281
|
+
"DriverError",
|
|
1282
|
+
"GenerationResult",
|
|
1283
|
+
"PreparedReader",
|
|
1284
|
+
"R2AGDriver",
|
|
1285
|
+
"TensorStage",
|
|
1286
|
+
"serve_jsonl",
|
|
1287
|
+
]
|