distributed-llm 0.4.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- distllm/__init__.py +47 -0
- distllm/api/__init__.py +11 -0
- distllm/api/api_settings.py +126 -0
- distllm/api/api_state.py +85 -0
- distllm/api/api_versioning.py +984 -0
- distllm/api/app_state.py +29 -0
- distllm/api/auth/__init__.py +181 -0
- distllm/api/auth/models.py +37 -0
- distllm/api/auth/oauth2.py +123 -0
- distllm/api/auth/oidc.py +317 -0
- distllm/api/auth/saml.py +107 -0
- distllm/api/auth/store.py +39 -0
- distllm/api/auth_deps.py +68 -0
- distllm/api/authz/__init__.py +30 -0
- distllm/api/authz/opa.py +297 -0
- distllm/api/background_tasks.py +295 -0
- distllm/api/body_cache_middleware.py +117 -0
- distllm/api/circuit_breaker_middleware.py +175 -0
- distllm/api/cost_middleware.py +186 -0
- distllm/api/csrf_middleware.py +196 -0
- distllm/api/dedup.py +180 -0
- distllm/api/errors.py +256 -0
- distllm/api/grpc_bridge.py +296 -0
- distllm/api/ip_utils.py +58 -0
- distllm/api/middleware.py +706 -0
- distllm/api/observability_middleware.py +169 -0
- distllm/api/persistent_store.py +295 -0
- distllm/api/prompt_injection.py +418 -0
- distllm/api/quota_middleware.py +186 -0
- distllm/api/rate_limit_middleware.py +13 -0
- distllm/api/rate_limiter.py +250 -0
- distllm/api/rate_limiter_unified.py +975 -0
- distllm/api/redis_rate_limiter.py +164 -0
- distllm/api/routes/__init__.py +42 -0
- distllm/api/routes/admin.py +599 -0
- distllm/api/routes/api_keys.py +115 -0
- distllm/api/routes/assistants.py +443 -0
- distllm/api/routes/batch.py +1094 -0
- distllm/api/routes/chat.py +875 -0
- distllm/api/routes/completion.py +141 -0
- distllm/api/routes/debug.py +185 -0
- distllm/api/routes/defrag.py +66 -0
- distllm/api/routes/embeddings.py +399 -0
- distllm/api/routes/eval.py +254 -0
- distllm/api/routes/exchange.py +367 -0
- distllm/api/routes/experiments.py +293 -0
- distllm/api/routes/federated.py +155 -0
- distllm/api/routes/files.py +243 -0
- distllm/api/routes/fine_tuning.py +381 -0
- distllm/api/routes/gossip.py +72 -0
- distllm/api/routes/health.py +395 -0
- distllm/api/routes/leaderboard.py +377 -0
- distllm/api/routes/marketplace.py +320 -0
- distllm/api/routes/metrics_history.py +415 -0
- distllm/api/routes/model_registry.py +255 -0
- distllm/api/routes/moderation.py +420 -0
- distllm/api/routes/plugins.py +184 -0
- distllm/api/routes/prompts.py +347 -0
- distllm/api/routes/router_admin.py +187 -0
- distllm/api/routes/scheduler.py +168 -0
- distllm/api/routes/tokenize.py +77 -0
- distllm/api/routes/tools.py +89 -0
- distllm/api/routes/webhooks.py +336 -0
- distllm/api/routes/webrtc.py +194 -0
- distllm/api/semantic_cache.py +337 -0
- distllm/api/server.py +2038 -0
- distllm/api/server_config.py +140 -0
- distllm/api/server_coordinator.py +125 -0
- distllm/api/server_errors.py +79 -0
- distllm/api/server_lifespan.py +184 -0
- distllm/api/server_routes_api.py +456 -0
- distllm/api/server_routes_dashboard.py +246 -0
- distllm/api/server_state.py +29 -0
- distllm/api/services/__init__.py +2 -0
- distllm/api/services/chat_service.py +677 -0
- distllm/api/services/completion_service.py +227 -0
- distllm/api/services/embedding_service.py +451 -0
- distllm/api/services/eval_service.py +198 -0
- distllm/api/sso_auth.py +644 -0
- distllm/api/sso_middleware.py +808 -0
- distllm/api/streaming.py +479 -0
- distllm/api/token_estimator.py +86 -0
- distllm/api/tool_calling.py +240 -0
- distllm/api/tracing_middleware.py +579 -0
- distllm/api/validation.py +58 -0
- distllm/api/waf.py +344 -0
- distllm/api/webhooks/__init__.py +17 -0
- distllm/api/webhooks/delivery.py +452 -0
- distllm/api/ws_chat.py +495 -0
- distllm/backends/__init__.py +121 -0
- distllm/backends/config.py +85 -0
- distllm/backends/exllama_backend.py +229 -0
- distllm/backends/llamacpp_backend.py +266 -0
- distllm/backends/mlx_backend.py +199 -0
- distllm/backends/nim_backend.py +838 -0
- distllm/backends/ollama_backend.py +48 -0
- distllm/backends/onnx_backend.py +239 -0
- distllm/backends/paged_attention.py +547 -0
- distllm/backends/paged_attention_quantized.py +319 -0
- distllm/backends/protocol.py +203 -0
- distllm/backends/pytorch_backend.py +157 -0
- distllm/backends/registry.py +428 -0
- distllm/backends/tensorrt_backend.py +333 -0
- distllm/backends/tgi_backend.py +120 -0
- distllm/backends/triton_backend.py +875 -0
- distllm/backends/vllm_backend.py +489 -0
- distllm/backends/webgpu_backend.py +255 -0
- distllm/benchmarks/__init__.py +10 -0
- distllm/benchmarks/evaluation_harness.py +69 -0
- distllm/benchmarks/scaling.py +298 -0
- distllm/cache/__init__.py +19 -0
- distllm/cache/crdt.py +507 -0
- distllm/cache/cross_cluster_prefix_index.py +709 -0
- distllm/cli/__init__.py +17 -0
- distllm/cli/adapters.py +93 -0
- distllm/cli/autopsy.py +287 -0
- distllm/cli/backup.py +118 -0
- distllm/cli/benchmark.py +241 -0
- distllm/cli/cert.py +86 -0
- distllm/cli/chat.py +79 -0
- distllm/cli/chat_stream.py +494 -0
- distllm/cli/client.py +233 -0
- distllm/cli/cluster.py +405 -0
- distllm/cli/completion.py +79 -0
- distllm/cli/compress.py +98 -0
- distllm/cli/config_commands.py +71 -0
- distllm/cli/cost_avoid.py +168 -0
- distllm/cli/daas_commands.py +123 -0
- distllm/cli/defrag_commands.py +128 -0
- distllm/cli/deploy.py +252 -0
- distllm/cli/doctor.py +688 -0
- distllm/cli/draft_commands.py +82 -0
- distllm/cli/error_handler.py +192 -0
- distllm/cli/eval.py +168 -0
- distllm/cli/federate_commands.py +125 -0
- distllm/cli/install.py +198 -0
- distllm/cli/logs.py +85 -0
- distllm/cli/main.py +1795 -0
- distllm/cli/models.py +74 -0
- distllm/cli/notify.py +91 -0
- distllm/cli/observe.py +314 -0
- distllm/cli/onboard.py +671 -0
- distllm/cli/output.py +437 -0
- distllm/cli/profile.py +152 -0
- distllm/cli/profile_presets.py +129 -0
- distllm/cli/prompts.py +139 -0
- distllm/cli/quota.py +227 -0
- distllm/cli/router.py +131 -0
- distllm/cli/run.py +59 -0
- distllm/cli/setup.py +104 -0
- distllm/cli/setup_wizard.py +191 -0
- distllm/cli/status.py +65 -0
- distllm/cli/system_commands.py +255 -0
- distllm/cli/tune.py +488 -0
- distllm/cli/tutorial.py +128 -0
- distllm/cli/verify.py +37 -0
- distllm/cli/webhook.py +85 -0
- distllm/client/__init__.py +38 -0
- distllm/client/client.py +503 -0
- distllm/cloud/__init__.py +53 -0
- distllm/cloud/aws.py +199 -0
- distllm/cloud/azure.py +179 -0
- distllm/cloud/common.py +217 -0
- distllm/cloud/gcp.py +238 -0
- distllm/cloud/spot_orchestrator.py +1642 -0
- distllm/cloud/worker_agent.py +344 -0
- distllm/compliance/__init__.py +10 -0
- distllm/compliance/evidence_pack.py +699 -0
- distllm/config/__init__.py +81 -0
- distllm/config/_application.py +41 -0
- distllm/config/_backends.py +80 -0
- distllm/config/_cache.py +121 -0
- distllm/config/_deployment.py +95 -0
- distllm/config/_generation.py +36 -0
- distllm/config/_hardware.py +42 -0
- distllm/config/_model.py +270 -0
- distllm/config/_network.py +235 -0
- distllm/config/_observability.py +42 -0
- distllm/config/_parallelism.py +172 -0
- distllm/config/_performance.py +47 -0
- distllm/config/loader.py +168 -0
- distllm/config/model_heuristics.py +169 -0
- distllm/config/profiles.py +168 -0
- distllm/config/reference.py +104 -0
- distllm/config/resolver.py +240 -0
- distllm/config/settings.py +463 -0
- distllm/constants.py +117 -0
- distllm/core/__init__.py +111 -0
- distllm/core/_advanced_scheduling_original.py +1892 -0
- distllm/core/ab_test_coordinator.py +178 -0
- distllm/core/activation_profiler.py +172 -0
- distllm/core/adaptive_batching.py +234 -0
- distllm/core/adaptive_cache_compressor.py +168 -0
- distllm/core/adaptive_compression.py +427 -0
- distllm/core/adaptive_compression_hierarchy.py +402 -0
- distllm/core/advanced_scheduling/__init__.py +39 -0
- distllm/core/advanced_scheduling/cost_aware.py +55 -0
- distllm/core/advanced_scheduling/disaggregated.py +352 -0
- distllm/core/advanced_scheduling/energy.py +58 -0
- distllm/core/advanced_scheduling/federated.py +59 -0
- distllm/core/advanced_scheduling/heterogeneous.py +84 -0
- distllm/core/advanced_scheduling/multi_objective.py +372 -0
- distllm/core/advanced_scheduling/policy.py +95 -0
- distllm/core/advanced_scheduling/predictive.py +47 -0
- distllm/core/advanced_scheduling/preemption.py +43 -0
- distllm/core/advanced_scheduling/tiered_store.py +604 -0
- distllm/core/advanced_scheduling/token_bank.py +62 -0
- distllm/core/advanced_scheduling/wan.py +85 -0
- distllm/core/aegis_compliance.py +890 -0
- distllm/core/aether_federated.py +1326 -0
- distllm/core/agentic_router.py +749 -0
- distllm/core/api_key_store.py +478 -0
- distllm/core/arbitrage_engine.py +613 -0
- distllm/core/aria_autoscaler.py +1339 -0
- distllm/core/async_connection_pool.py +187 -0
- distllm/core/async_pipelined_speculative.py +543 -0
- distllm/core/atlas_mesh.py +1437 -0
- distllm/core/auto_discovery.py +351 -0
- distllm/core/auto_partitioner.py +396 -0
- distllm/core/autonomous_healer.py +466 -0
- distllm/core/autoq.py +1545 -0
- distllm/core/backup_manager.py +344 -0
- distllm/core/bargaining_engine.py +450 -0
- distllm/core/batch_builder.py +480 -0
- distllm/core/batch_scheduler.py +1342 -0
- distllm/core/block_affinity_tracker.py +162 -0
- distllm/core/block_eviction_policy.py +363 -0
- distllm/core/cache_aware_router.py +116 -0
- distllm/core/cache_bench.py +239 -0
- distllm/core/cache_coherence.py +139 -0
- distllm/core/cache_doctor.py +220 -0
- distllm/core/cache_eviction.py +181 -0
- distllm/core/cache_index.py +91 -0
- distllm/core/cache_manager.py +703 -0
- distllm/core/cache_migration.py +134 -0
- distllm/core/cache_persistence.py +200 -0
- distllm/core/cache_query_log.py +195 -0
- distllm/core/cache_snapshot.py +178 -0
- distllm/core/cache_template_warmer.py +112 -0
- distllm/core/cache_warming.py +188 -0
- distllm/core/calibration.py +223 -0
- distllm/core/carbon_migration.py +338 -0
- distllm/core/cert_rotation.py +402 -0
- distllm/core/certificate_manager.py +414 -0
- distllm/core/cluster_manager.py +212 -0
- distllm/core/cluster_predictive_prefetcher.py +239 -0
- distllm/core/cluster_registry.py +227 -0
- distllm/core/cluster_state_store.py +442 -0
- distllm/core/compliance_evidence.py +503 -0
- distllm/core/compressed_speculative.py +399 -0
- distllm/core/connection_pool.py +387 -0
- distllm/core/constrained_decoder.py +678 -0
- distllm/core/coordinator.py +1594 -0
- distllm/core/coordinator_cli.py +130 -0
- distllm/core/coordinator_config.py +203 -0
- distllm/core/coordinator_config_wiring.py +316 -0
- distllm/core/coordinator_election.py +246 -0
- distllm/core/coordinator_failover.py +197 -0
- distllm/core/coordinator_health.py +135 -0
- distllm/core/coordinator_lifecycle.py +272 -0
- distllm/core/coordinator_metrics.py +155 -0
- distllm/core/coordinator_multi_model.py +41 -0
- distllm/core/coordinator_request.py +246 -0
- distllm/core/coordinator_state.py +215 -0
- distllm/core/coordinator_subsystem.py +614 -0
- distllm/core/correctness_cert.py +133 -0
- distllm/core/correctness_harness.py +421 -0
- distllm/core/cortex_multimodel.py +939 -0
- distllm/core/cost_comparison.py +250 -0
- distllm/core/cost_dashboard.py +363 -0
- distllm/core/cost_optimizer.py +317 -0
- distllm/core/cost_tracker.py +569 -0
- distllm/core/cross_cloud_router.py +924 -0
- distllm/core/cross_model_prefix_sharing.py +237 -0
- distllm/core/cuda_graph.py +165 -0
- distllm/core/debug.py +84 -0
- distllm/core/device_registry.py +289 -0
- distllm/core/di.py +153 -0
- distllm/core/differential_privacy.py +183 -0
- distllm/core/distributed_distillation.py +402 -0
- distllm/core/distributed_speculative.py +1563 -0
- distllm/core/dp_inference/__init__.py +657 -0
- distllm/core/dp_inference/accounting.py +144 -0
- distllm/core/dp_inference/config.py +139 -0
- distllm/core/dp_inference/mechanisms.py +137 -0
- distllm/core/dp_inference.py +1016 -0
- distllm/core/draft_model_router.py +435 -0
- distllm/core/draft_quality_scorer.py +157 -0
- distllm/core/draft_tree.py +232 -0
- distllm/core/dynamic_memory_budget.py +179 -0
- distllm/core/dynamic_sharder.py +364 -0
- distllm/core/dynamic_speculation.py +131 -0
- distllm/core/evaluation/__init__.py +57 -0
- distllm/core/evaluation/constants.py +86 -0
- distllm/core/evaluation/db.py +185 -0
- distllm/core/evaluation/formatters.py +82 -0
- distllm/core/evaluation/loaders.py +320 -0
- distllm/core/evaluation/models.py +66 -0
- distllm/core/evaluation/report.py +79 -0
- distllm/core/evaluation/runner.py +518 -0
- distllm/core/evaluation/scorers.py +278 -0
- distllm/core/evaluation/worker.py +123 -0
- distllm/core/evaluation_harness.py +1530 -0
- distllm/core/event_bus.py +516 -0
- distllm/core/faas_7b.py +201 -0
- distllm/core/feature_flags.py +245 -0
- distllm/core/federated_finetuner.py +377 -0
- distllm/core/federated_incentives.py +267 -0
- distllm/core/framework_adapters.py +390 -0
- distllm/core/gaia_cache.py +1330 -0
- distllm/core/gossip_cache_bridge.py +202 -0
- distllm/core/gpu_power_manager.py +173 -0
- distllm/core/gpu_profiler.py +59 -0
- distllm/core/gpu_resource_manager.py +338 -0
- distllm/core/graceful_degradation.py +264 -0
- distllm/core/grammar_constrained.py +549 -0
- distllm/core/grammar_constrained_draft.py +91 -0
- distllm/core/grammar_decoder.py +285 -0
- distllm/core/ha_coordinator.py +452 -0
- distllm/core/hardware_plugin_registry.py +126 -0
- distllm/core/hashing.py +29 -0
- distllm/core/health_manager.py +320 -0
- distllm/core/heterogeneous_scheduler.py +663 -0
- distllm/core/hibernation_manager.py +250 -0
- distllm/core/hierarchical_digest.py +168 -0
- distllm/core/hybrid_cache.py +253 -0
- distllm/core/hydra_diffusion.py +177 -0
- distllm/core/inference_engine.py +1072 -0
- distllm/core/intelligent_autoscaler.py +219 -0
- distllm/core/kraken_chaos.py +1407 -0
- distllm/core/kv_backup.py +217 -0
- distllm/core/kv_cache.py +1271 -0
- distllm/core/kv_cache_adaptive_quantizer.py +168 -0
- distllm/core/kv_cache_compressor.py +290 -0
- distllm/core/kv_cache_manager.py +142 -0
- distllm/core/kv_cache_marketplace.py +319 -0
- distllm/core/kv_cache_metrics.py +205 -0
- distllm/core/kv_cache_migration.py +137 -0
- distllm/core/kv_cache_paged.py +78 -0
- distllm/core/kv_cache_replication.py +107 -0
- distllm/core/kv_cache_serialization.py +182 -0
- distllm/core/latency_tracker.py +17 -0
- distllm/core/leaky_bucket_limiter.py +194 -0
- distllm/core/learning_router.py +652 -0
- distllm/core/llamacpp_backend.py +23 -0
- distllm/core/load_balancer.py +286 -0
- distllm/core/media_pipeline.py +387 -0
- distllm/core/memory_defragmenter.py +666 -0
- distllm/core/metering.py +449 -0
- distllm/core/metrics_collector.py +39 -0
- distllm/core/micro_batch_scheduler.py +170 -0
- distllm/core/model_hub_enhanced.py +382 -0
- distllm/core/model_registry.py +9 -0
- distllm/core/model_router.py +930 -0
- distllm/core/model_selector.py +245 -0
- distllm/core/model_sizing.py +334 -0
- distllm/core/model_version_manager.py +414 -0
- distllm/core/moe_orchestrator.py +205 -0
- distllm/core/money.py +53 -0
- distllm/core/monitor.py +141 -0
- distllm/core/mtp_head.py +444 -0
- distllm/core/multi_draft_verifier.py +346 -0
- distllm/core/multi_model_serving.py +624 -0
- distllm/core/multimodal_engine.py +194 -0
- distllm/core/neural_partition_optimizer.py +1398 -0
- distllm/core/node_recovery.py +27 -0
- distllm/core/notification_manager.py +317 -0
- distllm/core/param_update_channel.py +104 -0
- distllm/core/pareto_optimizer.py +227 -0
- distllm/core/performance_alerts.py +198 -0
- distllm/core/performance_baseline.py +252 -0
- distllm/core/persistence.py +947 -0
- distllm/core/persistent_store.py +485 -0
- distllm/core/pipeline_composer.py +211 -0
- distllm/core/pipeline_executor.py +500 -0
- distllm/core/pipeline_orchestrator.py +17 -0
- distllm/core/pipeline_overlap.py +177 -0
- distllm/core/placement.py +442 -0
- distllm/core/plugin_marketplace.py +483 -0
- distllm/core/plugin_sandbox.py +751 -0
- distllm/core/plugin_system.py +601 -0
- distllm/core/plugins/__init__.py +26 -0
- distllm/core/plugins/sandbox.py +38 -0
- distllm/core/predictive_cache.py +29 -0
- distllm/core/predictive_cache_warming.py +323 -0
- distllm/core/predictive_failure.py +163 -0
- distllm/core/preemptible_scheduler.py +343 -0
- distllm/core/preference_learning.py +606 -0
- distllm/core/prefix_cache.py +17 -0
- distllm/core/pricing_providers.py +576 -0
- distllm/core/priority_heap.py +62 -0
- distllm/core/privacy_budget.py +177 -0
- distllm/core/prompt_caching_service.py +228 -0
- distllm/core/prompt_library.py +838 -0
- distllm/core/protocols.py +213 -0
- distllm/core/provider_health.py +234 -0
- distllm/core/provisioning.py +329 -0
- distllm/core/pulse_performance_model.py +1558 -0
- distllm/core/quantization_selector.py +210 -0
- distllm/core/radix_tree_cache.py +306 -0
- distllm/core/rebalancer.py +25 -0
- distllm/core/redis_prompt_cache.py +311 -0
- distllm/core/replication_controller.py +128 -0
- distllm/core/request_auditor.py +215 -0
- distllm/core/request_fingerprinting.py +231 -0
- distllm/core/request_latency.py +211 -0
- distllm/core/request_pipeline.py +810 -0
- distllm/core/request_replay.py +253 -0
- distllm/core/request_tracker.py +184 -0
- distllm/core/resource_manager.py +657 -0
- distllm/core/route_audit.py +175 -0
- distllm/core/routing_extensions.py +604 -0
- distllm/core/schedule_simulator.py +289 -0
- distllm/core/schedule_viz.py +233 -0
- distllm/core/scheduler/__init__.py +34 -0
- distllm/core/scheduler/budget.py +25 -0
- distllm/core/scheduler/budget_computer.py +322 -0
- distllm/core/scheduler/chunked_prefill.py +52 -0
- distllm/core/scheduler/kv_cache_manager.py +313 -0
- distllm/core/scheduler/preemption_manager.py +251 -0
- distllm/core/scheduler/pressure.py +48 -0
- distllm/core/scheduler/sequence.py +179 -0
- distllm/core/secret_manager.py +315 -0
- distllm/core/semantic_cache.py +369 -0
- distllm/core/sentinel_qos.py +833 -0
- distllm/core/shadow_eval_runner.py +194 -0
- distllm/core/shared_layer_pool.py +279 -0
- distllm/core/smart_model_router.py +309 -0
- distllm/core/spec_equivalence.py +140 -0
- distllm/core/spec_verify.py +225 -0
- distllm/core/speculative_adaptor.py +108 -0
- distllm/core/speculative_dashboard.py +134 -0
- distllm/core/speculative_decoder.py +1286 -0
- distllm/core/speculative_profiler.py +132 -0
- distllm/core/split_brain.py +236 -0
- distllm/core/spot_failover.py +292 -0
- distllm/core/spot_forecasting.py +463 -0
- distllm/core/starvation_monitor.py +70 -0
- distllm/core/state_replication.py +261 -0
- distllm/core/stats_collector.py +64 -0
- distllm/core/step_processor.py +115 -0
- distllm/core/straggler_alerts.py +174 -0
- distllm/core/straggler_aware_scheduler.py +255 -0
- distllm/core/straggler_detector.py +29 -0
- distllm/core/streaming_cost.py +286 -0
- distllm/core/streaming_generator.py +228 -0
- distllm/core/structured_output/__init__.py +348 -0
- distllm/core/structured_output/config.py +38 -0
- distllm/core/structured_output/engine.py +367 -0
- distllm/core/structured_output/streaming.py +183 -0
- distllm/core/structured_output/validator.py +199 -0
- distllm/core/subsystem_registry.py +171 -0
- distllm/core/synapse_debugger.py +226 -0
- distllm/core/synth_data_generator.py +122 -0
- distllm/core/telemetry.py +240 -0
- distllm/core/tenant_billing.py +446 -0
- distllm/core/tenant_cost_attribution.py +314 -0
- distllm/core/token_generator.py +368 -0
- distllm/core/token_streaming_buffer.py +147 -0
- distllm/core/topology_aware_lb.py +324 -0
- distllm/core/topology_aware_tiering.py +344 -0
- distllm/core/tree_speculative_decoder.py +405 -0
- distllm/core/types.py +17 -0
- distllm/core/unified_router.py +439 -0
- distllm/core/unified_sla_router.py +242 -0
- distllm/core/usage_meter.py +915 -0
- distllm/core/vectorstore/__init__.py +30 -0
- distllm/core/vectorstore/base.py +106 -0
- distllm/core/vectorstore/chroma_store.py +99 -0
- distllm/core/vectorstore/factory.py +97 -0
- distllm/core/vectorstore/pgvector_store.py +123 -0
- distllm/core/vectorstore/providers/__init__.py +7 -0
- distllm/core/vectorstore/providers/milvus.py +219 -0
- distllm/core/vectorstore/providers/pinecone.py +142 -0
- distllm/core/vectorstore/providers/qdrant.py +176 -0
- distllm/core/vectorstore/providers/weaviate.py +185 -0
- distllm/core/vectorstore/qdrant_store.py +148 -0
- distllm/core/vectorstore/rag.py +151 -0
- distllm/core/vllm_backend.py +21 -0
- distllm/core/vllm_node.py +174 -0
- distllm/core/voyager_multimodal.py +1354 -0
- distllm/core/webgpu_manager.py +382 -0
- distllm/core/webhook_formatters.py +243 -0
- distllm/core/webhook_manager.py +433 -0
- distllm/core/wisp_wasm.py +934 -0
- distllm/core/workload_classifier.py +137 -0
- distllm/dashboard/__init__.py +17 -0
- distllm/dashboard/verification_history.py +192 -0
- distllm/dashboard/ws_handler.py +586 -0
- distllm/dist/__init__.py +85 -0
- distllm/dist/__init__.pyi +132 -0
- distllm/dist/_pipeline_original.py +2916 -0
- distllm/dist/adaptive_speculator.py +384 -0
- distllm/dist/admin_cli.py +293 -0
- distllm/dist/api_docs.py +594 -0
- distllm/dist/async_pipeline.py +578 -0
- distllm/dist/attention.py +1699 -0
- distllm/dist/attention_block_pool.py +634 -0
- distllm/dist/attention_distributed.py +138 -0
- distllm/dist/autoscaler.py +196 -0
- distllm/dist/backends/__init__.py +29 -0
- distllm/dist/backends/backend_profiles.py +357 -0
- distllm/dist/backends/graceful_degradation.py +409 -0
- distllm/dist/backends/health_monitor.py +518 -0
- distllm/dist/backends/llamacpp.py +125 -0
- distllm/dist/backends/ray.py +89 -0
- distllm/dist/backends/vllm.py +177 -0
- distllm/dist/block_pool.py +662 -0
- distllm/dist/block_transfer_service.py +267 -0
- distllm/dist/byzantine.py +2193 -0
- distllm/dist/cache.py +305 -0
- distllm/dist/cache_digest.py +405 -0
- distllm/dist/chaos/__init__.py +52 -0
- distllm/dist/chaos/chaos_orchestrator.py +229 -0
- distllm/dist/chaos/fault_injector.py +1273 -0
- distllm/dist/chunked_prefill.py +136 -0
- distllm/dist/cloud_selector.py +257 -0
- distllm/dist/config.py +70 -0
- distllm/dist/cross_cluster.py +231 -0
- distllm/dist/daas/__init__.py +46 -0
- distllm/dist/daas/load_balancer.py +275 -0
- distllm/dist/daas/marketplace_integration.py +374 -0
- distllm/dist/daas/resource_isolation.py +280 -0
- distllm/dist/daas/tenant_dispatcher.py +252 -0
- distllm/dist/daas/usage_meter.py +360 -0
- distllm/dist/daas_server.py +514 -0
- distllm/dist/disagg/__init__.py +162 -0
- distllm/dist/disagg/pool.py +214 -0
- distllm/dist/disagg/transfer.py +77 -0
- distllm/dist/disagg/transfer_compression.py +793 -0
- distllm/dist/disagg/transfer_engine.py +998 -0
- distllm/dist/discovery.py +166 -0
- distllm/dist/draft_bank.py +344 -0
- distllm/dist/draft_migration.py +382 -0
- distllm/dist/ebpf_transport.py +364 -0
- distllm/dist/edge_cloud.py +544 -0
- distllm/dist/edge_federation.py +426 -0
- distllm/dist/execution_planner.py +148 -0
- distllm/dist/federated_merge.py +560 -0
- distllm/dist/federation.py +1114 -0
- distllm/dist/flash_attention.py +237 -0
- distllm/dist/fsdp.py +295 -0
- distllm/dist/geo.py +361 -0
- distllm/dist/ice_transport.py +2968 -0
- distllm/dist/kv_migration.py +237 -0
- distllm/dist/latency.py +60 -0
- distllm/dist/lifecycle.py +467 -0
- distllm/dist/marketplace.py +891 -0
- distllm/dist/merkle.py +178 -0
- distllm/dist/metrics.py +456 -0
- distllm/dist/model_store.py +107 -0
- distllm/dist/multi_tenant.py +339 -0
- distllm/dist/multimodal.py +392 -0
- distllm/dist/nat.py +934 -0
- distllm/dist/nccl.py +733 -0
- distllm/dist/network.py +126 -0
- distllm/dist/node_client.py +568 -0
- distllm/dist/node_pb2.py +66 -0
- distllm/dist/node_pb2.pyi +236 -0
- distllm/dist/node_pb2_grpc.py +312 -0
- distllm/dist/node_pb2_grpc.pyi +41 -0
- distllm/dist/node_registrar.py +247 -0
- distllm/dist/node_service.py +572 -0
- distllm/dist/observability.py +258 -0
- distllm/dist/otel.py +158 -0
- distllm/dist/p2p/__init__.py +22 -0
- distllm/dist/p2p/discovery.py +132 -0
- distllm/dist/p2p/gossip.py +1339 -0
- distllm/dist/p2p/kademlia_dht.py +1058 -0
- distllm/dist/p2p/load_balancer.py +131 -0
- distllm/dist/p2p/quic_transport.py +700 -0
- distllm/dist/p2p/router.py +607 -0
- distllm/dist/p2p/transport.py +399 -0
- distllm/dist/p2p_model_distributor.py +160 -0
- distllm/dist/paged_attention_kernel.py +281 -0
- distllm/dist/parallel.py +1061 -0
- distllm/dist/parallel_planner.py +586 -0
- distllm/dist/parallel_topology.py +231 -0
- distllm/dist/partition/__init__.py +146 -0
- distllm/dist/partition/adaptive.py +532 -0
- distllm/dist/partition/benchmark_suite.py +479 -0
- distllm/dist/partition/cli.py +189 -0
- distllm/dist/partition/cloud_arbitrage.py +523 -0
- distllm/dist/partition/config.py +97 -0
- distllm/dist/partition/cost_model.py +553 -0
- distllm/dist/partition/learned_cost.py +653 -0
- distllm/dist/partition/mixed_precision_tuner.py +310 -0
- distllm/dist/partition/network_cost_model.py +664 -0
- distllm/dist/partition/optimizer.py +650 -0
- distllm/dist/partition/pareto_optimizer.py +466 -0
- distllm/dist/partition/partitioner.py +363 -0
- distllm/dist/partition/persistence.py +392 -0
- distllm/dist/partition/plugins.py +270 -0
- distllm/dist/partition/profiles.py +659 -0
- distllm/dist/partition/quant_bench.py +373 -0
- distllm/dist/partition/quant_calibrate.py +229 -0
- distllm/dist/partition/quant_coordinator.py +260 -0
- distllm/dist/partition/quant_cost.py +293 -0
- distllm/dist/partition/quant_partition.py +432 -0
- distllm/dist/partition/quant_report.py +301 -0
- distllm/dist/partition/quantization_metrics.py +355 -0
- distllm/dist/partition/quantization_search.py +290 -0
- distllm/dist/partition/quantization_tuner.py +1383 -0
- distllm/dist/partition/topology.py +218 -0
- distllm/dist/partition/validator.py +405 -0
- distllm/dist/partition/visualizer.py +275 -0
- distllm/dist/pipeline/__init__.py +71 -0
- distllm/dist/pipeline/bandwidth_controller.py +780 -0
- distllm/dist/pipeline/compression_negotiation.py +771 -0
- distllm/dist/pipeline/context.py +80 -0
- distllm/dist/pipeline/continuous_batching.py +435 -0
- distllm/dist/pipeline/disagg_orchestrator.py +284 -0
- distllm/dist/pipeline/orchestrator.py +727 -0
- distllm/dist/pipeline/pipeline_reconfig.py +1395 -0
- distllm/dist/pipeline/profiler.py +84 -0
- distllm/dist/pipeline/serialization.py +176 -0
- distllm/dist/pipeline/simulator.py +172 -0
- distllm/dist/pipeline/strategy.py +193 -0
- distllm/dist/pipeline/token_accumulator.py +211 -0
- distllm/dist/pipeline/transport.py +230 -0
- distllm/dist/pipeline/wan_disagg_orchestrator.py +564 -0
- distllm/dist/power_cap.py +358 -0
- distllm/dist/predictive_cache.py +399 -0
- distllm/dist/preemption.py +198 -0
- distllm/dist/prefix_cache.py +461 -0
- distllm/dist/prefix_clustering.py +305 -0
- distllm/dist/privacy/__init__.py +411 -0
- distllm/dist/privacy/privacy_accountant.py +344 -0
- distllm/dist/privacy.py +240 -0
- distllm/dist/provisioning.py +338 -0
- distllm/dist/quality.py +118 -0
- distllm/dist/quic_transport.py +335 -0
- distllm/dist/quota_enforcer.py +265 -0
- distllm/dist/rebalancer.py +234 -0
- distllm/dist/recovery.py +827 -0
- distllm/dist/recovery_drill.py +281 -0
- distllm/dist/redis_cache.py +265 -0
- distllm/dist/redundant.py +308 -0
- distllm/dist/redundant_executor.py +1536 -0
- distllm/dist/reputation.py +231 -0
- distllm/dist/router.py +382 -0
- distllm/dist/routing/__init__.py +19 -0
- distllm/dist/routing/arbitrage_engine.py +575 -0
- distllm/dist/routing/composite.py +179 -0
- distllm/dist/routing/consistent_hash.py +144 -0
- distllm/dist/routing/latency_aware.py +116 -0
- distllm/dist/routing/load_aware.py +114 -0
- distllm/dist/scheduling/__init__.py +38 -0
- distllm/dist/scheduling/batcher.py +95 -0
- distllm/dist/scheduling/classifier.py +153 -0
- distllm/dist/scheduling/deadline_scheduler.py +958 -0
- distllm/dist/scheduling/iteration.py +404 -0
- distllm/dist/scheduling/profiler.py +57 -0
- distllm/dist/simulation/__init__.py +54 -0
- distllm/dist/simulation/chaos_simulator.py +901 -0
- distllm/dist/simulation/cluster_simulator.py +667 -0
- distllm/dist/simulation/cost_aware_provisioning.py +370 -0
- distllm/dist/simulation/digital_twin.py +617 -0
- distllm/dist/simulation/topology_optimizer.py +747 -0
- distllm/dist/simulation/what_if.py +492 -0
- distllm/dist/speculative/__init__.py +47 -0
- distllm/dist/speculative/adaptive_spec.py +291 -0
- distllm/dist/speculative/draft_cache.py +246 -0
- distllm/dist/speculative/draft_orchestrator.py +634 -0
- distllm/dist/speculative/draft_registry.py +439 -0
- distllm/dist/speculative/multi_draft.py +304 -0
- distllm/dist/speculative/online_sisd.py +523 -0
- distllm/dist/straggler.py +819 -0
- distllm/dist/streaming_kv_transfer.py +174 -0
- distllm/dist/structured_output/__init__.py +7 -0
- distllm/dist/structured_output/engine.py +144 -0
- distllm/dist/tensor_pool.py +120 -0
- distllm/dist/tgi_compat.py +254 -0
- distllm/dist/topology.py +141 -0
- distllm/dist/topology_consensus.py +1961 -0
- distllm/dist/topology_dynamic.py +270 -0
- distllm/dist/tp_inprocess.py +214 -0
- distllm/dist/tp_launcher.py +310 -0
- distllm/dist/tracing.py +325 -0
- distllm/dist/transport.py +493 -0
- distllm/dist/wan_speculative.py +295 -0
- distllm/dist/wandb_integration.py +252 -0
- distllm/dist/webrtc.py +846 -0
- distllm/dist/wide_area.py +515 -0
- distllm/dist/worker.py +635 -0
- distllm/dist/zero_copy.py +307 -0
- distllm/errors/__init__.py +105 -0
- distllm/errors/policies.py +106 -0
- distllm/errors/retry.py +168 -0
- distllm/errors/types.py +721 -0
- distllm/health/__init__.py +6 -0
- distllm/health/failover.py +81 -0
- distllm/health/prober.py +31 -0
- distllm/health/service.py +173 -0
- distllm/health/state.py +76 -0
- distllm/integrations/__init__.py +72 -0
- distllm/integrations/_common/__init__.py +7 -0
- distllm/integrations/_common/base_tool_provider.py +189 -0
- distllm/integrations/_common/cost_tracker.py +144 -0
- distllm/integrations/_common/model_router.py +148 -0
- distllm/integrations/ci/__init__.py +24 -0
- distllm/integrations/ci/_common.py +92 -0
- distllm/integrations/ci/gitlab.py +345 -0
- distllm/integrations/ci/jenkins.py +273 -0
- distllm/integrations/mlflow_tracking.py +372 -0
- distllm/integrations/spark_connector.py +909 -0
- distllm/integrations/wandb/__init__.py +81 -0
- distllm/models/__init__.py +40 -0
- distllm/models/_trust_remote.py +48 -0
- distllm/models/adapter.py +768 -0
- distllm/models/adapter_router.py +198 -0
- distllm/models/cache.py +172 -0
- distllm/models/model_hub.py +650 -0
- distllm/models/partition_planner.py +384 -0
- distllm/models/partitioner.py +784 -0
- distllm/models/rope_scaling.py +119 -0
- distllm/models/safetensors_index.py +179 -0
- distllm/observability/__init__.py +107 -0
- distllm/observability/alerting_config.py +881 -0
- distllm/observability/capacity_models.py +398 -0
- distllm/observability/capacity_planning.py +2870 -0
- distllm/observability/exporter.py +228 -0
- distllm/observability/exporter_datadog.py +1362 -0
- distllm/observability/health_config.py +1282 -0
- distllm/observability/incident_response.py +779 -0
- distllm/observability/logging.py +211 -0
- distllm/observability/logging_config.py +864 -0
- distllm/observability/loki_sink.py +178 -0
- distllm/observability/metrics.py +111 -0
- distllm/observability/metrics_completeness.py +393 -0
- distllm/observability/otel_logging.py +290 -0
- distllm/observability/self_healing_config.py +1308 -0
- distllm/observability/slo_config.py +1520 -0
- distllm/observability/spans.py +235 -0
- distllm/observability/tracing.py +268 -0
- distllm/observability/tracing_config.py +676 -0
- distllm/observability/wandb_monitor.py +66 -0
- distllm/plugins/__init__.py +14 -0
- distllm/plugins/airflow.py +110 -0
- distllm/plugins/auth_plugin.py +576 -0
- distllm/plugins/builtin.py +376 -0
- distllm/plugins/cache_plugin.py +576 -0
- distllm/plugins/health_plugin.py +396 -0
- distllm/plugins/kubeflow.py +84 -0
- distllm/plugins/mlflow_plugin.py +71 -0
- distllm/plugins/registry.py +135 -0
- distllm/prompts/__init__.py +39 -0
- distllm/prompts/analysis.py +103 -0
- distllm/prompts/code.py +172 -0
- distllm/prompts/creative.py +115 -0
- distllm/prompts/education.py +91 -0
- distllm/prompts/engine.py +119 -0
- distllm/prompts/exchange.py +705 -0
- distllm/prompts/language.py +81 -0
- distllm/prompts/library.py +1198 -0
- distllm/prompts/management.py +35 -0
- distllm/prompts/productivity.py +86 -0
- distllm/prompts/professional.py +111 -0
- distllm/prompts/prompt_def.py +33 -0
- distllm/prompts/reasoning.py +108 -0
- distllm/prompts/specialized.py +140 -0
- distllm/prompts/templates.py +128 -0
- distllm/prompts/writing.py +140 -0
- distllm/py.typed +0 -0
- distllm/sdk/__init__.py +82 -0
- distllm/sdk/circuit_breaker.py +139 -0
- distllm/sdk/client.py +1191 -0
- distllm/sdk/observability.py +42 -0
- distllm/sdk/streaming.py +228 -0
- distllm/sdk/transport.py +224 -0
- distllm/sdk/types.py +288 -0
- distllm/security/__init__.py +63 -0
- distllm/security/content_moderation/__init__.py +44 -0
- distllm/security/content_moderation/base.py +405 -0
- distllm/security/content_moderation/jailbreak.py +192 -0
- distllm/security/content_moderation/pii.py +262 -0
- distllm/security/content_moderation/pipeline.py +272 -0
- distllm/security/content_moderation/topics.py +153 -0
- distllm/security/content_moderation/toxicity.py +131 -0
- distllm/security/e2e.py +437 -0
- distllm/security/edge_attestation.py +381 -0
- distllm/security/log_redaction.py +244 -0
- distllm/security/quantum_safe_tls.py +314 -0
- distllm/security/spiffe.py +347 -0
- distllm/security/tee.py +461 -0
- distllm/security/utils.py +139 -0
- distllm/security/watermark.py +779 -0
- distllm/terraform/__init__.py +260 -0
- distllm/ui/__init__.py +7 -0
- distllm/ui/app.py +98 -0
- distllm/utils/__init__.py +0 -0
- distllm/utils/gbnf_grammar.py +99 -0
- distllm/utils/lazy_imports.py +90 -0
- distllm/utils/scheduling.py +38 -0
- distllm/verification/__init__.py +49 -0
- distllm/verification/comparator.py +263 -0
- distllm/verification/hash_registry.py +220 -0
- distllm/verification/report.py +240 -0
- distllm/verification/runner.py +822 -0
- distllm/verification/test_verify.py +205 -0
- distributed_llm-0.4.1.dist-info/METADATA +315 -0
- distributed_llm-0.4.1.dist-info/RECORD +806 -0
- distributed_llm-0.4.1.dist-info/WHEEL +5 -0
- distributed_llm-0.4.1.dist-info/entry_points.txt +5 -0
- distributed_llm-0.4.1.dist-info/licenses/LICENSE +97 -0
- distributed_llm-0.4.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,984 @@
|
|
|
1
|
+
"""API versioning: Accept-Version header negotiation, deprecation scheduling,
|
|
2
|
+
field-level compatibility mapping, and a middleware that enforces the
|
|
3
|
+
version contract at the HTTP layer.
|
|
4
|
+
|
|
5
|
+
Architecture
|
|
6
|
+
------------
|
|
7
|
+
The versioning system has four components that work together:
|
|
8
|
+
|
|
9
|
+
1. **APIVersion** (enum) — Single source of truth for supported API versions.
|
|
10
|
+
Versions are date-based strings (e.g. ``"2026-01"``) following the
|
|
11
|
+
``YYYY-MM`` convention, not ordinal integers.
|
|
12
|
+
|
|
13
|
+
2. **VersionNegotiator** — Determines which API version a request targets by
|
|
14
|
+
checking, in order:
|
|
15
|
+
a. The ``Accept-Version`` HTTP header
|
|
16
|
+
b. The URL path prefix (``/v1/``, ``/v2/``)
|
|
17
|
+
c. A configured default version (fallback)
|
|
18
|
+
Returns the resolved version plus optional deprecation warnings.
|
|
19
|
+
|
|
20
|
+
3. **DeprecationSchedule** — Tracks sunset dates per version and provides
|
|
21
|
+
migration path descriptions so clients know what to change.
|
|
22
|
+
|
|
23
|
+
4. **VersionCompatibilityLayer** — Bidirectional field-level mapping between
|
|
24
|
+
versioned request/response shapes. Allows the server to accept old-format
|
|
25
|
+
payloads and return old-format responses even when the internal handler
|
|
26
|
+
has moved to a newer schema.
|
|
27
|
+
|
|
28
|
+
5. **VersionMiddleware** (Starlette ``BaseHTTPMiddleware``) — Wires the
|
|
29
|
+
negotiator, schedule, and compatibility layer into the request pipeline:
|
|
30
|
+
- Extracts the version from every incoming request and stores it on
|
|
31
|
+
``request.state.api_version``.
|
|
32
|
+
- Adds ``X-API-Version``, ``Sunset``, and ``X-API-Deprecation`` headers
|
|
33
|
+
to every response.
|
|
34
|
+
- Returns **410 Gone** for requests targeting a fully deprecated version
|
|
35
|
+
whose sunset date has passed.
|
|
36
|
+
- Optionally transforms request/response bodies through the compatibility
|
|
37
|
+
layer when the request version does not match the handler's internal
|
|
38
|
+
version.
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
from __future__ import annotations
|
|
42
|
+
|
|
43
|
+
import datetime
|
|
44
|
+
import re
|
|
45
|
+
from dataclasses import dataclass, field
|
|
46
|
+
from enum import Enum
|
|
47
|
+
from typing import Any, Callable
|
|
48
|
+
|
|
49
|
+
from fastapi import Request, Response
|
|
50
|
+
from loguru import logger
|
|
51
|
+
from starlette.middleware.base import BaseHTTPMiddleware
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
# ── Constants ─────────────────────────────────────────────────────────────────
|
|
55
|
+
|
|
56
|
+
SUNSET_HEADER = "Sunset"
|
|
57
|
+
DEPRECATION_HEADER = "X-API-Deprecation"
|
|
58
|
+
VERSION_HEADER = "X-API-Version"
|
|
59
|
+
ACCEPT_VERSION_HEADER = "Accept-Version"
|
|
60
|
+
|
|
61
|
+
# Default version used when no version indication is present.
|
|
62
|
+
DEFAULT_API_VERSION = "2026-07"
|
|
63
|
+
|
|
64
|
+
# Warning threshold: a version is considered "near sunset" when fewer than
|
|
65
|
+
# this many days remain before its sunset date.
|
|
66
|
+
NEAR_SUNSET_DAYS = 60
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
# ── Version Enum ──────────────────────────────────────────────────────────────
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class APIVersion(str, Enum):
|
|
73
|
+
"""Canonical API version identifiers (date-based).
|
|
74
|
+
|
|
75
|
+
Values are ``YYYY-MM`` strings so that ordering is lexicographically
|
|
76
|
+
equivalent to chronological ordering.
|
|
77
|
+
|
|
78
|
+
Add a new member each time the API surface changes in a way that is not
|
|
79
|
+
backward-compatible.
|
|
80
|
+
"""
|
|
81
|
+
|
|
82
|
+
V1 = "2026-01"
|
|
83
|
+
V2 = "2026-07"
|
|
84
|
+
|
|
85
|
+
@classmethod
|
|
86
|
+
def from_string(cls, value: str) -> APIVersion | None:
|
|
87
|
+
"""Parse a string into an ``APIVersion`` member, returning ``None`` on
|
|
88
|
+
mismatch."""
|
|
89
|
+
value = value.strip()
|
|
90
|
+
for member in cls:
|
|
91
|
+
if member.value == value:
|
|
92
|
+
return member
|
|
93
|
+
return None
|
|
94
|
+
|
|
95
|
+
@classmethod
|
|
96
|
+
def from_path_prefix(cls, path: str) -> APIVersion | None:
|
|
97
|
+
"""Extract an API version from a URL path like ``/v1/...`` or ``/v2/...``.
|
|
98
|
+
|
|
99
|
+
Returns ``None`` when the path carries no version prefix (e.g. unversioned
|
|
100
|
+
endpoints such as ``/health`` or ``/metrics``).
|
|
101
|
+
"""
|
|
102
|
+
match = re.match(r"^/(v\d+)/", path)
|
|
103
|
+
if match is None:
|
|
104
|
+
return None
|
|
105
|
+
prefix = match.group(1)
|
|
106
|
+
mapping: dict[str, APIVersion] = {
|
|
107
|
+
"v1": APIVersion.V1,
|
|
108
|
+
"v2": APIVersion.V2,
|
|
109
|
+
}
|
|
110
|
+
return mapping.get(prefix)
|
|
111
|
+
|
|
112
|
+
def to_path_prefix(self) -> str:
|
|
113
|
+
"""Return the URL path prefix for this version (e.g. ``/v1``)."""
|
|
114
|
+
ordinal: dict[APIVersion, str] = {
|
|
115
|
+
APIVersion.V1: "v1",
|
|
116
|
+
APIVersion.V2: "v2",
|
|
117
|
+
}
|
|
118
|
+
return f"/{ordinal[self]}"
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
# ── Version Negotiation Result ────────────────────────────────────────────────
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
@dataclass(frozen=True)
|
|
125
|
+
class VersionNegotiationResult:
|
|
126
|
+
"""The outcome of version negotiation for a single request.
|
|
127
|
+
|
|
128
|
+
Attributes
|
|
129
|
+
----------
|
|
130
|
+
version:
|
|
131
|
+
The resolved ``APIVersion``, or ``None`` when the request could not
|
|
132
|
+
be matched (caller should reject).
|
|
133
|
+
source:
|
|
134
|
+
How the version was determined: ``"header"``, ``"path"``, or
|
|
135
|
+
``"default"``.
|
|
136
|
+
warnings:
|
|
137
|
+
Human-readable deprecation warnings to emit as response headers.
|
|
138
|
+
"""
|
|
139
|
+
|
|
140
|
+
version: APIVersion | None
|
|
141
|
+
source: str
|
|
142
|
+
warnings: list[str] = field(default_factory=list)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
# ── Version Negotiator ────────────────────────────────────────────────────────
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
class VersionNegotiator:
|
|
149
|
+
"""Resolves the API version for an incoming request.
|
|
150
|
+
|
|
151
|
+
Negotiation order:
|
|
152
|
+
|
|
153
|
+
1. **Accept-Version header** — exact match against ``APIVersion`` values.
|
|
154
|
+
2. **URL path prefix** — ``/v1/``, ``/v2/``.
|
|
155
|
+
3. **Default** — configured fallback version.
|
|
156
|
+
|
|
157
|
+
Parameters
|
|
158
|
+
----------
|
|
159
|
+
default_version:
|
|
160
|
+
Fallback version when no version indication is present.
|
|
161
|
+
"""
|
|
162
|
+
|
|
163
|
+
def __init__(self, default_version: str = DEFAULT_API_VERSION) -> None:
|
|
164
|
+
self._default = APIVersion.from_string(default_version)
|
|
165
|
+
if self._default is None:
|
|
166
|
+
logger.warning(
|
|
167
|
+
"Default version %r is not a known APIVersion; falling back to %s",
|
|
168
|
+
default_version,
|
|
169
|
+
DEFAULT_API_VERSION,
|
|
170
|
+
)
|
|
171
|
+
self._default = APIVersion.from_string(DEFAULT_API_VERSION)
|
|
172
|
+
assert self._default is not None
|
|
173
|
+
|
|
174
|
+
# ── Public API ────────────────────────────────────────────────────────
|
|
175
|
+
|
|
176
|
+
def negotiate(self, request: Request) -> VersionNegotiationResult:
|
|
177
|
+
"""Determine the API version for *request*.
|
|
178
|
+
|
|
179
|
+
Returns a frozen ``VersionNegotiationResult``. When the result's
|
|
180
|
+
``version`` is ``None`` the version could not be determined and the
|
|
181
|
+
caller should respond with a client error.
|
|
182
|
+
"""
|
|
183
|
+
# 1. Accept-Version header
|
|
184
|
+
header_val = request.headers.get(ACCEPT_VERSION_HEADER)
|
|
185
|
+
if header_val:
|
|
186
|
+
parsed = self._parse_accept_version(header_val)
|
|
187
|
+
if parsed is not None:
|
|
188
|
+
return VersionNegotiationResult(
|
|
189
|
+
version=parsed,
|
|
190
|
+
source="header",
|
|
191
|
+
warnings=[],
|
|
192
|
+
)
|
|
193
|
+
# Header present but unparseable — warn but continue to path.
|
|
194
|
+
logger.debug("Unparseable Accept-Version header: %r", header_val)
|
|
195
|
+
|
|
196
|
+
# 2. URL path prefix
|
|
197
|
+
path = request.url.path
|
|
198
|
+
path_version = APIVersion.from_path_prefix(path)
|
|
199
|
+
if path_version is not None:
|
|
200
|
+
return VersionNegotiationResult(
|
|
201
|
+
version=path_version,
|
|
202
|
+
source="path",
|
|
203
|
+
warnings=[],
|
|
204
|
+
)
|
|
205
|
+
|
|
206
|
+
# 3. Default fallback
|
|
207
|
+
return VersionNegotiationResult(
|
|
208
|
+
version=self._default,
|
|
209
|
+
source="default",
|
|
210
|
+
warnings=[],
|
|
211
|
+
)
|
|
212
|
+
|
|
213
|
+
# ── Internals ─────────────────────────────────────────────────────────
|
|
214
|
+
|
|
215
|
+
@staticmethod
|
|
216
|
+
def _parse_accept_version(header_value: str) -> APIVersion | None:
|
|
217
|
+
"""Try to parse a raw ``Accept-Version`` header into an ``APIVersion``.
|
|
218
|
+
|
|
219
|
+
The header may be a bare date (``2026-01``) or a qualified form
|
|
220
|
+
(``api=2026-01``). Returns ``None`` when no member matches.
|
|
221
|
+
"""
|
|
222
|
+
cleaned = header_value.strip()
|
|
223
|
+
# Accept \"api=2026-01\" or \"2026-01\"
|
|
224
|
+
for sep in ("api=", "version=", ""):
|
|
225
|
+
candidate = cleaned.removeprefix(sep)
|
|
226
|
+
candidate = candidate.strip()
|
|
227
|
+
parsed = APIVersion.from_string(candidate)
|
|
228
|
+
if parsed is not None:
|
|
229
|
+
return parsed
|
|
230
|
+
return None
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
# ── Deprecation Schedule ──────────────────────────────────────────────────────
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
@dataclass(frozen=True)
|
|
237
|
+
class MigrationStep:
|
|
238
|
+
"""A single action a client must take to migrate from one version to another.
|
|
239
|
+
|
|
240
|
+
Attributes
|
|
241
|
+
----------
|
|
242
|
+
description:
|
|
243
|
+
Human-readable instruction describing what changed and what the
|
|
244
|
+
client should do.
|
|
245
|
+
field:
|
|
246
|
+
The request or response field affected (e.g. ``"request.model"``).
|
|
247
|
+
Empty string when the step is not field-specific.
|
|
248
|
+
breaking:
|
|
249
|
+
Whether this step represents a breaking change.
|
|
250
|
+
"""
|
|
251
|
+
|
|
252
|
+
description: str
|
|
253
|
+
field: str = ""
|
|
254
|
+
breaking: bool = True
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
class DeprecationSchedule:
|
|
258
|
+
"""Manages sunset dates and migration paths for API versions.
|
|
259
|
+
|
|
260
|
+
Parameters
|
|
261
|
+
----------
|
|
262
|
+
sunset_dates:
|
|
263
|
+
Mapping from ``APIVersion`` to an optional ``datetime.date``.
|
|
264
|
+
When the date is ``None`` the version has no planned sunset.
|
|
265
|
+
migration_steps:
|
|
266
|
+
Mapping from ``(from_version, to_version)`` to a list of
|
|
267
|
+
``MigrationStep`` instances describing what the client must change.
|
|
268
|
+
"""
|
|
269
|
+
|
|
270
|
+
def __init__(
|
|
271
|
+
self,
|
|
272
|
+
sunset_dates: dict[APIVersion, datetime.date | None] | None = None,
|
|
273
|
+
migration_steps: dict[tuple[APIVersion, APIVersion], list[MigrationStep]]
|
|
274
|
+
| None = None,
|
|
275
|
+
) -> None:
|
|
276
|
+
self._sunset_dates: dict[APIVersion, datetime.date | None] = (
|
|
277
|
+
dict(sunset_dates) if sunset_dates else {}
|
|
278
|
+
)
|
|
279
|
+
self._migration_steps: dict[
|
|
280
|
+
tuple[APIVersion, APIVersion], list[MigrationStep]
|
|
281
|
+
] = (dict(migration_steps) if migration_steps else {})
|
|
282
|
+
|
|
283
|
+
# ── Public API ────────────────────────────────────────────────────────
|
|
284
|
+
|
|
285
|
+
@classmethod
|
|
286
|
+
def default_schedule(cls) -> DeprecationSchedule:
|
|
287
|
+
"""Return the built-in deprecation schedule.
|
|
288
|
+
|
|
289
|
+
Current schedule (as of 2026-07):
|
|
290
|
+
- V1 (2026-01): sunset 2026-10-01, migrate to V2.
|
|
291
|
+
- V2 (2026-07): no sunset (latest stable).
|
|
292
|
+
"""
|
|
293
|
+
sunset = {
|
|
294
|
+
APIVersion.V1: datetime.date(2026, 10, 1),
|
|
295
|
+
APIVersion.V2: None, # latest stable — no sunset
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
migration: dict[tuple[APIVersion, APIVersion], list[MigrationStep]] = {
|
|
299
|
+
(APIVersion.V1, APIVersion.V2): [
|
|
300
|
+
MigrationStep(
|
|
301
|
+
description=(
|
|
302
|
+
"Response object type changed from "
|
|
303
|
+
"``chat.completion`` to ``chat.completion.v2``."
|
|
304
|
+
),
|
|
305
|
+
field="response.object",
|
|
306
|
+
breaking=True,
|
|
307
|
+
),
|
|
308
|
+
MigrationStep(
|
|
309
|
+
description=(
|
|
310
|
+
"Response includes ``system_fingerprint`` field."
|
|
311
|
+
),
|
|
312
|
+
field="response.system_fingerprint",
|
|
313
|
+
breaking=False,
|
|
314
|
+
),
|
|
315
|
+
MigrationStep(
|
|
316
|
+
description=(
|
|
317
|
+
"Request field ``user`` is now validated as a "
|
|
318
|
+
"non-empty string."
|
|
319
|
+
),
|
|
320
|
+
field="request.user",
|
|
321
|
+
breaking=False,
|
|
322
|
+
),
|
|
323
|
+
MigrationStep(
|
|
324
|
+
description=(
|
|
325
|
+
"Removed deprecated ``logit_bias`` field; use "
|
|
326
|
+
"``logprobs`` instead."
|
|
327
|
+
),
|
|
328
|
+
field="request.logit_bias",
|
|
329
|
+
breaking=True,
|
|
330
|
+
),
|
|
331
|
+
],
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
return cls(sunset_dates=sunset, migration_steps=migration)
|
|
335
|
+
|
|
336
|
+
def is_deprecated(self, version: APIVersion) -> bool:
|
|
337
|
+
"""Return ``True`` when *version* has a sunset date in the past."""
|
|
338
|
+
sunset = self._sunset_dates.get(version)
|
|
339
|
+
if sunset is None:
|
|
340
|
+
return False
|
|
341
|
+
return datetime.date.today() >= sunset
|
|
342
|
+
|
|
343
|
+
def is_near_sunset(self, version: APIVersion, days: int = NEAR_SUNSET_DAYS) -> bool:
|
|
344
|
+
"""Return ``True`` when *version* will sunset within *days* days."""
|
|
345
|
+
sunset = self._sunset_dates.get(version)
|
|
346
|
+
if sunset is None:
|
|
347
|
+
return False
|
|
348
|
+
remaining = (sunset - datetime.date.today()).days
|
|
349
|
+
return 0 <= remaining <= days
|
|
350
|
+
|
|
351
|
+
def days_until_sunset(self, version: APIVersion) -> int | None:
|
|
352
|
+
"""Return the number of days until *version* is sunset.
|
|
353
|
+
|
|
354
|
+
Returns ``None`` when the version has no sunset date or the sunset
|
|
355
|
+
has already passed.
|
|
356
|
+
"""
|
|
357
|
+
sunset = self._sunset_dates.get(version)
|
|
358
|
+
if sunset is None:
|
|
359
|
+
return None
|
|
360
|
+
remaining = (sunset - datetime.date.today()).days
|
|
361
|
+
return remaining if remaining >= 0 else None
|
|
362
|
+
|
|
363
|
+
def get_sunset_date(self, version: APIVersion) -> datetime.date | None:
|
|
364
|
+
"""Return the sunset date for *version*, or ``None``."""
|
|
365
|
+
return self._sunset_dates.get(version)
|
|
366
|
+
|
|
367
|
+
def sunset_header_value(self, version: APIVersion) -> str | None:
|
|
368
|
+
"""Return an HTTP-date string suitable for the ``Sunset`` header.
|
|
369
|
+
|
|
370
|
+
Returns ``None`` when the version has no sunset.
|
|
371
|
+
"""
|
|
372
|
+
sunset = self._sunset_dates.get(version)
|
|
373
|
+
if sunset is None:
|
|
374
|
+
return None
|
|
375
|
+
# RFC 7231 / HTTP-date: IMF-fixdate
|
|
376
|
+
return sunset.strftime("%a, %d %b %Y %H:%M:%S GMT")
|
|
377
|
+
|
|
378
|
+
def get_migration_path(
|
|
379
|
+
self,
|
|
380
|
+
current_version: APIVersion,
|
|
381
|
+
target_version: APIVersion,
|
|
382
|
+
) -> list[MigrationStep]:
|
|
383
|
+
"""Return the ordered list of migration steps to go from
|
|
384
|
+
*current_version* to *target_version*.
|
|
385
|
+
|
|
386
|
+
If no explicit path exists, returns a single generic step advising the
|
|
387
|
+
client to consult the changelog.
|
|
388
|
+
"""
|
|
389
|
+
path = self._migration_steps.get((current_version, target_version))
|
|
390
|
+
if path is not None:
|
|
391
|
+
return list(path)
|
|
392
|
+
|
|
393
|
+
# Generic fallback.
|
|
394
|
+
return [
|
|
395
|
+
MigrationStep(
|
|
396
|
+
description=(
|
|
397
|
+
f"No explicit migration path documented for "
|
|
398
|
+
f"{current_version.value} -> {target_version.value}. "
|
|
399
|
+
f"See https://docs.distllm.dev/api-versions for details."
|
|
400
|
+
),
|
|
401
|
+
breaking=True,
|
|
402
|
+
),
|
|
403
|
+
]
|
|
404
|
+
|
|
405
|
+
def auto_deprecate(self, now: datetime.date | None = None) -> list[APIVersion]:
|
|
406
|
+
"""Check all versions and return those whose sunset date has passed.
|
|
407
|
+
|
|
408
|
+
This is intended to be called periodically (e.g. via a background
|
|
409
|
+
task or during server startup) so that the server can log or reject
|
|
410
|
+
versions whose grace period has fully elapsed.
|
|
411
|
+
|
|
412
|
+
Parameters
|
|
413
|
+
----------
|
|
414
|
+
now:
|
|
415
|
+
Reference date. Defaults to ``datetime.date.today()``.
|
|
416
|
+
"""
|
|
417
|
+
if now is None:
|
|
418
|
+
now = datetime.date.today()
|
|
419
|
+
expired: list[APIVersion] = []
|
|
420
|
+
for version, sunset in self._sunset_dates.items():
|
|
421
|
+
if sunset is not None and now >= sunset:
|
|
422
|
+
expired.append(version)
|
|
423
|
+
return expired
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
# ── Version Compatibility Layer ───────────────────────────────────────────────
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
@dataclass
|
|
430
|
+
class FieldMapping:
|
|
431
|
+
"""Describes how a field is represented in two different API versions.
|
|
432
|
+
|
|
433
|
+
Attributes
|
|
434
|
+
----------
|
|
435
|
+
source_field:
|
|
436
|
+
Dot-separated path of the field in the source version
|
|
437
|
+
(e.g. ``"response.choices[].finish_reason"``).
|
|
438
|
+
target_field:
|
|
439
|
+
Dot-separated path of the field in the target version.
|
|
440
|
+
transform:
|
|
441
|
+
Optional callable that converts the source value to the target
|
|
442
|
+
representation. When ``None`` the value is copied as-is.
|
|
443
|
+
"""
|
|
444
|
+
|
|
445
|
+
source_field: str
|
|
446
|
+
target_field: str
|
|
447
|
+
transform: Callable[[Any], Any] | None = None
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
class VersionCompatibilityLayer:
|
|
451
|
+
"""Bidirectional field-level mapping between API versions.
|
|
452
|
+
|
|
453
|
+
The layer stores a set of ``FieldMapping`` instances for each version
|
|
454
|
+
pair and direction (request / response). When transforming, it walks
|
|
455
|
+
the source dict and applies every registered mapping.
|
|
456
|
+
|
|
457
|
+
Usage::
|
|
458
|
+
|
|
459
|
+
layer = VersionCompatibilityLayer()
|
|
460
|
+
layer.add_mapping(
|
|
461
|
+
from_version=APIVersion.V1,
|
|
462
|
+
to_version=APIVersion.V2,
|
|
463
|
+
field_maps=[
|
|
464
|
+
FieldMapping("request.logit_bias", "request.logprobs"),
|
|
465
|
+
FieldMapping(
|
|
466
|
+
"response.object",
|
|
467
|
+
"response.object",
|
|
468
|
+
transform=lambda _: "chat.completion.v2",
|
|
469
|
+
),
|
|
470
|
+
],
|
|
471
|
+
)
|
|
472
|
+
|
|
473
|
+
# Upgrade an old V1 request to V2 format:
|
|
474
|
+
v2_data = layer.transform_request(v1_data, APIVersion.V1, APIVersion.V2)
|
|
475
|
+
|
|
476
|
+
# Downgrade a V2 response back to V1 format:
|
|
477
|
+
v1_data = layer.transform_response(v2_data, APIVersion.V2, APIVersion.V1)
|
|
478
|
+
"""
|
|
479
|
+
|
|
480
|
+
def __init__(self) -> None:
|
|
481
|
+
# _request_mappings: (from_ver, to_ver) -> list[FieldMapping]
|
|
482
|
+
self._request_mappings: dict[
|
|
483
|
+
tuple[APIVersion, APIVersion], list[FieldMapping]
|
|
484
|
+
] = {}
|
|
485
|
+
# _response_mappings: (from_ver, to_ver) -> list[FieldMapping]
|
|
486
|
+
self._response_mappings: dict[
|
|
487
|
+
tuple[APIVersion, APIVersion], list[FieldMapping]
|
|
488
|
+
] = {}
|
|
489
|
+
|
|
490
|
+
# ── Registration ──────────────────────────────────────────────────────
|
|
491
|
+
|
|
492
|
+
def add_mapping(
|
|
493
|
+
self,
|
|
494
|
+
from_version: APIVersion,
|
|
495
|
+
to_version: APIVersion,
|
|
496
|
+
field_maps: list[FieldMapping],
|
|
497
|
+
*,
|
|
498
|
+
direction: str = "both",
|
|
499
|
+
) -> None:
|
|
500
|
+
"""Register field mappings between two versions.
|
|
501
|
+
|
|
502
|
+
Parameters
|
|
503
|
+
----------
|
|
504
|
+
from_version:
|
|
505
|
+
The source version of the data.
|
|
506
|
+
to_version:
|
|
507
|
+
The target version to transform into.
|
|
508
|
+
field_maps:
|
|
509
|
+
One or more ``FieldMapping`` descriptors.
|
|
510
|
+
direction:
|
|
511
|
+
``"request"``, ``"response"``, or ``"both"`` (default).
|
|
512
|
+
"""
|
|
513
|
+
if direction in ("request", "both"):
|
|
514
|
+
self._request_mappings.setdefault(
|
|
515
|
+
(from_version, to_version), []
|
|
516
|
+
).extend(field_maps)
|
|
517
|
+
if direction in ("response", "both"):
|
|
518
|
+
self._response_mappings.setdefault(
|
|
519
|
+
(from_version, to_version), []
|
|
520
|
+
).extend(field_maps)
|
|
521
|
+
|
|
522
|
+
# ── Transformation ────────────────────────────────────────────────────
|
|
523
|
+
|
|
524
|
+
def transform_request(
|
|
525
|
+
self,
|
|
526
|
+
data: dict[str, Any],
|
|
527
|
+
from_version: APIVersion,
|
|
528
|
+
to_version: APIVersion,
|
|
529
|
+
) -> dict[str, Any]:
|
|
530
|
+
"""Transform a request body from *from_version* schema to
|
|
531
|
+
*to_version* schema.
|
|
532
|
+
|
|
533
|
+
Returns a new dict; the original is not mutated.
|
|
534
|
+
"""
|
|
535
|
+
mappings = self._request_mappings.get((from_version, to_version))
|
|
536
|
+
if mappings is None:
|
|
537
|
+
return dict(data)
|
|
538
|
+
return self._apply_mappings(data, mappings)
|
|
539
|
+
|
|
540
|
+
def transform_response(
|
|
541
|
+
self,
|
|
542
|
+
data: dict[str, Any],
|
|
543
|
+
from_version: APIVersion,
|
|
544
|
+
to_version: APIVersion,
|
|
545
|
+
) -> dict[str, Any]:
|
|
546
|
+
"""Transform a response body from *from_version* schema to
|
|
547
|
+
*to_version* schema.
|
|
548
|
+
|
|
549
|
+
Returns a new dict; the original is not mutated.
|
|
550
|
+
"""
|
|
551
|
+
mappings = self._response_mappings.get((from_version, to_version))
|
|
552
|
+
if mappings is None:
|
|
553
|
+
return dict(data)
|
|
554
|
+
return self._apply_mappings(data, mappings)
|
|
555
|
+
|
|
556
|
+
# ── Internals ─────────────────────────────────────────────────────────
|
|
557
|
+
|
|
558
|
+
@staticmethod
|
|
559
|
+
def _apply_mappings(
|
|
560
|
+
data: dict[str, Any],
|
|
561
|
+
mappings: list[FieldMapping],
|
|
562
|
+
) -> dict[str, Any]:
|
|
563
|
+
"""Apply a list of ``FieldMapping`` entries to a nested dict.
|
|
564
|
+
|
|
565
|
+
The implementation uses dot-separated paths for field access.
|
|
566
|
+
Array indexes are represented by ``[]`` segments (e.g.
|
|
567
|
+
``"choices[].finish_reason"``), meaning the mapping applies to
|
|
568
|
+
every element of the array.
|
|
569
|
+
"""
|
|
570
|
+
result = dict(data)
|
|
571
|
+
|
|
572
|
+
for mapping in mappings:
|
|
573
|
+
source_value = _get_nested(result, mapping.source_field)
|
|
574
|
+
if source_value is _MISSING:
|
|
575
|
+
continue
|
|
576
|
+
|
|
577
|
+
transformed = (
|
|
578
|
+
mapping.transform(source_value)
|
|
579
|
+
if mapping.transform is not None
|
|
580
|
+
else source_value
|
|
581
|
+
)
|
|
582
|
+
|
|
583
|
+
result = _set_nested(result, mapping.target_field, transformed)
|
|
584
|
+
|
|
585
|
+
return result
|
|
586
|
+
|
|
587
|
+
|
|
588
|
+
# ── Sentinel for missing values ───────────────────────────────────────────────
|
|
589
|
+
|
|
590
|
+
|
|
591
|
+
class _Missing:
|
|
592
|
+
"""Sentinel to distinguish ``None`` from absent."""
|
|
593
|
+
|
|
594
|
+
def __repr__(self) -> str:
|
|
595
|
+
return "<MISSING>"
|
|
596
|
+
|
|
597
|
+
|
|
598
|
+
_MISSING = _Missing()
|
|
599
|
+
|
|
600
|
+
|
|
601
|
+
def _get_nested(d: dict[str, Any], path: str) -> Any:
|
|
602
|
+
"""Retrieve a value from a nested dict using a dot-separated *path*.
|
|
603
|
+
|
|
604
|
+
Supports array segments with ``[]`` to apply a lookup across all
|
|
605
|
+
elements of a list (returns the first matched value, since transforms
|
|
606
|
+
are uniform across elements).
|
|
607
|
+
"""
|
|
608
|
+
parts = _split_path(path)
|
|
609
|
+
current: Any = d
|
|
610
|
+
for part in parts:
|
|
611
|
+
if part == "[]" and isinstance(current, list):
|
|
612
|
+
# Apply the remainder of the path to every element; return
|
|
613
|
+
# the value from the first element that has it.
|
|
614
|
+
remainder = ".".join(parts[parts.index("[]") + 1 :])
|
|
615
|
+
for item in current:
|
|
616
|
+
if isinstance(item, dict):
|
|
617
|
+
val = _get_nested(item, remainder)
|
|
618
|
+
if val is not _MISSING:
|
|
619
|
+
return val
|
|
620
|
+
return _MISSING
|
|
621
|
+
if isinstance(current, dict):
|
|
622
|
+
current = current.get(part, _MISSING)
|
|
623
|
+
else:
|
|
624
|
+
return _MISSING
|
|
625
|
+
if current is _MISSING:
|
|
626
|
+
return _MISSING
|
|
627
|
+
return current
|
|
628
|
+
|
|
629
|
+
|
|
630
|
+
def _set_nested(d: dict[str, Any], path: str, value: Any) -> dict[str, Any]:
|
|
631
|
+
"""Set a value in a nested dict, returning a new dict (shallow copy at
|
|
632
|
+
each level). Does not mutate *d*.
|
|
633
|
+
|
|
634
|
+
Supports array segments with ``[]`` to set the value on every element
|
|
635
|
+
of a list.
|
|
636
|
+
"""
|
|
637
|
+
parts = _split_path(path)
|
|
638
|
+
if not parts:
|
|
639
|
+
return d
|
|
640
|
+
|
|
641
|
+
result = dict(d)
|
|
642
|
+
current = result
|
|
643
|
+
|
|
644
|
+
for i, part in enumerate(parts):
|
|
645
|
+
if part == "[]" and isinstance(current, list):
|
|
646
|
+
remainder = ".".join(parts[i + 1 :])
|
|
647
|
+
new_list = []
|
|
648
|
+
for item in current:
|
|
649
|
+
if isinstance(item, dict):
|
|
650
|
+
new_list.append(
|
|
651
|
+
_set_nested(item, remainder, value)
|
|
652
|
+
if remainder
|
|
653
|
+
else value
|
|
654
|
+
)
|
|
655
|
+
else:
|
|
656
|
+
new_list.append(item)
|
|
657
|
+
# Walk back to parent to set the modified list.
|
|
658
|
+
# For simplicity, we mutate result directly at the known path.
|
|
659
|
+
# Full immutable walk is expensive; we accept shallow mutation
|
|
660
|
+
# of the list container since it is local to this function and
|
|
661
|
+
# never exposed.
|
|
662
|
+
_set_nested_impl(result, parts[:i], new_list)
|
|
663
|
+
return result
|
|
664
|
+
|
|
665
|
+
if i == len(parts) - 1:
|
|
666
|
+
# Last part — set the value.
|
|
667
|
+
if isinstance(current, dict):
|
|
668
|
+
current[part] = value
|
|
669
|
+
else:
|
|
670
|
+
if part not in current:
|
|
671
|
+
current[part] = {}
|
|
672
|
+
next_current = current.get(part)
|
|
673
|
+
if isinstance(next_current, dict):
|
|
674
|
+
current = next_current
|
|
675
|
+
elif isinstance(next_current, list):
|
|
676
|
+
# Enter list context for the next iteration.
|
|
677
|
+
current = next_current
|
|
678
|
+
else:
|
|
679
|
+
# Intermediate missing — create dict.
|
|
680
|
+
current[part] = {}
|
|
681
|
+
current = current[part]
|
|
682
|
+
|
|
683
|
+
return result
|
|
684
|
+
|
|
685
|
+
|
|
686
|
+
def _set_nested_impl(d: dict[str, Any], path_parts: list[str], value: Any) -> None:
|
|
687
|
+
"""Mutate *d* at *path_parts* — used internally by ``_set_nested`` for
|
|
688
|
+
the array-branch case where immutability is already guaranteed by the
|
|
689
|
+
caller."""
|
|
690
|
+
current = d
|
|
691
|
+
for i, part in enumerate(path_parts):
|
|
692
|
+
if i == len(path_parts) - 1:
|
|
693
|
+
current[part] = value
|
|
694
|
+
else:
|
|
695
|
+
next_val = current.get(part)
|
|
696
|
+
if isinstance(next_val, dict):
|
|
697
|
+
current = next_val
|
|
698
|
+
else:
|
|
699
|
+
current[part] = {}
|
|
700
|
+
current = current[part]
|
|
701
|
+
|
|
702
|
+
|
|
703
|
+
def _split_path(path: str) -> list[str]:
|
|
704
|
+
"""Split a dot-separated path into segments, preserving ``[]`` as a
|
|
705
|
+
single token."""
|
|
706
|
+
segments: list[str] = []
|
|
707
|
+
for segment in path.split("."):
|
|
708
|
+
if "[]" in segment:
|
|
709
|
+
# Handle both "choices[]" and "choices.[]"
|
|
710
|
+
parts = segment.split("[]")
|
|
711
|
+
for p in parts:
|
|
712
|
+
if p:
|
|
713
|
+
segments.append(p)
|
|
714
|
+
segments.append("[]")
|
|
715
|
+
# Pop the trailing [] if the path ended with it and we added it
|
|
716
|
+
if segments and segments[-1] == "[]" and not path.endswith("[]"):
|
|
717
|
+
segments.pop()
|
|
718
|
+
else:
|
|
719
|
+
segments.append(segment)
|
|
720
|
+
return segments
|
|
721
|
+
|
|
722
|
+
|
|
723
|
+
# ── Version Middleware ────────────────────────────────────────────────────────
|
|
724
|
+
|
|
725
|
+
|
|
726
|
+
class VersionMiddleware(BaseHTTPMiddleware):
|
|
727
|
+
"""Starlette middleware that enforces the API versioning contract.
|
|
728
|
+
|
|
729
|
+
What it does for every request:
|
|
730
|
+
|
|
731
|
+
1. Calls ``VersionNegotiator.negotiate()`` to determine the target
|
|
732
|
+
version from the ``Accept-Version`` header or URL path.
|
|
733
|
+
2. Stores the resolved version on ``request.state.api_version``.
|
|
734
|
+
3. If the resolved version is fully deprecated (past sunset), responds
|
|
735
|
+
with **410 Gone**.
|
|
736
|
+
4. If the resolved version is *near* sunset (within 60 days), adds a
|
|
737
|
+
``Warning`` response header in addition to the standard deprecation
|
|
738
|
+
headers.
|
|
739
|
+
5. Appends ``X-API-Version``, ``Sunset``, and ``X-API-Deprecation``
|
|
740
|
+
headers to every response.
|
|
741
|
+
6. Optionally transforms request bodies from the negotiated version to
|
|
742
|
+
the server's internal version, and response bodies back.
|
|
743
|
+
|
|
744
|
+
Parameters
|
|
745
|
+
----------
|
|
746
|
+
negotiator:
|
|
747
|
+
The ``VersionNegotiator`` instance. Defaults to a fresh negotiator.
|
|
748
|
+
schedule:
|
|
749
|
+
The ``DeprecationSchedule`` instance. Defaults to
|
|
750
|
+
``DeprecationSchedule.default_schedule()``.
|
|
751
|
+
compatibility_layer:
|
|
752
|
+
Optional ``VersionCompatibilityLayer`` for request/response body
|
|
753
|
+
transformation.
|
|
754
|
+
internal_version:
|
|
755
|
+
The server's internal version. Requests at other versions are
|
|
756
|
+
transformed to this version before reaching the route handler, and
|
|
757
|
+
responses are transformed back. Defaults to ``APIVersion.V2``.
|
|
758
|
+
reject_deprecated:
|
|
759
|
+
When ``True`` (default), requests to fully deprecated versions
|
|
760
|
+
receive a 410 response. Set to ``False`` to only warn (useful
|
|
761
|
+
during a migration grace period).
|
|
762
|
+
"""
|
|
763
|
+
|
|
764
|
+
def __init__(
|
|
765
|
+
self,
|
|
766
|
+
app: Any,
|
|
767
|
+
negotiator: VersionNegotiator | None = None,
|
|
768
|
+
schedule: DeprecationSchedule | None = None,
|
|
769
|
+
compatibility_layer: VersionCompatibilityLayer | None = None,
|
|
770
|
+
internal_version: APIVersion = APIVersion.V2,
|
|
771
|
+
reject_deprecated: bool = True,
|
|
772
|
+
) -> None:
|
|
773
|
+
super().__init__(app)
|
|
774
|
+
self._negotiator = negotiator or VersionNegotiator()
|
|
775
|
+
self._schedule = schedule or DeprecationSchedule.default_schedule()
|
|
776
|
+
self._compat = compatibility_layer
|
|
777
|
+
self._internal_version = internal_version
|
|
778
|
+
self._reject_deprecated = reject_deprecated
|
|
779
|
+
|
|
780
|
+
# ── Middleware Dispatch ───────────────────────────────────────────────
|
|
781
|
+
|
|
782
|
+
async def dispatch(self, request: Request, call_next: Callable) -> Response:
|
|
783
|
+
# 1. Negotiate version.
|
|
784
|
+
result = self._negotiator.negotiate(request)
|
|
785
|
+
|
|
786
|
+
if result.version is None:
|
|
787
|
+
return Response(
|
|
788
|
+
status_code=400,
|
|
789
|
+
content=(
|
|
790
|
+
"Could not determine API version. "
|
|
791
|
+
"Set the Accept-Version header or use a versioned path "
|
|
792
|
+
"(e.g. /v1/...)."
|
|
793
|
+
),
|
|
794
|
+
media_type="text/plain",
|
|
795
|
+
)
|
|
796
|
+
|
|
797
|
+
# 2. Store version on request state for downstream use.
|
|
798
|
+
request.state.api_version = result.version
|
|
799
|
+
request.state.api_version_source = result.source
|
|
800
|
+
|
|
801
|
+
# 3. Check full deprecation (past sunset).
|
|
802
|
+
if self._reject_deprecated and self._schedule.is_deprecated(result.version):
|
|
803
|
+
sunset = self._schedule.get_sunset_date(result.version)
|
|
804
|
+
logger.warning(
|
|
805
|
+
"Rejected request to deprecated version %s (sunset %s) from %s",
|
|
806
|
+
result.version.value,
|
|
807
|
+
sunset,
|
|
808
|
+
request.client.host if request.client else "unknown",
|
|
809
|
+
)
|
|
810
|
+
return Response(
|
|
811
|
+
status_code=410,
|
|
812
|
+
content=(
|
|
813
|
+
f"API version {result.version.value} is no longer supported. "
|
|
814
|
+
f"It was removed on {sunset}. "
|
|
815
|
+
f"Please migrate to a newer version. "
|
|
816
|
+
f"See https://docs.distllm.dev/api-versions for migration."
|
|
817
|
+
),
|
|
818
|
+
media_type="text/plain",
|
|
819
|
+
headers={
|
|
820
|
+
VERSION_HEADER: result.version.value,
|
|
821
|
+
SUNSET_HEADER: self._schedule.sunset_header_value(result.version)
|
|
822
|
+
or "",
|
|
823
|
+
},
|
|
824
|
+
)
|
|
825
|
+
|
|
826
|
+
# 4. Transform request body if compatibility layer is configured and
|
|
827
|
+
# the request version differs from the internal version.
|
|
828
|
+
if (
|
|
829
|
+
self._compat is not None
|
|
830
|
+
and result.version != self._internal_version
|
|
831
|
+
):
|
|
832
|
+
request.state._original_api_version = result.version
|
|
833
|
+
# Note: actual body transformation happens in a helper that must
|
|
834
|
+
# be called from the route handler, because FastAPI/Starlette
|
|
835
|
+
# does not expose the parsed body to middleware in a
|
|
836
|
+
# framework-independent way. We store the target version so
|
|
837
|
+
# route handlers can opt in.
|
|
838
|
+
request.state.api_compat_target = self._internal_version
|
|
839
|
+
else:
|
|
840
|
+
request.state.api_compat_target = None
|
|
841
|
+
|
|
842
|
+
# 5. Process the request.
|
|
843
|
+
response = await call_next(request)
|
|
844
|
+
|
|
845
|
+
# 6. Add version headers to the response.
|
|
846
|
+
response.headers[VERSION_HEADER] = result.version.value
|
|
847
|
+
|
|
848
|
+
sunset_header = self._schedule.sunset_header_value(result.version)
|
|
849
|
+
if sunset_header is not None:
|
|
850
|
+
response.headers[SUNSET_HEADER] = sunset_header
|
|
851
|
+
|
|
852
|
+
# 7. Add deprecation warning header for versions near sunset.
|
|
853
|
+
if self._schedule.is_near_sunset(result.version):
|
|
854
|
+
sunset_date = self._schedule.get_sunset_date(result.version)
|
|
855
|
+
days_left = self._schedule.days_until_sunset(result.version)
|
|
856
|
+
migration_steps = self._schedule.get_migration_path(
|
|
857
|
+
result.version, self._internal_version
|
|
858
|
+
)
|
|
859
|
+
breaking_count = sum(1 for s in migration_steps if s.breaking)
|
|
860
|
+
response.headers[DEPRECATION_HEADER] = (
|
|
861
|
+
f"Version {result.version.value} will be removed on "
|
|
862
|
+
f"{sunset_date} ({days_left} days remaining). "
|
|
863
|
+
f"{breaking_count} breaking change(s) to address. "
|
|
864
|
+
f"See https://docs.distllm.dev/api-versions for migration."
|
|
865
|
+
)
|
|
866
|
+
|
|
867
|
+
# 8. Transform response body back to the request version if the
|
|
868
|
+
# compatibility layer is active and the request was upgraded.
|
|
869
|
+
original_version: APIVersion | None = getattr(
|
|
870
|
+
request.state, "_original_api_version", None
|
|
871
|
+
)
|
|
872
|
+
if (
|
|
873
|
+
self._compat is not None
|
|
874
|
+
and original_version is not None
|
|
875
|
+
and original_version != self._internal_version
|
|
876
|
+
):
|
|
877
|
+
# The response body may already be streamed at this point.
|
|
878
|
+
# Full body transformation for buffered responses only.
|
|
879
|
+
if hasattr(response, "body") and callable(response.body):
|
|
880
|
+
body_value = response.body # type: ignore[union-attr]
|
|
881
|
+
if isinstance(body_value, bytes):
|
|
882
|
+
import json
|
|
883
|
+
|
|
884
|
+
try:
|
|
885
|
+
body_dict = json.loads(body_value)
|
|
886
|
+
transformed = self._compat.transform_response(
|
|
887
|
+
body_dict,
|
|
888
|
+
from_version=self._internal_version,
|
|
889
|
+
to_version=original_version,
|
|
890
|
+
)
|
|
891
|
+
# Build a new response with the transformed body.
|
|
892
|
+
from starlette.responses import JSONResponse
|
|
893
|
+
|
|
894
|
+
new_response = JSONResponse(
|
|
895
|
+
content=transformed,
|
|
896
|
+
status_code=response.status_code,
|
|
897
|
+
headers=dict(response.headers),
|
|
898
|
+
media_type=response.media_type,
|
|
899
|
+
)
|
|
900
|
+
return new_response
|
|
901
|
+
except (json.JSONDecodeError, TypeError, ValueError):
|
|
902
|
+
logger.debug(
|
|
903
|
+
"Could not transform response body for version %s",
|
|
904
|
+
original_version.value,
|
|
905
|
+
)
|
|
906
|
+
|
|
907
|
+
return response
|
|
908
|
+
|
|
909
|
+
|
|
910
|
+
# ── Convenience factory ───────────────────────────────────────────────────────
|
|
911
|
+
|
|
912
|
+
|
|
913
|
+
def create_version_middleware(
|
|
914
|
+
*,
|
|
915
|
+
default_version: str = DEFAULT_API_VERSION,
|
|
916
|
+
internal_version: APIVersion = APIVersion.V2,
|
|
917
|
+
reject_deprecated: bool = True,
|
|
918
|
+
enable_compatibility: bool = True,
|
|
919
|
+
) -> VersionMiddleware:
|
|
920
|
+
"""Create a fully-configured ``VersionMiddleware`` with sane defaults.
|
|
921
|
+
|
|
922
|
+
Parameters
|
|
923
|
+
----------
|
|
924
|
+
default_version:
|
|
925
|
+
Fallback version string (default ``"2026-07"``).
|
|
926
|
+
internal_version:
|
|
927
|
+
The server's internal API version (default V2).
|
|
928
|
+
reject_deprecated:
|
|
929
|
+
Whether to return 410 for past-sunset versions (default ``True``).
|
|
930
|
+
enable_compatibility:
|
|
931
|
+
Whether to register the built-in field mappings for request/response
|
|
932
|
+
transformation (default ``True``).
|
|
933
|
+
"""
|
|
934
|
+
schedule = DeprecationSchedule.default_schedule()
|
|
935
|
+
negotiator = VersionNegotiator(default_version=default_version)
|
|
936
|
+
|
|
937
|
+
compat: VersionCompatibilityLayer | None = None
|
|
938
|
+
if enable_compatibility:
|
|
939
|
+
compat = VersionCompatibilityLayer()
|
|
940
|
+
_register_default_mappings(compat)
|
|
941
|
+
|
|
942
|
+
return VersionMiddleware(
|
|
943
|
+
app=None, # type: ignore[arg-type] # set when added via add_middleware
|
|
944
|
+
negotiator=negotiator,
|
|
945
|
+
schedule=schedule,
|
|
946
|
+
compatibility_layer=compat,
|
|
947
|
+
internal_version=internal_version,
|
|
948
|
+
reject_deprecated=reject_deprecated,
|
|
949
|
+
)
|
|
950
|
+
|
|
951
|
+
|
|
952
|
+
def _register_default_mappings(layer: VersionCompatibilityLayer) -> None:
|
|
953
|
+
"""Register the built-in field mappings for V1 <-> V2 compatibility."""
|
|
954
|
+
layer.add_mapping(
|
|
955
|
+
from_version=APIVersion.V1,
|
|
956
|
+
to_version=APIVersion.V2,
|
|
957
|
+
field_maps=[
|
|
958
|
+
FieldMapping(
|
|
959
|
+
source_field="logit_bias",
|
|
960
|
+
target_field="logprobs",
|
|
961
|
+
transform=lambda val: (
|
|
962
|
+
{"top_logprobs": len(val)} if isinstance(val, dict) else None
|
|
963
|
+
),
|
|
964
|
+
),
|
|
965
|
+
],
|
|
966
|
+
direction="request",
|
|
967
|
+
)
|
|
968
|
+
layer.add_mapping(
|
|
969
|
+
from_version=APIVersion.V2,
|
|
970
|
+
to_version=APIVersion.V1,
|
|
971
|
+
field_maps=[
|
|
972
|
+
FieldMapping(
|
|
973
|
+
source_field="response.object",
|
|
974
|
+
target_field="response.object",
|
|
975
|
+
transform=lambda _: "chat.completion",
|
|
976
|
+
),
|
|
977
|
+
FieldMapping(
|
|
978
|
+
source_field="response.system_fingerprint",
|
|
979
|
+
target_field="response.system_fingerprint",
|
|
980
|
+
transform=lambda _: None,
|
|
981
|
+
),
|
|
982
|
+
],
|
|
983
|
+
direction="response",
|
|
984
|
+
)
|