distributed-llm 0.4.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (806) hide show
  1. distllm/__init__.py +47 -0
  2. distllm/api/__init__.py +11 -0
  3. distllm/api/api_settings.py +126 -0
  4. distllm/api/api_state.py +85 -0
  5. distllm/api/api_versioning.py +984 -0
  6. distllm/api/app_state.py +29 -0
  7. distllm/api/auth/__init__.py +181 -0
  8. distllm/api/auth/models.py +37 -0
  9. distllm/api/auth/oauth2.py +123 -0
  10. distllm/api/auth/oidc.py +317 -0
  11. distllm/api/auth/saml.py +107 -0
  12. distllm/api/auth/store.py +39 -0
  13. distllm/api/auth_deps.py +68 -0
  14. distllm/api/authz/__init__.py +30 -0
  15. distllm/api/authz/opa.py +297 -0
  16. distllm/api/background_tasks.py +295 -0
  17. distllm/api/body_cache_middleware.py +117 -0
  18. distllm/api/circuit_breaker_middleware.py +175 -0
  19. distllm/api/cost_middleware.py +186 -0
  20. distllm/api/csrf_middleware.py +196 -0
  21. distllm/api/dedup.py +180 -0
  22. distllm/api/errors.py +256 -0
  23. distllm/api/grpc_bridge.py +296 -0
  24. distllm/api/ip_utils.py +58 -0
  25. distllm/api/middleware.py +706 -0
  26. distllm/api/observability_middleware.py +169 -0
  27. distllm/api/persistent_store.py +295 -0
  28. distllm/api/prompt_injection.py +418 -0
  29. distllm/api/quota_middleware.py +186 -0
  30. distllm/api/rate_limit_middleware.py +13 -0
  31. distllm/api/rate_limiter.py +250 -0
  32. distllm/api/rate_limiter_unified.py +975 -0
  33. distllm/api/redis_rate_limiter.py +164 -0
  34. distllm/api/routes/__init__.py +42 -0
  35. distllm/api/routes/admin.py +599 -0
  36. distllm/api/routes/api_keys.py +115 -0
  37. distllm/api/routes/assistants.py +443 -0
  38. distllm/api/routes/batch.py +1094 -0
  39. distllm/api/routes/chat.py +875 -0
  40. distllm/api/routes/completion.py +141 -0
  41. distllm/api/routes/debug.py +185 -0
  42. distllm/api/routes/defrag.py +66 -0
  43. distllm/api/routes/embeddings.py +399 -0
  44. distllm/api/routes/eval.py +254 -0
  45. distllm/api/routes/exchange.py +367 -0
  46. distllm/api/routes/experiments.py +293 -0
  47. distllm/api/routes/federated.py +155 -0
  48. distllm/api/routes/files.py +243 -0
  49. distllm/api/routes/fine_tuning.py +381 -0
  50. distllm/api/routes/gossip.py +72 -0
  51. distllm/api/routes/health.py +395 -0
  52. distllm/api/routes/leaderboard.py +377 -0
  53. distllm/api/routes/marketplace.py +320 -0
  54. distllm/api/routes/metrics_history.py +415 -0
  55. distllm/api/routes/model_registry.py +255 -0
  56. distllm/api/routes/moderation.py +420 -0
  57. distllm/api/routes/plugins.py +184 -0
  58. distllm/api/routes/prompts.py +347 -0
  59. distllm/api/routes/router_admin.py +187 -0
  60. distllm/api/routes/scheduler.py +168 -0
  61. distllm/api/routes/tokenize.py +77 -0
  62. distllm/api/routes/tools.py +89 -0
  63. distllm/api/routes/webhooks.py +336 -0
  64. distllm/api/routes/webrtc.py +194 -0
  65. distllm/api/semantic_cache.py +337 -0
  66. distllm/api/server.py +2038 -0
  67. distllm/api/server_config.py +140 -0
  68. distllm/api/server_coordinator.py +125 -0
  69. distllm/api/server_errors.py +79 -0
  70. distllm/api/server_lifespan.py +184 -0
  71. distllm/api/server_routes_api.py +456 -0
  72. distllm/api/server_routes_dashboard.py +246 -0
  73. distllm/api/server_state.py +29 -0
  74. distllm/api/services/__init__.py +2 -0
  75. distllm/api/services/chat_service.py +677 -0
  76. distllm/api/services/completion_service.py +227 -0
  77. distllm/api/services/embedding_service.py +451 -0
  78. distllm/api/services/eval_service.py +198 -0
  79. distllm/api/sso_auth.py +644 -0
  80. distllm/api/sso_middleware.py +808 -0
  81. distllm/api/streaming.py +479 -0
  82. distllm/api/token_estimator.py +86 -0
  83. distllm/api/tool_calling.py +240 -0
  84. distllm/api/tracing_middleware.py +579 -0
  85. distllm/api/validation.py +58 -0
  86. distllm/api/waf.py +344 -0
  87. distllm/api/webhooks/__init__.py +17 -0
  88. distllm/api/webhooks/delivery.py +452 -0
  89. distllm/api/ws_chat.py +495 -0
  90. distllm/backends/__init__.py +121 -0
  91. distllm/backends/config.py +85 -0
  92. distllm/backends/exllama_backend.py +229 -0
  93. distllm/backends/llamacpp_backend.py +266 -0
  94. distllm/backends/mlx_backend.py +199 -0
  95. distllm/backends/nim_backend.py +838 -0
  96. distllm/backends/ollama_backend.py +48 -0
  97. distllm/backends/onnx_backend.py +239 -0
  98. distllm/backends/paged_attention.py +547 -0
  99. distllm/backends/paged_attention_quantized.py +319 -0
  100. distllm/backends/protocol.py +203 -0
  101. distllm/backends/pytorch_backend.py +157 -0
  102. distllm/backends/registry.py +428 -0
  103. distllm/backends/tensorrt_backend.py +333 -0
  104. distllm/backends/tgi_backend.py +120 -0
  105. distllm/backends/triton_backend.py +875 -0
  106. distllm/backends/vllm_backend.py +489 -0
  107. distllm/backends/webgpu_backend.py +255 -0
  108. distllm/benchmarks/__init__.py +10 -0
  109. distllm/benchmarks/evaluation_harness.py +69 -0
  110. distllm/benchmarks/scaling.py +298 -0
  111. distllm/cache/__init__.py +19 -0
  112. distllm/cache/crdt.py +507 -0
  113. distllm/cache/cross_cluster_prefix_index.py +709 -0
  114. distllm/cli/__init__.py +17 -0
  115. distllm/cli/adapters.py +93 -0
  116. distllm/cli/autopsy.py +287 -0
  117. distllm/cli/backup.py +118 -0
  118. distllm/cli/benchmark.py +241 -0
  119. distllm/cli/cert.py +86 -0
  120. distllm/cli/chat.py +79 -0
  121. distllm/cli/chat_stream.py +494 -0
  122. distllm/cli/client.py +233 -0
  123. distllm/cli/cluster.py +405 -0
  124. distllm/cli/completion.py +79 -0
  125. distllm/cli/compress.py +98 -0
  126. distllm/cli/config_commands.py +71 -0
  127. distllm/cli/cost_avoid.py +168 -0
  128. distllm/cli/daas_commands.py +123 -0
  129. distllm/cli/defrag_commands.py +128 -0
  130. distllm/cli/deploy.py +252 -0
  131. distllm/cli/doctor.py +688 -0
  132. distllm/cli/draft_commands.py +82 -0
  133. distllm/cli/error_handler.py +192 -0
  134. distllm/cli/eval.py +168 -0
  135. distllm/cli/federate_commands.py +125 -0
  136. distllm/cli/install.py +198 -0
  137. distllm/cli/logs.py +85 -0
  138. distllm/cli/main.py +1795 -0
  139. distllm/cli/models.py +74 -0
  140. distllm/cli/notify.py +91 -0
  141. distllm/cli/observe.py +314 -0
  142. distllm/cli/onboard.py +671 -0
  143. distllm/cli/output.py +437 -0
  144. distllm/cli/profile.py +152 -0
  145. distllm/cli/profile_presets.py +129 -0
  146. distllm/cli/prompts.py +139 -0
  147. distllm/cli/quota.py +227 -0
  148. distllm/cli/router.py +131 -0
  149. distllm/cli/run.py +59 -0
  150. distllm/cli/setup.py +104 -0
  151. distllm/cli/setup_wizard.py +191 -0
  152. distllm/cli/status.py +65 -0
  153. distllm/cli/system_commands.py +255 -0
  154. distllm/cli/tune.py +488 -0
  155. distllm/cli/tutorial.py +128 -0
  156. distllm/cli/verify.py +37 -0
  157. distllm/cli/webhook.py +85 -0
  158. distllm/client/__init__.py +38 -0
  159. distllm/client/client.py +503 -0
  160. distllm/cloud/__init__.py +53 -0
  161. distllm/cloud/aws.py +199 -0
  162. distllm/cloud/azure.py +179 -0
  163. distllm/cloud/common.py +217 -0
  164. distllm/cloud/gcp.py +238 -0
  165. distllm/cloud/spot_orchestrator.py +1642 -0
  166. distllm/cloud/worker_agent.py +344 -0
  167. distllm/compliance/__init__.py +10 -0
  168. distllm/compliance/evidence_pack.py +699 -0
  169. distllm/config/__init__.py +81 -0
  170. distllm/config/_application.py +41 -0
  171. distllm/config/_backends.py +80 -0
  172. distllm/config/_cache.py +121 -0
  173. distllm/config/_deployment.py +95 -0
  174. distllm/config/_generation.py +36 -0
  175. distllm/config/_hardware.py +42 -0
  176. distllm/config/_model.py +270 -0
  177. distllm/config/_network.py +235 -0
  178. distllm/config/_observability.py +42 -0
  179. distllm/config/_parallelism.py +172 -0
  180. distllm/config/_performance.py +47 -0
  181. distllm/config/loader.py +168 -0
  182. distllm/config/model_heuristics.py +169 -0
  183. distllm/config/profiles.py +168 -0
  184. distllm/config/reference.py +104 -0
  185. distllm/config/resolver.py +240 -0
  186. distllm/config/settings.py +463 -0
  187. distllm/constants.py +117 -0
  188. distllm/core/__init__.py +111 -0
  189. distllm/core/_advanced_scheduling_original.py +1892 -0
  190. distllm/core/ab_test_coordinator.py +178 -0
  191. distllm/core/activation_profiler.py +172 -0
  192. distllm/core/adaptive_batching.py +234 -0
  193. distllm/core/adaptive_cache_compressor.py +168 -0
  194. distllm/core/adaptive_compression.py +427 -0
  195. distllm/core/adaptive_compression_hierarchy.py +402 -0
  196. distllm/core/advanced_scheduling/__init__.py +39 -0
  197. distllm/core/advanced_scheduling/cost_aware.py +55 -0
  198. distllm/core/advanced_scheduling/disaggregated.py +352 -0
  199. distllm/core/advanced_scheduling/energy.py +58 -0
  200. distllm/core/advanced_scheduling/federated.py +59 -0
  201. distllm/core/advanced_scheduling/heterogeneous.py +84 -0
  202. distllm/core/advanced_scheduling/multi_objective.py +372 -0
  203. distllm/core/advanced_scheduling/policy.py +95 -0
  204. distllm/core/advanced_scheduling/predictive.py +47 -0
  205. distllm/core/advanced_scheduling/preemption.py +43 -0
  206. distllm/core/advanced_scheduling/tiered_store.py +604 -0
  207. distllm/core/advanced_scheduling/token_bank.py +62 -0
  208. distllm/core/advanced_scheduling/wan.py +85 -0
  209. distllm/core/aegis_compliance.py +890 -0
  210. distllm/core/aether_federated.py +1326 -0
  211. distllm/core/agentic_router.py +749 -0
  212. distllm/core/api_key_store.py +478 -0
  213. distllm/core/arbitrage_engine.py +613 -0
  214. distllm/core/aria_autoscaler.py +1339 -0
  215. distllm/core/async_connection_pool.py +187 -0
  216. distllm/core/async_pipelined_speculative.py +543 -0
  217. distllm/core/atlas_mesh.py +1437 -0
  218. distllm/core/auto_discovery.py +351 -0
  219. distllm/core/auto_partitioner.py +396 -0
  220. distllm/core/autonomous_healer.py +466 -0
  221. distllm/core/autoq.py +1545 -0
  222. distllm/core/backup_manager.py +344 -0
  223. distllm/core/bargaining_engine.py +450 -0
  224. distllm/core/batch_builder.py +480 -0
  225. distllm/core/batch_scheduler.py +1342 -0
  226. distllm/core/block_affinity_tracker.py +162 -0
  227. distllm/core/block_eviction_policy.py +363 -0
  228. distllm/core/cache_aware_router.py +116 -0
  229. distllm/core/cache_bench.py +239 -0
  230. distllm/core/cache_coherence.py +139 -0
  231. distllm/core/cache_doctor.py +220 -0
  232. distllm/core/cache_eviction.py +181 -0
  233. distllm/core/cache_index.py +91 -0
  234. distllm/core/cache_manager.py +703 -0
  235. distllm/core/cache_migration.py +134 -0
  236. distllm/core/cache_persistence.py +200 -0
  237. distllm/core/cache_query_log.py +195 -0
  238. distllm/core/cache_snapshot.py +178 -0
  239. distllm/core/cache_template_warmer.py +112 -0
  240. distllm/core/cache_warming.py +188 -0
  241. distllm/core/calibration.py +223 -0
  242. distllm/core/carbon_migration.py +338 -0
  243. distllm/core/cert_rotation.py +402 -0
  244. distllm/core/certificate_manager.py +414 -0
  245. distllm/core/cluster_manager.py +212 -0
  246. distllm/core/cluster_predictive_prefetcher.py +239 -0
  247. distllm/core/cluster_registry.py +227 -0
  248. distllm/core/cluster_state_store.py +442 -0
  249. distllm/core/compliance_evidence.py +503 -0
  250. distllm/core/compressed_speculative.py +399 -0
  251. distllm/core/connection_pool.py +387 -0
  252. distllm/core/constrained_decoder.py +678 -0
  253. distllm/core/coordinator.py +1594 -0
  254. distllm/core/coordinator_cli.py +130 -0
  255. distllm/core/coordinator_config.py +203 -0
  256. distllm/core/coordinator_config_wiring.py +316 -0
  257. distllm/core/coordinator_election.py +246 -0
  258. distllm/core/coordinator_failover.py +197 -0
  259. distllm/core/coordinator_health.py +135 -0
  260. distllm/core/coordinator_lifecycle.py +272 -0
  261. distllm/core/coordinator_metrics.py +155 -0
  262. distllm/core/coordinator_multi_model.py +41 -0
  263. distllm/core/coordinator_request.py +246 -0
  264. distllm/core/coordinator_state.py +215 -0
  265. distllm/core/coordinator_subsystem.py +614 -0
  266. distllm/core/correctness_cert.py +133 -0
  267. distllm/core/correctness_harness.py +421 -0
  268. distllm/core/cortex_multimodel.py +939 -0
  269. distllm/core/cost_comparison.py +250 -0
  270. distllm/core/cost_dashboard.py +363 -0
  271. distllm/core/cost_optimizer.py +317 -0
  272. distllm/core/cost_tracker.py +569 -0
  273. distllm/core/cross_cloud_router.py +924 -0
  274. distllm/core/cross_model_prefix_sharing.py +237 -0
  275. distllm/core/cuda_graph.py +165 -0
  276. distllm/core/debug.py +84 -0
  277. distllm/core/device_registry.py +289 -0
  278. distllm/core/di.py +153 -0
  279. distllm/core/differential_privacy.py +183 -0
  280. distllm/core/distributed_distillation.py +402 -0
  281. distllm/core/distributed_speculative.py +1563 -0
  282. distllm/core/dp_inference/__init__.py +657 -0
  283. distllm/core/dp_inference/accounting.py +144 -0
  284. distllm/core/dp_inference/config.py +139 -0
  285. distllm/core/dp_inference/mechanisms.py +137 -0
  286. distllm/core/dp_inference.py +1016 -0
  287. distllm/core/draft_model_router.py +435 -0
  288. distllm/core/draft_quality_scorer.py +157 -0
  289. distllm/core/draft_tree.py +232 -0
  290. distllm/core/dynamic_memory_budget.py +179 -0
  291. distllm/core/dynamic_sharder.py +364 -0
  292. distllm/core/dynamic_speculation.py +131 -0
  293. distllm/core/evaluation/__init__.py +57 -0
  294. distllm/core/evaluation/constants.py +86 -0
  295. distllm/core/evaluation/db.py +185 -0
  296. distllm/core/evaluation/formatters.py +82 -0
  297. distllm/core/evaluation/loaders.py +320 -0
  298. distllm/core/evaluation/models.py +66 -0
  299. distllm/core/evaluation/report.py +79 -0
  300. distllm/core/evaluation/runner.py +518 -0
  301. distllm/core/evaluation/scorers.py +278 -0
  302. distllm/core/evaluation/worker.py +123 -0
  303. distllm/core/evaluation_harness.py +1530 -0
  304. distllm/core/event_bus.py +516 -0
  305. distllm/core/faas_7b.py +201 -0
  306. distllm/core/feature_flags.py +245 -0
  307. distllm/core/federated_finetuner.py +377 -0
  308. distllm/core/federated_incentives.py +267 -0
  309. distllm/core/framework_adapters.py +390 -0
  310. distllm/core/gaia_cache.py +1330 -0
  311. distllm/core/gossip_cache_bridge.py +202 -0
  312. distllm/core/gpu_power_manager.py +173 -0
  313. distllm/core/gpu_profiler.py +59 -0
  314. distllm/core/gpu_resource_manager.py +338 -0
  315. distllm/core/graceful_degradation.py +264 -0
  316. distllm/core/grammar_constrained.py +549 -0
  317. distllm/core/grammar_constrained_draft.py +91 -0
  318. distllm/core/grammar_decoder.py +285 -0
  319. distllm/core/ha_coordinator.py +452 -0
  320. distllm/core/hardware_plugin_registry.py +126 -0
  321. distllm/core/hashing.py +29 -0
  322. distllm/core/health_manager.py +320 -0
  323. distllm/core/heterogeneous_scheduler.py +663 -0
  324. distllm/core/hibernation_manager.py +250 -0
  325. distllm/core/hierarchical_digest.py +168 -0
  326. distllm/core/hybrid_cache.py +253 -0
  327. distllm/core/hydra_diffusion.py +177 -0
  328. distllm/core/inference_engine.py +1072 -0
  329. distllm/core/intelligent_autoscaler.py +219 -0
  330. distllm/core/kraken_chaos.py +1407 -0
  331. distllm/core/kv_backup.py +217 -0
  332. distllm/core/kv_cache.py +1271 -0
  333. distllm/core/kv_cache_adaptive_quantizer.py +168 -0
  334. distllm/core/kv_cache_compressor.py +290 -0
  335. distllm/core/kv_cache_manager.py +142 -0
  336. distllm/core/kv_cache_marketplace.py +319 -0
  337. distllm/core/kv_cache_metrics.py +205 -0
  338. distllm/core/kv_cache_migration.py +137 -0
  339. distllm/core/kv_cache_paged.py +78 -0
  340. distllm/core/kv_cache_replication.py +107 -0
  341. distllm/core/kv_cache_serialization.py +182 -0
  342. distllm/core/latency_tracker.py +17 -0
  343. distllm/core/leaky_bucket_limiter.py +194 -0
  344. distllm/core/learning_router.py +652 -0
  345. distllm/core/llamacpp_backend.py +23 -0
  346. distllm/core/load_balancer.py +286 -0
  347. distllm/core/media_pipeline.py +387 -0
  348. distllm/core/memory_defragmenter.py +666 -0
  349. distllm/core/metering.py +449 -0
  350. distllm/core/metrics_collector.py +39 -0
  351. distllm/core/micro_batch_scheduler.py +170 -0
  352. distllm/core/model_hub_enhanced.py +382 -0
  353. distllm/core/model_registry.py +9 -0
  354. distllm/core/model_router.py +930 -0
  355. distllm/core/model_selector.py +245 -0
  356. distllm/core/model_sizing.py +334 -0
  357. distllm/core/model_version_manager.py +414 -0
  358. distllm/core/moe_orchestrator.py +205 -0
  359. distllm/core/money.py +53 -0
  360. distllm/core/monitor.py +141 -0
  361. distllm/core/mtp_head.py +444 -0
  362. distllm/core/multi_draft_verifier.py +346 -0
  363. distllm/core/multi_model_serving.py +624 -0
  364. distllm/core/multimodal_engine.py +194 -0
  365. distllm/core/neural_partition_optimizer.py +1398 -0
  366. distllm/core/node_recovery.py +27 -0
  367. distllm/core/notification_manager.py +317 -0
  368. distllm/core/param_update_channel.py +104 -0
  369. distllm/core/pareto_optimizer.py +227 -0
  370. distllm/core/performance_alerts.py +198 -0
  371. distllm/core/performance_baseline.py +252 -0
  372. distllm/core/persistence.py +947 -0
  373. distllm/core/persistent_store.py +485 -0
  374. distllm/core/pipeline_composer.py +211 -0
  375. distllm/core/pipeline_executor.py +500 -0
  376. distllm/core/pipeline_orchestrator.py +17 -0
  377. distllm/core/pipeline_overlap.py +177 -0
  378. distllm/core/placement.py +442 -0
  379. distllm/core/plugin_marketplace.py +483 -0
  380. distllm/core/plugin_sandbox.py +751 -0
  381. distllm/core/plugin_system.py +601 -0
  382. distllm/core/plugins/__init__.py +26 -0
  383. distllm/core/plugins/sandbox.py +38 -0
  384. distllm/core/predictive_cache.py +29 -0
  385. distllm/core/predictive_cache_warming.py +323 -0
  386. distllm/core/predictive_failure.py +163 -0
  387. distllm/core/preemptible_scheduler.py +343 -0
  388. distllm/core/preference_learning.py +606 -0
  389. distllm/core/prefix_cache.py +17 -0
  390. distllm/core/pricing_providers.py +576 -0
  391. distllm/core/priority_heap.py +62 -0
  392. distllm/core/privacy_budget.py +177 -0
  393. distllm/core/prompt_caching_service.py +228 -0
  394. distllm/core/prompt_library.py +838 -0
  395. distllm/core/protocols.py +213 -0
  396. distllm/core/provider_health.py +234 -0
  397. distllm/core/provisioning.py +329 -0
  398. distllm/core/pulse_performance_model.py +1558 -0
  399. distllm/core/quantization_selector.py +210 -0
  400. distllm/core/radix_tree_cache.py +306 -0
  401. distllm/core/rebalancer.py +25 -0
  402. distllm/core/redis_prompt_cache.py +311 -0
  403. distllm/core/replication_controller.py +128 -0
  404. distllm/core/request_auditor.py +215 -0
  405. distllm/core/request_fingerprinting.py +231 -0
  406. distllm/core/request_latency.py +211 -0
  407. distllm/core/request_pipeline.py +810 -0
  408. distllm/core/request_replay.py +253 -0
  409. distllm/core/request_tracker.py +184 -0
  410. distllm/core/resource_manager.py +657 -0
  411. distllm/core/route_audit.py +175 -0
  412. distllm/core/routing_extensions.py +604 -0
  413. distllm/core/schedule_simulator.py +289 -0
  414. distllm/core/schedule_viz.py +233 -0
  415. distllm/core/scheduler/__init__.py +34 -0
  416. distllm/core/scheduler/budget.py +25 -0
  417. distllm/core/scheduler/budget_computer.py +322 -0
  418. distllm/core/scheduler/chunked_prefill.py +52 -0
  419. distllm/core/scheduler/kv_cache_manager.py +313 -0
  420. distllm/core/scheduler/preemption_manager.py +251 -0
  421. distllm/core/scheduler/pressure.py +48 -0
  422. distllm/core/scheduler/sequence.py +179 -0
  423. distllm/core/secret_manager.py +315 -0
  424. distllm/core/semantic_cache.py +369 -0
  425. distllm/core/sentinel_qos.py +833 -0
  426. distllm/core/shadow_eval_runner.py +194 -0
  427. distllm/core/shared_layer_pool.py +279 -0
  428. distllm/core/smart_model_router.py +309 -0
  429. distllm/core/spec_equivalence.py +140 -0
  430. distllm/core/spec_verify.py +225 -0
  431. distllm/core/speculative_adaptor.py +108 -0
  432. distllm/core/speculative_dashboard.py +134 -0
  433. distllm/core/speculative_decoder.py +1286 -0
  434. distllm/core/speculative_profiler.py +132 -0
  435. distllm/core/split_brain.py +236 -0
  436. distllm/core/spot_failover.py +292 -0
  437. distllm/core/spot_forecasting.py +463 -0
  438. distllm/core/starvation_monitor.py +70 -0
  439. distllm/core/state_replication.py +261 -0
  440. distllm/core/stats_collector.py +64 -0
  441. distllm/core/step_processor.py +115 -0
  442. distllm/core/straggler_alerts.py +174 -0
  443. distllm/core/straggler_aware_scheduler.py +255 -0
  444. distllm/core/straggler_detector.py +29 -0
  445. distllm/core/streaming_cost.py +286 -0
  446. distllm/core/streaming_generator.py +228 -0
  447. distllm/core/structured_output/__init__.py +348 -0
  448. distllm/core/structured_output/config.py +38 -0
  449. distllm/core/structured_output/engine.py +367 -0
  450. distllm/core/structured_output/streaming.py +183 -0
  451. distllm/core/structured_output/validator.py +199 -0
  452. distllm/core/subsystem_registry.py +171 -0
  453. distllm/core/synapse_debugger.py +226 -0
  454. distllm/core/synth_data_generator.py +122 -0
  455. distllm/core/telemetry.py +240 -0
  456. distllm/core/tenant_billing.py +446 -0
  457. distllm/core/tenant_cost_attribution.py +314 -0
  458. distllm/core/token_generator.py +368 -0
  459. distllm/core/token_streaming_buffer.py +147 -0
  460. distllm/core/topology_aware_lb.py +324 -0
  461. distllm/core/topology_aware_tiering.py +344 -0
  462. distllm/core/tree_speculative_decoder.py +405 -0
  463. distllm/core/types.py +17 -0
  464. distllm/core/unified_router.py +439 -0
  465. distllm/core/unified_sla_router.py +242 -0
  466. distllm/core/usage_meter.py +915 -0
  467. distllm/core/vectorstore/__init__.py +30 -0
  468. distllm/core/vectorstore/base.py +106 -0
  469. distllm/core/vectorstore/chroma_store.py +99 -0
  470. distllm/core/vectorstore/factory.py +97 -0
  471. distllm/core/vectorstore/pgvector_store.py +123 -0
  472. distllm/core/vectorstore/providers/__init__.py +7 -0
  473. distllm/core/vectorstore/providers/milvus.py +219 -0
  474. distllm/core/vectorstore/providers/pinecone.py +142 -0
  475. distllm/core/vectorstore/providers/qdrant.py +176 -0
  476. distllm/core/vectorstore/providers/weaviate.py +185 -0
  477. distllm/core/vectorstore/qdrant_store.py +148 -0
  478. distllm/core/vectorstore/rag.py +151 -0
  479. distllm/core/vllm_backend.py +21 -0
  480. distllm/core/vllm_node.py +174 -0
  481. distllm/core/voyager_multimodal.py +1354 -0
  482. distllm/core/webgpu_manager.py +382 -0
  483. distllm/core/webhook_formatters.py +243 -0
  484. distllm/core/webhook_manager.py +433 -0
  485. distllm/core/wisp_wasm.py +934 -0
  486. distllm/core/workload_classifier.py +137 -0
  487. distllm/dashboard/__init__.py +17 -0
  488. distllm/dashboard/verification_history.py +192 -0
  489. distllm/dashboard/ws_handler.py +586 -0
  490. distllm/dist/__init__.py +85 -0
  491. distllm/dist/__init__.pyi +132 -0
  492. distllm/dist/_pipeline_original.py +2916 -0
  493. distllm/dist/adaptive_speculator.py +384 -0
  494. distllm/dist/admin_cli.py +293 -0
  495. distllm/dist/api_docs.py +594 -0
  496. distllm/dist/async_pipeline.py +578 -0
  497. distllm/dist/attention.py +1699 -0
  498. distllm/dist/attention_block_pool.py +634 -0
  499. distllm/dist/attention_distributed.py +138 -0
  500. distllm/dist/autoscaler.py +196 -0
  501. distllm/dist/backends/__init__.py +29 -0
  502. distllm/dist/backends/backend_profiles.py +357 -0
  503. distllm/dist/backends/graceful_degradation.py +409 -0
  504. distllm/dist/backends/health_monitor.py +518 -0
  505. distllm/dist/backends/llamacpp.py +125 -0
  506. distllm/dist/backends/ray.py +89 -0
  507. distllm/dist/backends/vllm.py +177 -0
  508. distllm/dist/block_pool.py +662 -0
  509. distllm/dist/block_transfer_service.py +267 -0
  510. distllm/dist/byzantine.py +2193 -0
  511. distllm/dist/cache.py +305 -0
  512. distllm/dist/cache_digest.py +405 -0
  513. distllm/dist/chaos/__init__.py +52 -0
  514. distllm/dist/chaos/chaos_orchestrator.py +229 -0
  515. distllm/dist/chaos/fault_injector.py +1273 -0
  516. distllm/dist/chunked_prefill.py +136 -0
  517. distllm/dist/cloud_selector.py +257 -0
  518. distllm/dist/config.py +70 -0
  519. distllm/dist/cross_cluster.py +231 -0
  520. distllm/dist/daas/__init__.py +46 -0
  521. distllm/dist/daas/load_balancer.py +275 -0
  522. distllm/dist/daas/marketplace_integration.py +374 -0
  523. distllm/dist/daas/resource_isolation.py +280 -0
  524. distllm/dist/daas/tenant_dispatcher.py +252 -0
  525. distllm/dist/daas/usage_meter.py +360 -0
  526. distllm/dist/daas_server.py +514 -0
  527. distllm/dist/disagg/__init__.py +162 -0
  528. distllm/dist/disagg/pool.py +214 -0
  529. distllm/dist/disagg/transfer.py +77 -0
  530. distllm/dist/disagg/transfer_compression.py +793 -0
  531. distllm/dist/disagg/transfer_engine.py +998 -0
  532. distllm/dist/discovery.py +166 -0
  533. distllm/dist/draft_bank.py +344 -0
  534. distllm/dist/draft_migration.py +382 -0
  535. distllm/dist/ebpf_transport.py +364 -0
  536. distllm/dist/edge_cloud.py +544 -0
  537. distllm/dist/edge_federation.py +426 -0
  538. distllm/dist/execution_planner.py +148 -0
  539. distllm/dist/federated_merge.py +560 -0
  540. distllm/dist/federation.py +1114 -0
  541. distllm/dist/flash_attention.py +237 -0
  542. distllm/dist/fsdp.py +295 -0
  543. distllm/dist/geo.py +361 -0
  544. distllm/dist/ice_transport.py +2968 -0
  545. distllm/dist/kv_migration.py +237 -0
  546. distllm/dist/latency.py +60 -0
  547. distllm/dist/lifecycle.py +467 -0
  548. distllm/dist/marketplace.py +891 -0
  549. distllm/dist/merkle.py +178 -0
  550. distllm/dist/metrics.py +456 -0
  551. distllm/dist/model_store.py +107 -0
  552. distllm/dist/multi_tenant.py +339 -0
  553. distllm/dist/multimodal.py +392 -0
  554. distllm/dist/nat.py +934 -0
  555. distllm/dist/nccl.py +733 -0
  556. distllm/dist/network.py +126 -0
  557. distllm/dist/node_client.py +568 -0
  558. distllm/dist/node_pb2.py +66 -0
  559. distllm/dist/node_pb2.pyi +236 -0
  560. distllm/dist/node_pb2_grpc.py +312 -0
  561. distllm/dist/node_pb2_grpc.pyi +41 -0
  562. distllm/dist/node_registrar.py +247 -0
  563. distllm/dist/node_service.py +572 -0
  564. distllm/dist/observability.py +258 -0
  565. distllm/dist/otel.py +158 -0
  566. distllm/dist/p2p/__init__.py +22 -0
  567. distllm/dist/p2p/discovery.py +132 -0
  568. distllm/dist/p2p/gossip.py +1339 -0
  569. distllm/dist/p2p/kademlia_dht.py +1058 -0
  570. distllm/dist/p2p/load_balancer.py +131 -0
  571. distllm/dist/p2p/quic_transport.py +700 -0
  572. distllm/dist/p2p/router.py +607 -0
  573. distllm/dist/p2p/transport.py +399 -0
  574. distllm/dist/p2p_model_distributor.py +160 -0
  575. distllm/dist/paged_attention_kernel.py +281 -0
  576. distllm/dist/parallel.py +1061 -0
  577. distllm/dist/parallel_planner.py +586 -0
  578. distllm/dist/parallel_topology.py +231 -0
  579. distllm/dist/partition/__init__.py +146 -0
  580. distllm/dist/partition/adaptive.py +532 -0
  581. distllm/dist/partition/benchmark_suite.py +479 -0
  582. distllm/dist/partition/cli.py +189 -0
  583. distllm/dist/partition/cloud_arbitrage.py +523 -0
  584. distllm/dist/partition/config.py +97 -0
  585. distllm/dist/partition/cost_model.py +553 -0
  586. distllm/dist/partition/learned_cost.py +653 -0
  587. distllm/dist/partition/mixed_precision_tuner.py +310 -0
  588. distllm/dist/partition/network_cost_model.py +664 -0
  589. distllm/dist/partition/optimizer.py +650 -0
  590. distllm/dist/partition/pareto_optimizer.py +466 -0
  591. distllm/dist/partition/partitioner.py +363 -0
  592. distllm/dist/partition/persistence.py +392 -0
  593. distllm/dist/partition/plugins.py +270 -0
  594. distllm/dist/partition/profiles.py +659 -0
  595. distllm/dist/partition/quant_bench.py +373 -0
  596. distllm/dist/partition/quant_calibrate.py +229 -0
  597. distllm/dist/partition/quant_coordinator.py +260 -0
  598. distllm/dist/partition/quant_cost.py +293 -0
  599. distllm/dist/partition/quant_partition.py +432 -0
  600. distllm/dist/partition/quant_report.py +301 -0
  601. distllm/dist/partition/quantization_metrics.py +355 -0
  602. distllm/dist/partition/quantization_search.py +290 -0
  603. distllm/dist/partition/quantization_tuner.py +1383 -0
  604. distllm/dist/partition/topology.py +218 -0
  605. distllm/dist/partition/validator.py +405 -0
  606. distllm/dist/partition/visualizer.py +275 -0
  607. distllm/dist/pipeline/__init__.py +71 -0
  608. distllm/dist/pipeline/bandwidth_controller.py +780 -0
  609. distllm/dist/pipeline/compression_negotiation.py +771 -0
  610. distllm/dist/pipeline/context.py +80 -0
  611. distllm/dist/pipeline/continuous_batching.py +435 -0
  612. distllm/dist/pipeline/disagg_orchestrator.py +284 -0
  613. distllm/dist/pipeline/orchestrator.py +727 -0
  614. distllm/dist/pipeline/pipeline_reconfig.py +1395 -0
  615. distllm/dist/pipeline/profiler.py +84 -0
  616. distllm/dist/pipeline/serialization.py +176 -0
  617. distllm/dist/pipeline/simulator.py +172 -0
  618. distllm/dist/pipeline/strategy.py +193 -0
  619. distllm/dist/pipeline/token_accumulator.py +211 -0
  620. distllm/dist/pipeline/transport.py +230 -0
  621. distllm/dist/pipeline/wan_disagg_orchestrator.py +564 -0
  622. distllm/dist/power_cap.py +358 -0
  623. distllm/dist/predictive_cache.py +399 -0
  624. distllm/dist/preemption.py +198 -0
  625. distllm/dist/prefix_cache.py +461 -0
  626. distllm/dist/prefix_clustering.py +305 -0
  627. distllm/dist/privacy/__init__.py +411 -0
  628. distllm/dist/privacy/privacy_accountant.py +344 -0
  629. distllm/dist/privacy.py +240 -0
  630. distllm/dist/provisioning.py +338 -0
  631. distllm/dist/quality.py +118 -0
  632. distllm/dist/quic_transport.py +335 -0
  633. distllm/dist/quota_enforcer.py +265 -0
  634. distllm/dist/rebalancer.py +234 -0
  635. distllm/dist/recovery.py +827 -0
  636. distllm/dist/recovery_drill.py +281 -0
  637. distllm/dist/redis_cache.py +265 -0
  638. distllm/dist/redundant.py +308 -0
  639. distllm/dist/redundant_executor.py +1536 -0
  640. distllm/dist/reputation.py +231 -0
  641. distllm/dist/router.py +382 -0
  642. distllm/dist/routing/__init__.py +19 -0
  643. distllm/dist/routing/arbitrage_engine.py +575 -0
  644. distllm/dist/routing/composite.py +179 -0
  645. distllm/dist/routing/consistent_hash.py +144 -0
  646. distllm/dist/routing/latency_aware.py +116 -0
  647. distllm/dist/routing/load_aware.py +114 -0
  648. distllm/dist/scheduling/__init__.py +38 -0
  649. distllm/dist/scheduling/batcher.py +95 -0
  650. distllm/dist/scheduling/classifier.py +153 -0
  651. distllm/dist/scheduling/deadline_scheduler.py +958 -0
  652. distllm/dist/scheduling/iteration.py +404 -0
  653. distllm/dist/scheduling/profiler.py +57 -0
  654. distllm/dist/simulation/__init__.py +54 -0
  655. distllm/dist/simulation/chaos_simulator.py +901 -0
  656. distllm/dist/simulation/cluster_simulator.py +667 -0
  657. distllm/dist/simulation/cost_aware_provisioning.py +370 -0
  658. distllm/dist/simulation/digital_twin.py +617 -0
  659. distllm/dist/simulation/topology_optimizer.py +747 -0
  660. distllm/dist/simulation/what_if.py +492 -0
  661. distllm/dist/speculative/__init__.py +47 -0
  662. distllm/dist/speculative/adaptive_spec.py +291 -0
  663. distllm/dist/speculative/draft_cache.py +246 -0
  664. distllm/dist/speculative/draft_orchestrator.py +634 -0
  665. distllm/dist/speculative/draft_registry.py +439 -0
  666. distllm/dist/speculative/multi_draft.py +304 -0
  667. distllm/dist/speculative/online_sisd.py +523 -0
  668. distllm/dist/straggler.py +819 -0
  669. distllm/dist/streaming_kv_transfer.py +174 -0
  670. distllm/dist/structured_output/__init__.py +7 -0
  671. distllm/dist/structured_output/engine.py +144 -0
  672. distllm/dist/tensor_pool.py +120 -0
  673. distllm/dist/tgi_compat.py +254 -0
  674. distllm/dist/topology.py +141 -0
  675. distllm/dist/topology_consensus.py +1961 -0
  676. distllm/dist/topology_dynamic.py +270 -0
  677. distllm/dist/tp_inprocess.py +214 -0
  678. distllm/dist/tp_launcher.py +310 -0
  679. distllm/dist/tracing.py +325 -0
  680. distllm/dist/transport.py +493 -0
  681. distllm/dist/wan_speculative.py +295 -0
  682. distllm/dist/wandb_integration.py +252 -0
  683. distllm/dist/webrtc.py +846 -0
  684. distllm/dist/wide_area.py +515 -0
  685. distllm/dist/worker.py +635 -0
  686. distllm/dist/zero_copy.py +307 -0
  687. distllm/errors/__init__.py +105 -0
  688. distllm/errors/policies.py +106 -0
  689. distllm/errors/retry.py +168 -0
  690. distllm/errors/types.py +721 -0
  691. distllm/health/__init__.py +6 -0
  692. distllm/health/failover.py +81 -0
  693. distllm/health/prober.py +31 -0
  694. distllm/health/service.py +173 -0
  695. distllm/health/state.py +76 -0
  696. distllm/integrations/__init__.py +72 -0
  697. distllm/integrations/_common/__init__.py +7 -0
  698. distllm/integrations/_common/base_tool_provider.py +189 -0
  699. distllm/integrations/_common/cost_tracker.py +144 -0
  700. distllm/integrations/_common/model_router.py +148 -0
  701. distllm/integrations/ci/__init__.py +24 -0
  702. distllm/integrations/ci/_common.py +92 -0
  703. distllm/integrations/ci/gitlab.py +345 -0
  704. distllm/integrations/ci/jenkins.py +273 -0
  705. distllm/integrations/mlflow_tracking.py +372 -0
  706. distllm/integrations/spark_connector.py +909 -0
  707. distllm/integrations/wandb/__init__.py +81 -0
  708. distllm/models/__init__.py +40 -0
  709. distllm/models/_trust_remote.py +48 -0
  710. distllm/models/adapter.py +768 -0
  711. distllm/models/adapter_router.py +198 -0
  712. distllm/models/cache.py +172 -0
  713. distllm/models/model_hub.py +650 -0
  714. distllm/models/partition_planner.py +384 -0
  715. distllm/models/partitioner.py +784 -0
  716. distllm/models/rope_scaling.py +119 -0
  717. distllm/models/safetensors_index.py +179 -0
  718. distllm/observability/__init__.py +107 -0
  719. distllm/observability/alerting_config.py +881 -0
  720. distllm/observability/capacity_models.py +398 -0
  721. distllm/observability/capacity_planning.py +2870 -0
  722. distllm/observability/exporter.py +228 -0
  723. distllm/observability/exporter_datadog.py +1362 -0
  724. distllm/observability/health_config.py +1282 -0
  725. distllm/observability/incident_response.py +779 -0
  726. distllm/observability/logging.py +211 -0
  727. distllm/observability/logging_config.py +864 -0
  728. distllm/observability/loki_sink.py +178 -0
  729. distllm/observability/metrics.py +111 -0
  730. distllm/observability/metrics_completeness.py +393 -0
  731. distllm/observability/otel_logging.py +290 -0
  732. distllm/observability/self_healing_config.py +1308 -0
  733. distllm/observability/slo_config.py +1520 -0
  734. distllm/observability/spans.py +235 -0
  735. distllm/observability/tracing.py +268 -0
  736. distllm/observability/tracing_config.py +676 -0
  737. distllm/observability/wandb_monitor.py +66 -0
  738. distllm/plugins/__init__.py +14 -0
  739. distllm/plugins/airflow.py +110 -0
  740. distllm/plugins/auth_plugin.py +576 -0
  741. distllm/plugins/builtin.py +376 -0
  742. distllm/plugins/cache_plugin.py +576 -0
  743. distllm/plugins/health_plugin.py +396 -0
  744. distllm/plugins/kubeflow.py +84 -0
  745. distllm/plugins/mlflow_plugin.py +71 -0
  746. distllm/plugins/registry.py +135 -0
  747. distllm/prompts/__init__.py +39 -0
  748. distllm/prompts/analysis.py +103 -0
  749. distllm/prompts/code.py +172 -0
  750. distllm/prompts/creative.py +115 -0
  751. distllm/prompts/education.py +91 -0
  752. distllm/prompts/engine.py +119 -0
  753. distllm/prompts/exchange.py +705 -0
  754. distllm/prompts/language.py +81 -0
  755. distllm/prompts/library.py +1198 -0
  756. distllm/prompts/management.py +35 -0
  757. distllm/prompts/productivity.py +86 -0
  758. distllm/prompts/professional.py +111 -0
  759. distllm/prompts/prompt_def.py +33 -0
  760. distllm/prompts/reasoning.py +108 -0
  761. distllm/prompts/specialized.py +140 -0
  762. distllm/prompts/templates.py +128 -0
  763. distllm/prompts/writing.py +140 -0
  764. distllm/py.typed +0 -0
  765. distllm/sdk/__init__.py +82 -0
  766. distllm/sdk/circuit_breaker.py +139 -0
  767. distllm/sdk/client.py +1191 -0
  768. distllm/sdk/observability.py +42 -0
  769. distllm/sdk/streaming.py +228 -0
  770. distllm/sdk/transport.py +224 -0
  771. distllm/sdk/types.py +288 -0
  772. distllm/security/__init__.py +63 -0
  773. distllm/security/content_moderation/__init__.py +44 -0
  774. distllm/security/content_moderation/base.py +405 -0
  775. distllm/security/content_moderation/jailbreak.py +192 -0
  776. distllm/security/content_moderation/pii.py +262 -0
  777. distllm/security/content_moderation/pipeline.py +272 -0
  778. distllm/security/content_moderation/topics.py +153 -0
  779. distllm/security/content_moderation/toxicity.py +131 -0
  780. distllm/security/e2e.py +437 -0
  781. distllm/security/edge_attestation.py +381 -0
  782. distllm/security/log_redaction.py +244 -0
  783. distllm/security/quantum_safe_tls.py +314 -0
  784. distllm/security/spiffe.py +347 -0
  785. distllm/security/tee.py +461 -0
  786. distllm/security/utils.py +139 -0
  787. distllm/security/watermark.py +779 -0
  788. distllm/terraform/__init__.py +260 -0
  789. distllm/ui/__init__.py +7 -0
  790. distllm/ui/app.py +98 -0
  791. distllm/utils/__init__.py +0 -0
  792. distllm/utils/gbnf_grammar.py +99 -0
  793. distllm/utils/lazy_imports.py +90 -0
  794. distllm/utils/scheduling.py +38 -0
  795. distllm/verification/__init__.py +49 -0
  796. distllm/verification/comparator.py +263 -0
  797. distllm/verification/hash_registry.py +220 -0
  798. distllm/verification/report.py +240 -0
  799. distllm/verification/runner.py +822 -0
  800. distllm/verification/test_verify.py +205 -0
  801. distributed_llm-0.4.1.dist-info/METADATA +315 -0
  802. distributed_llm-0.4.1.dist-info/RECORD +806 -0
  803. distributed_llm-0.4.1.dist-info/WHEEL +5 -0
  804. distributed_llm-0.4.1.dist-info/entry_points.txt +5 -0
  805. distributed_llm-0.4.1.dist-info/licenses/LICENSE +97 -0
  806. distributed_llm-0.4.1.dist-info/top_level.txt +1 -0
@@ -0,0 +1,984 @@
1
+ """API versioning: Accept-Version header negotiation, deprecation scheduling,
2
+ field-level compatibility mapping, and a middleware that enforces the
3
+ version contract at the HTTP layer.
4
+
5
+ Architecture
6
+ ------------
7
+ The versioning system has four components that work together:
8
+
9
+ 1. **APIVersion** (enum) — Single source of truth for supported API versions.
10
+ Versions are date-based strings (e.g. ``"2026-01"``) following the
11
+ ``YYYY-MM`` convention, not ordinal integers.
12
+
13
+ 2. **VersionNegotiator** — Determines which API version a request targets by
14
+ checking, in order:
15
+ a. The ``Accept-Version`` HTTP header
16
+ b. The URL path prefix (``/v1/``, ``/v2/``)
17
+ c. A configured default version (fallback)
18
+ Returns the resolved version plus optional deprecation warnings.
19
+
20
+ 3. **DeprecationSchedule** — Tracks sunset dates per version and provides
21
+ migration path descriptions so clients know what to change.
22
+
23
+ 4. **VersionCompatibilityLayer** — Bidirectional field-level mapping between
24
+ versioned request/response shapes. Allows the server to accept old-format
25
+ payloads and return old-format responses even when the internal handler
26
+ has moved to a newer schema.
27
+
28
+ 5. **VersionMiddleware** (Starlette ``BaseHTTPMiddleware``) — Wires the
29
+ negotiator, schedule, and compatibility layer into the request pipeline:
30
+ - Extracts the version from every incoming request and stores it on
31
+ ``request.state.api_version``.
32
+ - Adds ``X-API-Version``, ``Sunset``, and ``X-API-Deprecation`` headers
33
+ to every response.
34
+ - Returns **410 Gone** for requests targeting a fully deprecated version
35
+ whose sunset date has passed.
36
+ - Optionally transforms request/response bodies through the compatibility
37
+ layer when the request version does not match the handler's internal
38
+ version.
39
+ """
40
+
41
+ from __future__ import annotations
42
+
43
+ import datetime
44
+ import re
45
+ from dataclasses import dataclass, field
46
+ from enum import Enum
47
+ from typing import Any, Callable
48
+
49
+ from fastapi import Request, Response
50
+ from loguru import logger
51
+ from starlette.middleware.base import BaseHTTPMiddleware
52
+
53
+
54
+ # ── Constants ─────────────────────────────────────────────────────────────────
55
+
56
+ SUNSET_HEADER = "Sunset"
57
+ DEPRECATION_HEADER = "X-API-Deprecation"
58
+ VERSION_HEADER = "X-API-Version"
59
+ ACCEPT_VERSION_HEADER = "Accept-Version"
60
+
61
+ # Default version used when no version indication is present.
62
+ DEFAULT_API_VERSION = "2026-07"
63
+
64
+ # Warning threshold: a version is considered "near sunset" when fewer than
65
+ # this many days remain before its sunset date.
66
+ NEAR_SUNSET_DAYS = 60
67
+
68
+
69
+ # ── Version Enum ──────────────────────────────────────────────────────────────
70
+
71
+
72
+ class APIVersion(str, Enum):
73
+ """Canonical API version identifiers (date-based).
74
+
75
+ Values are ``YYYY-MM`` strings so that ordering is lexicographically
76
+ equivalent to chronological ordering.
77
+
78
+ Add a new member each time the API surface changes in a way that is not
79
+ backward-compatible.
80
+ """
81
+
82
+ V1 = "2026-01"
83
+ V2 = "2026-07"
84
+
85
+ @classmethod
86
+ def from_string(cls, value: str) -> APIVersion | None:
87
+ """Parse a string into an ``APIVersion`` member, returning ``None`` on
88
+ mismatch."""
89
+ value = value.strip()
90
+ for member in cls:
91
+ if member.value == value:
92
+ return member
93
+ return None
94
+
95
+ @classmethod
96
+ def from_path_prefix(cls, path: str) -> APIVersion | None:
97
+ """Extract an API version from a URL path like ``/v1/...`` or ``/v2/...``.
98
+
99
+ Returns ``None`` when the path carries no version prefix (e.g. unversioned
100
+ endpoints such as ``/health`` or ``/metrics``).
101
+ """
102
+ match = re.match(r"^/(v\d+)/", path)
103
+ if match is None:
104
+ return None
105
+ prefix = match.group(1)
106
+ mapping: dict[str, APIVersion] = {
107
+ "v1": APIVersion.V1,
108
+ "v2": APIVersion.V2,
109
+ }
110
+ return mapping.get(prefix)
111
+
112
+ def to_path_prefix(self) -> str:
113
+ """Return the URL path prefix for this version (e.g. ``/v1``)."""
114
+ ordinal: dict[APIVersion, str] = {
115
+ APIVersion.V1: "v1",
116
+ APIVersion.V2: "v2",
117
+ }
118
+ return f"/{ordinal[self]}"
119
+
120
+
121
+ # ── Version Negotiation Result ────────────────────────────────────────────────
122
+
123
+
124
+ @dataclass(frozen=True)
125
+ class VersionNegotiationResult:
126
+ """The outcome of version negotiation for a single request.
127
+
128
+ Attributes
129
+ ----------
130
+ version:
131
+ The resolved ``APIVersion``, or ``None`` when the request could not
132
+ be matched (caller should reject).
133
+ source:
134
+ How the version was determined: ``"header"``, ``"path"``, or
135
+ ``"default"``.
136
+ warnings:
137
+ Human-readable deprecation warnings to emit as response headers.
138
+ """
139
+
140
+ version: APIVersion | None
141
+ source: str
142
+ warnings: list[str] = field(default_factory=list)
143
+
144
+
145
+ # ── Version Negotiator ────────────────────────────────────────────────────────
146
+
147
+
148
+ class VersionNegotiator:
149
+ """Resolves the API version for an incoming request.
150
+
151
+ Negotiation order:
152
+
153
+ 1. **Accept-Version header** — exact match against ``APIVersion`` values.
154
+ 2. **URL path prefix** — ``/v1/``, ``/v2/``.
155
+ 3. **Default** — configured fallback version.
156
+
157
+ Parameters
158
+ ----------
159
+ default_version:
160
+ Fallback version when no version indication is present.
161
+ """
162
+
163
+ def __init__(self, default_version: str = DEFAULT_API_VERSION) -> None:
164
+ self._default = APIVersion.from_string(default_version)
165
+ if self._default is None:
166
+ logger.warning(
167
+ "Default version %r is not a known APIVersion; falling back to %s",
168
+ default_version,
169
+ DEFAULT_API_VERSION,
170
+ )
171
+ self._default = APIVersion.from_string(DEFAULT_API_VERSION)
172
+ assert self._default is not None
173
+
174
+ # ── Public API ────────────────────────────────────────────────────────
175
+
176
+ def negotiate(self, request: Request) -> VersionNegotiationResult:
177
+ """Determine the API version for *request*.
178
+
179
+ Returns a frozen ``VersionNegotiationResult``. When the result's
180
+ ``version`` is ``None`` the version could not be determined and the
181
+ caller should respond with a client error.
182
+ """
183
+ # 1. Accept-Version header
184
+ header_val = request.headers.get(ACCEPT_VERSION_HEADER)
185
+ if header_val:
186
+ parsed = self._parse_accept_version(header_val)
187
+ if parsed is not None:
188
+ return VersionNegotiationResult(
189
+ version=parsed,
190
+ source="header",
191
+ warnings=[],
192
+ )
193
+ # Header present but unparseable — warn but continue to path.
194
+ logger.debug("Unparseable Accept-Version header: %r", header_val)
195
+
196
+ # 2. URL path prefix
197
+ path = request.url.path
198
+ path_version = APIVersion.from_path_prefix(path)
199
+ if path_version is not None:
200
+ return VersionNegotiationResult(
201
+ version=path_version,
202
+ source="path",
203
+ warnings=[],
204
+ )
205
+
206
+ # 3. Default fallback
207
+ return VersionNegotiationResult(
208
+ version=self._default,
209
+ source="default",
210
+ warnings=[],
211
+ )
212
+
213
+ # ── Internals ─────────────────────────────────────────────────────────
214
+
215
+ @staticmethod
216
+ def _parse_accept_version(header_value: str) -> APIVersion | None:
217
+ """Try to parse a raw ``Accept-Version`` header into an ``APIVersion``.
218
+
219
+ The header may be a bare date (``2026-01``) or a qualified form
220
+ (``api=2026-01``). Returns ``None`` when no member matches.
221
+ """
222
+ cleaned = header_value.strip()
223
+ # Accept \"api=2026-01\" or \"2026-01\"
224
+ for sep in ("api=", "version=", ""):
225
+ candidate = cleaned.removeprefix(sep)
226
+ candidate = candidate.strip()
227
+ parsed = APIVersion.from_string(candidate)
228
+ if parsed is not None:
229
+ return parsed
230
+ return None
231
+
232
+
233
+ # ── Deprecation Schedule ──────────────────────────────────────────────────────
234
+
235
+
236
+ @dataclass(frozen=True)
237
+ class MigrationStep:
238
+ """A single action a client must take to migrate from one version to another.
239
+
240
+ Attributes
241
+ ----------
242
+ description:
243
+ Human-readable instruction describing what changed and what the
244
+ client should do.
245
+ field:
246
+ The request or response field affected (e.g. ``"request.model"``).
247
+ Empty string when the step is not field-specific.
248
+ breaking:
249
+ Whether this step represents a breaking change.
250
+ """
251
+
252
+ description: str
253
+ field: str = ""
254
+ breaking: bool = True
255
+
256
+
257
+ class DeprecationSchedule:
258
+ """Manages sunset dates and migration paths for API versions.
259
+
260
+ Parameters
261
+ ----------
262
+ sunset_dates:
263
+ Mapping from ``APIVersion`` to an optional ``datetime.date``.
264
+ When the date is ``None`` the version has no planned sunset.
265
+ migration_steps:
266
+ Mapping from ``(from_version, to_version)`` to a list of
267
+ ``MigrationStep`` instances describing what the client must change.
268
+ """
269
+
270
+ def __init__(
271
+ self,
272
+ sunset_dates: dict[APIVersion, datetime.date | None] | None = None,
273
+ migration_steps: dict[tuple[APIVersion, APIVersion], list[MigrationStep]]
274
+ | None = None,
275
+ ) -> None:
276
+ self._sunset_dates: dict[APIVersion, datetime.date | None] = (
277
+ dict(sunset_dates) if sunset_dates else {}
278
+ )
279
+ self._migration_steps: dict[
280
+ tuple[APIVersion, APIVersion], list[MigrationStep]
281
+ ] = (dict(migration_steps) if migration_steps else {})
282
+
283
+ # ── Public API ────────────────────────────────────────────────────────
284
+
285
+ @classmethod
286
+ def default_schedule(cls) -> DeprecationSchedule:
287
+ """Return the built-in deprecation schedule.
288
+
289
+ Current schedule (as of 2026-07):
290
+ - V1 (2026-01): sunset 2026-10-01, migrate to V2.
291
+ - V2 (2026-07): no sunset (latest stable).
292
+ """
293
+ sunset = {
294
+ APIVersion.V1: datetime.date(2026, 10, 1),
295
+ APIVersion.V2: None, # latest stable — no sunset
296
+ }
297
+
298
+ migration: dict[tuple[APIVersion, APIVersion], list[MigrationStep]] = {
299
+ (APIVersion.V1, APIVersion.V2): [
300
+ MigrationStep(
301
+ description=(
302
+ "Response object type changed from "
303
+ "``chat.completion`` to ``chat.completion.v2``."
304
+ ),
305
+ field="response.object",
306
+ breaking=True,
307
+ ),
308
+ MigrationStep(
309
+ description=(
310
+ "Response includes ``system_fingerprint`` field."
311
+ ),
312
+ field="response.system_fingerprint",
313
+ breaking=False,
314
+ ),
315
+ MigrationStep(
316
+ description=(
317
+ "Request field ``user`` is now validated as a "
318
+ "non-empty string."
319
+ ),
320
+ field="request.user",
321
+ breaking=False,
322
+ ),
323
+ MigrationStep(
324
+ description=(
325
+ "Removed deprecated ``logit_bias`` field; use "
326
+ "``logprobs`` instead."
327
+ ),
328
+ field="request.logit_bias",
329
+ breaking=True,
330
+ ),
331
+ ],
332
+ }
333
+
334
+ return cls(sunset_dates=sunset, migration_steps=migration)
335
+
336
+ def is_deprecated(self, version: APIVersion) -> bool:
337
+ """Return ``True`` when *version* has a sunset date in the past."""
338
+ sunset = self._sunset_dates.get(version)
339
+ if sunset is None:
340
+ return False
341
+ return datetime.date.today() >= sunset
342
+
343
+ def is_near_sunset(self, version: APIVersion, days: int = NEAR_SUNSET_DAYS) -> bool:
344
+ """Return ``True`` when *version* will sunset within *days* days."""
345
+ sunset = self._sunset_dates.get(version)
346
+ if sunset is None:
347
+ return False
348
+ remaining = (sunset - datetime.date.today()).days
349
+ return 0 <= remaining <= days
350
+
351
+ def days_until_sunset(self, version: APIVersion) -> int | None:
352
+ """Return the number of days until *version* is sunset.
353
+
354
+ Returns ``None`` when the version has no sunset date or the sunset
355
+ has already passed.
356
+ """
357
+ sunset = self._sunset_dates.get(version)
358
+ if sunset is None:
359
+ return None
360
+ remaining = (sunset - datetime.date.today()).days
361
+ return remaining if remaining >= 0 else None
362
+
363
+ def get_sunset_date(self, version: APIVersion) -> datetime.date | None:
364
+ """Return the sunset date for *version*, or ``None``."""
365
+ return self._sunset_dates.get(version)
366
+
367
+ def sunset_header_value(self, version: APIVersion) -> str | None:
368
+ """Return an HTTP-date string suitable for the ``Sunset`` header.
369
+
370
+ Returns ``None`` when the version has no sunset.
371
+ """
372
+ sunset = self._sunset_dates.get(version)
373
+ if sunset is None:
374
+ return None
375
+ # RFC 7231 / HTTP-date: IMF-fixdate
376
+ return sunset.strftime("%a, %d %b %Y %H:%M:%S GMT")
377
+
378
+ def get_migration_path(
379
+ self,
380
+ current_version: APIVersion,
381
+ target_version: APIVersion,
382
+ ) -> list[MigrationStep]:
383
+ """Return the ordered list of migration steps to go from
384
+ *current_version* to *target_version*.
385
+
386
+ If no explicit path exists, returns a single generic step advising the
387
+ client to consult the changelog.
388
+ """
389
+ path = self._migration_steps.get((current_version, target_version))
390
+ if path is not None:
391
+ return list(path)
392
+
393
+ # Generic fallback.
394
+ return [
395
+ MigrationStep(
396
+ description=(
397
+ f"No explicit migration path documented for "
398
+ f"{current_version.value} -> {target_version.value}. "
399
+ f"See https://docs.distllm.dev/api-versions for details."
400
+ ),
401
+ breaking=True,
402
+ ),
403
+ ]
404
+
405
+ def auto_deprecate(self, now: datetime.date | None = None) -> list[APIVersion]:
406
+ """Check all versions and return those whose sunset date has passed.
407
+
408
+ This is intended to be called periodically (e.g. via a background
409
+ task or during server startup) so that the server can log or reject
410
+ versions whose grace period has fully elapsed.
411
+
412
+ Parameters
413
+ ----------
414
+ now:
415
+ Reference date. Defaults to ``datetime.date.today()``.
416
+ """
417
+ if now is None:
418
+ now = datetime.date.today()
419
+ expired: list[APIVersion] = []
420
+ for version, sunset in self._sunset_dates.items():
421
+ if sunset is not None and now >= sunset:
422
+ expired.append(version)
423
+ return expired
424
+
425
+
426
+ # ── Version Compatibility Layer ───────────────────────────────────────────────
427
+
428
+
429
+ @dataclass
430
+ class FieldMapping:
431
+ """Describes how a field is represented in two different API versions.
432
+
433
+ Attributes
434
+ ----------
435
+ source_field:
436
+ Dot-separated path of the field in the source version
437
+ (e.g. ``"response.choices[].finish_reason"``).
438
+ target_field:
439
+ Dot-separated path of the field in the target version.
440
+ transform:
441
+ Optional callable that converts the source value to the target
442
+ representation. When ``None`` the value is copied as-is.
443
+ """
444
+
445
+ source_field: str
446
+ target_field: str
447
+ transform: Callable[[Any], Any] | None = None
448
+
449
+
450
+ class VersionCompatibilityLayer:
451
+ """Bidirectional field-level mapping between API versions.
452
+
453
+ The layer stores a set of ``FieldMapping`` instances for each version
454
+ pair and direction (request / response). When transforming, it walks
455
+ the source dict and applies every registered mapping.
456
+
457
+ Usage::
458
+
459
+ layer = VersionCompatibilityLayer()
460
+ layer.add_mapping(
461
+ from_version=APIVersion.V1,
462
+ to_version=APIVersion.V2,
463
+ field_maps=[
464
+ FieldMapping("request.logit_bias", "request.logprobs"),
465
+ FieldMapping(
466
+ "response.object",
467
+ "response.object",
468
+ transform=lambda _: "chat.completion.v2",
469
+ ),
470
+ ],
471
+ )
472
+
473
+ # Upgrade an old V1 request to V2 format:
474
+ v2_data = layer.transform_request(v1_data, APIVersion.V1, APIVersion.V2)
475
+
476
+ # Downgrade a V2 response back to V1 format:
477
+ v1_data = layer.transform_response(v2_data, APIVersion.V2, APIVersion.V1)
478
+ """
479
+
480
+ def __init__(self) -> None:
481
+ # _request_mappings: (from_ver, to_ver) -> list[FieldMapping]
482
+ self._request_mappings: dict[
483
+ tuple[APIVersion, APIVersion], list[FieldMapping]
484
+ ] = {}
485
+ # _response_mappings: (from_ver, to_ver) -> list[FieldMapping]
486
+ self._response_mappings: dict[
487
+ tuple[APIVersion, APIVersion], list[FieldMapping]
488
+ ] = {}
489
+
490
+ # ── Registration ──────────────────────────────────────────────────────
491
+
492
+ def add_mapping(
493
+ self,
494
+ from_version: APIVersion,
495
+ to_version: APIVersion,
496
+ field_maps: list[FieldMapping],
497
+ *,
498
+ direction: str = "both",
499
+ ) -> None:
500
+ """Register field mappings between two versions.
501
+
502
+ Parameters
503
+ ----------
504
+ from_version:
505
+ The source version of the data.
506
+ to_version:
507
+ The target version to transform into.
508
+ field_maps:
509
+ One or more ``FieldMapping`` descriptors.
510
+ direction:
511
+ ``"request"``, ``"response"``, or ``"both"`` (default).
512
+ """
513
+ if direction in ("request", "both"):
514
+ self._request_mappings.setdefault(
515
+ (from_version, to_version), []
516
+ ).extend(field_maps)
517
+ if direction in ("response", "both"):
518
+ self._response_mappings.setdefault(
519
+ (from_version, to_version), []
520
+ ).extend(field_maps)
521
+
522
+ # ── Transformation ────────────────────────────────────────────────────
523
+
524
+ def transform_request(
525
+ self,
526
+ data: dict[str, Any],
527
+ from_version: APIVersion,
528
+ to_version: APIVersion,
529
+ ) -> dict[str, Any]:
530
+ """Transform a request body from *from_version* schema to
531
+ *to_version* schema.
532
+
533
+ Returns a new dict; the original is not mutated.
534
+ """
535
+ mappings = self._request_mappings.get((from_version, to_version))
536
+ if mappings is None:
537
+ return dict(data)
538
+ return self._apply_mappings(data, mappings)
539
+
540
+ def transform_response(
541
+ self,
542
+ data: dict[str, Any],
543
+ from_version: APIVersion,
544
+ to_version: APIVersion,
545
+ ) -> dict[str, Any]:
546
+ """Transform a response body from *from_version* schema to
547
+ *to_version* schema.
548
+
549
+ Returns a new dict; the original is not mutated.
550
+ """
551
+ mappings = self._response_mappings.get((from_version, to_version))
552
+ if mappings is None:
553
+ return dict(data)
554
+ return self._apply_mappings(data, mappings)
555
+
556
+ # ── Internals ─────────────────────────────────────────────────────────
557
+
558
+ @staticmethod
559
+ def _apply_mappings(
560
+ data: dict[str, Any],
561
+ mappings: list[FieldMapping],
562
+ ) -> dict[str, Any]:
563
+ """Apply a list of ``FieldMapping`` entries to a nested dict.
564
+
565
+ The implementation uses dot-separated paths for field access.
566
+ Array indexes are represented by ``[]`` segments (e.g.
567
+ ``"choices[].finish_reason"``), meaning the mapping applies to
568
+ every element of the array.
569
+ """
570
+ result = dict(data)
571
+
572
+ for mapping in mappings:
573
+ source_value = _get_nested(result, mapping.source_field)
574
+ if source_value is _MISSING:
575
+ continue
576
+
577
+ transformed = (
578
+ mapping.transform(source_value)
579
+ if mapping.transform is not None
580
+ else source_value
581
+ )
582
+
583
+ result = _set_nested(result, mapping.target_field, transformed)
584
+
585
+ return result
586
+
587
+
588
+ # ── Sentinel for missing values ───────────────────────────────────────────────
589
+
590
+
591
+ class _Missing:
592
+ """Sentinel to distinguish ``None`` from absent."""
593
+
594
+ def __repr__(self) -> str:
595
+ return "<MISSING>"
596
+
597
+
598
+ _MISSING = _Missing()
599
+
600
+
601
+ def _get_nested(d: dict[str, Any], path: str) -> Any:
602
+ """Retrieve a value from a nested dict using a dot-separated *path*.
603
+
604
+ Supports array segments with ``[]`` to apply a lookup across all
605
+ elements of a list (returns the first matched value, since transforms
606
+ are uniform across elements).
607
+ """
608
+ parts = _split_path(path)
609
+ current: Any = d
610
+ for part in parts:
611
+ if part == "[]" and isinstance(current, list):
612
+ # Apply the remainder of the path to every element; return
613
+ # the value from the first element that has it.
614
+ remainder = ".".join(parts[parts.index("[]") + 1 :])
615
+ for item in current:
616
+ if isinstance(item, dict):
617
+ val = _get_nested(item, remainder)
618
+ if val is not _MISSING:
619
+ return val
620
+ return _MISSING
621
+ if isinstance(current, dict):
622
+ current = current.get(part, _MISSING)
623
+ else:
624
+ return _MISSING
625
+ if current is _MISSING:
626
+ return _MISSING
627
+ return current
628
+
629
+
630
+ def _set_nested(d: dict[str, Any], path: str, value: Any) -> dict[str, Any]:
631
+ """Set a value in a nested dict, returning a new dict (shallow copy at
632
+ each level). Does not mutate *d*.
633
+
634
+ Supports array segments with ``[]`` to set the value on every element
635
+ of a list.
636
+ """
637
+ parts = _split_path(path)
638
+ if not parts:
639
+ return d
640
+
641
+ result = dict(d)
642
+ current = result
643
+
644
+ for i, part in enumerate(parts):
645
+ if part == "[]" and isinstance(current, list):
646
+ remainder = ".".join(parts[i + 1 :])
647
+ new_list = []
648
+ for item in current:
649
+ if isinstance(item, dict):
650
+ new_list.append(
651
+ _set_nested(item, remainder, value)
652
+ if remainder
653
+ else value
654
+ )
655
+ else:
656
+ new_list.append(item)
657
+ # Walk back to parent to set the modified list.
658
+ # For simplicity, we mutate result directly at the known path.
659
+ # Full immutable walk is expensive; we accept shallow mutation
660
+ # of the list container since it is local to this function and
661
+ # never exposed.
662
+ _set_nested_impl(result, parts[:i], new_list)
663
+ return result
664
+
665
+ if i == len(parts) - 1:
666
+ # Last part — set the value.
667
+ if isinstance(current, dict):
668
+ current[part] = value
669
+ else:
670
+ if part not in current:
671
+ current[part] = {}
672
+ next_current = current.get(part)
673
+ if isinstance(next_current, dict):
674
+ current = next_current
675
+ elif isinstance(next_current, list):
676
+ # Enter list context for the next iteration.
677
+ current = next_current
678
+ else:
679
+ # Intermediate missing — create dict.
680
+ current[part] = {}
681
+ current = current[part]
682
+
683
+ return result
684
+
685
+
686
+ def _set_nested_impl(d: dict[str, Any], path_parts: list[str], value: Any) -> None:
687
+ """Mutate *d* at *path_parts* — used internally by ``_set_nested`` for
688
+ the array-branch case where immutability is already guaranteed by the
689
+ caller."""
690
+ current = d
691
+ for i, part in enumerate(path_parts):
692
+ if i == len(path_parts) - 1:
693
+ current[part] = value
694
+ else:
695
+ next_val = current.get(part)
696
+ if isinstance(next_val, dict):
697
+ current = next_val
698
+ else:
699
+ current[part] = {}
700
+ current = current[part]
701
+
702
+
703
+ def _split_path(path: str) -> list[str]:
704
+ """Split a dot-separated path into segments, preserving ``[]`` as a
705
+ single token."""
706
+ segments: list[str] = []
707
+ for segment in path.split("."):
708
+ if "[]" in segment:
709
+ # Handle both "choices[]" and "choices.[]"
710
+ parts = segment.split("[]")
711
+ for p in parts:
712
+ if p:
713
+ segments.append(p)
714
+ segments.append("[]")
715
+ # Pop the trailing [] if the path ended with it and we added it
716
+ if segments and segments[-1] == "[]" and not path.endswith("[]"):
717
+ segments.pop()
718
+ else:
719
+ segments.append(segment)
720
+ return segments
721
+
722
+
723
+ # ── Version Middleware ────────────────────────────────────────────────────────
724
+
725
+
726
+ class VersionMiddleware(BaseHTTPMiddleware):
727
+ """Starlette middleware that enforces the API versioning contract.
728
+
729
+ What it does for every request:
730
+
731
+ 1. Calls ``VersionNegotiator.negotiate()`` to determine the target
732
+ version from the ``Accept-Version`` header or URL path.
733
+ 2. Stores the resolved version on ``request.state.api_version``.
734
+ 3. If the resolved version is fully deprecated (past sunset), responds
735
+ with **410 Gone**.
736
+ 4. If the resolved version is *near* sunset (within 60 days), adds a
737
+ ``Warning`` response header in addition to the standard deprecation
738
+ headers.
739
+ 5. Appends ``X-API-Version``, ``Sunset``, and ``X-API-Deprecation``
740
+ headers to every response.
741
+ 6. Optionally transforms request bodies from the negotiated version to
742
+ the server's internal version, and response bodies back.
743
+
744
+ Parameters
745
+ ----------
746
+ negotiator:
747
+ The ``VersionNegotiator`` instance. Defaults to a fresh negotiator.
748
+ schedule:
749
+ The ``DeprecationSchedule`` instance. Defaults to
750
+ ``DeprecationSchedule.default_schedule()``.
751
+ compatibility_layer:
752
+ Optional ``VersionCompatibilityLayer`` for request/response body
753
+ transformation.
754
+ internal_version:
755
+ The server's internal version. Requests at other versions are
756
+ transformed to this version before reaching the route handler, and
757
+ responses are transformed back. Defaults to ``APIVersion.V2``.
758
+ reject_deprecated:
759
+ When ``True`` (default), requests to fully deprecated versions
760
+ receive a 410 response. Set to ``False`` to only warn (useful
761
+ during a migration grace period).
762
+ """
763
+
764
+ def __init__(
765
+ self,
766
+ app: Any,
767
+ negotiator: VersionNegotiator | None = None,
768
+ schedule: DeprecationSchedule | None = None,
769
+ compatibility_layer: VersionCompatibilityLayer | None = None,
770
+ internal_version: APIVersion = APIVersion.V2,
771
+ reject_deprecated: bool = True,
772
+ ) -> None:
773
+ super().__init__(app)
774
+ self._negotiator = negotiator or VersionNegotiator()
775
+ self._schedule = schedule or DeprecationSchedule.default_schedule()
776
+ self._compat = compatibility_layer
777
+ self._internal_version = internal_version
778
+ self._reject_deprecated = reject_deprecated
779
+
780
+ # ── Middleware Dispatch ───────────────────────────────────────────────
781
+
782
+ async def dispatch(self, request: Request, call_next: Callable) -> Response:
783
+ # 1. Negotiate version.
784
+ result = self._negotiator.negotiate(request)
785
+
786
+ if result.version is None:
787
+ return Response(
788
+ status_code=400,
789
+ content=(
790
+ "Could not determine API version. "
791
+ "Set the Accept-Version header or use a versioned path "
792
+ "(e.g. /v1/...)."
793
+ ),
794
+ media_type="text/plain",
795
+ )
796
+
797
+ # 2. Store version on request state for downstream use.
798
+ request.state.api_version = result.version
799
+ request.state.api_version_source = result.source
800
+
801
+ # 3. Check full deprecation (past sunset).
802
+ if self._reject_deprecated and self._schedule.is_deprecated(result.version):
803
+ sunset = self._schedule.get_sunset_date(result.version)
804
+ logger.warning(
805
+ "Rejected request to deprecated version %s (sunset %s) from %s",
806
+ result.version.value,
807
+ sunset,
808
+ request.client.host if request.client else "unknown",
809
+ )
810
+ return Response(
811
+ status_code=410,
812
+ content=(
813
+ f"API version {result.version.value} is no longer supported. "
814
+ f"It was removed on {sunset}. "
815
+ f"Please migrate to a newer version. "
816
+ f"See https://docs.distllm.dev/api-versions for migration."
817
+ ),
818
+ media_type="text/plain",
819
+ headers={
820
+ VERSION_HEADER: result.version.value,
821
+ SUNSET_HEADER: self._schedule.sunset_header_value(result.version)
822
+ or "",
823
+ },
824
+ )
825
+
826
+ # 4. Transform request body if compatibility layer is configured and
827
+ # the request version differs from the internal version.
828
+ if (
829
+ self._compat is not None
830
+ and result.version != self._internal_version
831
+ ):
832
+ request.state._original_api_version = result.version
833
+ # Note: actual body transformation happens in a helper that must
834
+ # be called from the route handler, because FastAPI/Starlette
835
+ # does not expose the parsed body to middleware in a
836
+ # framework-independent way. We store the target version so
837
+ # route handlers can opt in.
838
+ request.state.api_compat_target = self._internal_version
839
+ else:
840
+ request.state.api_compat_target = None
841
+
842
+ # 5. Process the request.
843
+ response = await call_next(request)
844
+
845
+ # 6. Add version headers to the response.
846
+ response.headers[VERSION_HEADER] = result.version.value
847
+
848
+ sunset_header = self._schedule.sunset_header_value(result.version)
849
+ if sunset_header is not None:
850
+ response.headers[SUNSET_HEADER] = sunset_header
851
+
852
+ # 7. Add deprecation warning header for versions near sunset.
853
+ if self._schedule.is_near_sunset(result.version):
854
+ sunset_date = self._schedule.get_sunset_date(result.version)
855
+ days_left = self._schedule.days_until_sunset(result.version)
856
+ migration_steps = self._schedule.get_migration_path(
857
+ result.version, self._internal_version
858
+ )
859
+ breaking_count = sum(1 for s in migration_steps if s.breaking)
860
+ response.headers[DEPRECATION_HEADER] = (
861
+ f"Version {result.version.value} will be removed on "
862
+ f"{sunset_date} ({days_left} days remaining). "
863
+ f"{breaking_count} breaking change(s) to address. "
864
+ f"See https://docs.distllm.dev/api-versions for migration."
865
+ )
866
+
867
+ # 8. Transform response body back to the request version if the
868
+ # compatibility layer is active and the request was upgraded.
869
+ original_version: APIVersion | None = getattr(
870
+ request.state, "_original_api_version", None
871
+ )
872
+ if (
873
+ self._compat is not None
874
+ and original_version is not None
875
+ and original_version != self._internal_version
876
+ ):
877
+ # The response body may already be streamed at this point.
878
+ # Full body transformation for buffered responses only.
879
+ if hasattr(response, "body") and callable(response.body):
880
+ body_value = response.body # type: ignore[union-attr]
881
+ if isinstance(body_value, bytes):
882
+ import json
883
+
884
+ try:
885
+ body_dict = json.loads(body_value)
886
+ transformed = self._compat.transform_response(
887
+ body_dict,
888
+ from_version=self._internal_version,
889
+ to_version=original_version,
890
+ )
891
+ # Build a new response with the transformed body.
892
+ from starlette.responses import JSONResponse
893
+
894
+ new_response = JSONResponse(
895
+ content=transformed,
896
+ status_code=response.status_code,
897
+ headers=dict(response.headers),
898
+ media_type=response.media_type,
899
+ )
900
+ return new_response
901
+ except (json.JSONDecodeError, TypeError, ValueError):
902
+ logger.debug(
903
+ "Could not transform response body for version %s",
904
+ original_version.value,
905
+ )
906
+
907
+ return response
908
+
909
+
910
+ # ── Convenience factory ───────────────────────────────────────────────────────
911
+
912
+
913
+ def create_version_middleware(
914
+ *,
915
+ default_version: str = DEFAULT_API_VERSION,
916
+ internal_version: APIVersion = APIVersion.V2,
917
+ reject_deprecated: bool = True,
918
+ enable_compatibility: bool = True,
919
+ ) -> VersionMiddleware:
920
+ """Create a fully-configured ``VersionMiddleware`` with sane defaults.
921
+
922
+ Parameters
923
+ ----------
924
+ default_version:
925
+ Fallback version string (default ``"2026-07"``).
926
+ internal_version:
927
+ The server's internal API version (default V2).
928
+ reject_deprecated:
929
+ Whether to return 410 for past-sunset versions (default ``True``).
930
+ enable_compatibility:
931
+ Whether to register the built-in field mappings for request/response
932
+ transformation (default ``True``).
933
+ """
934
+ schedule = DeprecationSchedule.default_schedule()
935
+ negotiator = VersionNegotiator(default_version=default_version)
936
+
937
+ compat: VersionCompatibilityLayer | None = None
938
+ if enable_compatibility:
939
+ compat = VersionCompatibilityLayer()
940
+ _register_default_mappings(compat)
941
+
942
+ return VersionMiddleware(
943
+ app=None, # type: ignore[arg-type] # set when added via add_middleware
944
+ negotiator=negotiator,
945
+ schedule=schedule,
946
+ compatibility_layer=compat,
947
+ internal_version=internal_version,
948
+ reject_deprecated=reject_deprecated,
949
+ )
950
+
951
+
952
+ def _register_default_mappings(layer: VersionCompatibilityLayer) -> None:
953
+ """Register the built-in field mappings for V1 <-> V2 compatibility."""
954
+ layer.add_mapping(
955
+ from_version=APIVersion.V1,
956
+ to_version=APIVersion.V2,
957
+ field_maps=[
958
+ FieldMapping(
959
+ source_field="logit_bias",
960
+ target_field="logprobs",
961
+ transform=lambda val: (
962
+ {"top_logprobs": len(val)} if isinstance(val, dict) else None
963
+ ),
964
+ ),
965
+ ],
966
+ direction="request",
967
+ )
968
+ layer.add_mapping(
969
+ from_version=APIVersion.V2,
970
+ to_version=APIVersion.V1,
971
+ field_maps=[
972
+ FieldMapping(
973
+ source_field="response.object",
974
+ target_field="response.object",
975
+ transform=lambda _: "chat.completion",
976
+ ),
977
+ FieldMapping(
978
+ source_field="response.system_fingerprint",
979
+ target_field="response.system_fingerprint",
980
+ transform=lambda _: None,
981
+ ),
982
+ ],
983
+ direction="response",
984
+ )