distributed-llm 0.4.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (806) hide show
  1. distllm/__init__.py +47 -0
  2. distllm/api/__init__.py +11 -0
  3. distllm/api/api_settings.py +126 -0
  4. distllm/api/api_state.py +85 -0
  5. distllm/api/api_versioning.py +984 -0
  6. distllm/api/app_state.py +29 -0
  7. distllm/api/auth/__init__.py +181 -0
  8. distllm/api/auth/models.py +37 -0
  9. distllm/api/auth/oauth2.py +123 -0
  10. distllm/api/auth/oidc.py +317 -0
  11. distllm/api/auth/saml.py +107 -0
  12. distllm/api/auth/store.py +39 -0
  13. distllm/api/auth_deps.py +68 -0
  14. distllm/api/authz/__init__.py +30 -0
  15. distllm/api/authz/opa.py +297 -0
  16. distllm/api/background_tasks.py +295 -0
  17. distllm/api/body_cache_middleware.py +117 -0
  18. distllm/api/circuit_breaker_middleware.py +175 -0
  19. distllm/api/cost_middleware.py +186 -0
  20. distllm/api/csrf_middleware.py +196 -0
  21. distllm/api/dedup.py +180 -0
  22. distllm/api/errors.py +256 -0
  23. distllm/api/grpc_bridge.py +296 -0
  24. distllm/api/ip_utils.py +58 -0
  25. distllm/api/middleware.py +706 -0
  26. distllm/api/observability_middleware.py +169 -0
  27. distllm/api/persistent_store.py +295 -0
  28. distllm/api/prompt_injection.py +418 -0
  29. distllm/api/quota_middleware.py +186 -0
  30. distllm/api/rate_limit_middleware.py +13 -0
  31. distllm/api/rate_limiter.py +250 -0
  32. distllm/api/rate_limiter_unified.py +975 -0
  33. distllm/api/redis_rate_limiter.py +164 -0
  34. distllm/api/routes/__init__.py +42 -0
  35. distllm/api/routes/admin.py +599 -0
  36. distllm/api/routes/api_keys.py +115 -0
  37. distllm/api/routes/assistants.py +443 -0
  38. distllm/api/routes/batch.py +1094 -0
  39. distllm/api/routes/chat.py +875 -0
  40. distllm/api/routes/completion.py +141 -0
  41. distllm/api/routes/debug.py +185 -0
  42. distllm/api/routes/defrag.py +66 -0
  43. distllm/api/routes/embeddings.py +399 -0
  44. distllm/api/routes/eval.py +254 -0
  45. distllm/api/routes/exchange.py +367 -0
  46. distllm/api/routes/experiments.py +293 -0
  47. distllm/api/routes/federated.py +155 -0
  48. distllm/api/routes/files.py +243 -0
  49. distllm/api/routes/fine_tuning.py +381 -0
  50. distllm/api/routes/gossip.py +72 -0
  51. distllm/api/routes/health.py +395 -0
  52. distllm/api/routes/leaderboard.py +377 -0
  53. distllm/api/routes/marketplace.py +320 -0
  54. distllm/api/routes/metrics_history.py +415 -0
  55. distllm/api/routes/model_registry.py +255 -0
  56. distllm/api/routes/moderation.py +420 -0
  57. distllm/api/routes/plugins.py +184 -0
  58. distllm/api/routes/prompts.py +347 -0
  59. distllm/api/routes/router_admin.py +187 -0
  60. distllm/api/routes/scheduler.py +168 -0
  61. distllm/api/routes/tokenize.py +77 -0
  62. distllm/api/routes/tools.py +89 -0
  63. distllm/api/routes/webhooks.py +336 -0
  64. distllm/api/routes/webrtc.py +194 -0
  65. distllm/api/semantic_cache.py +337 -0
  66. distllm/api/server.py +2038 -0
  67. distllm/api/server_config.py +140 -0
  68. distllm/api/server_coordinator.py +125 -0
  69. distllm/api/server_errors.py +79 -0
  70. distllm/api/server_lifespan.py +184 -0
  71. distllm/api/server_routes_api.py +456 -0
  72. distllm/api/server_routes_dashboard.py +246 -0
  73. distllm/api/server_state.py +29 -0
  74. distllm/api/services/__init__.py +2 -0
  75. distllm/api/services/chat_service.py +677 -0
  76. distllm/api/services/completion_service.py +227 -0
  77. distllm/api/services/embedding_service.py +451 -0
  78. distllm/api/services/eval_service.py +198 -0
  79. distllm/api/sso_auth.py +644 -0
  80. distllm/api/sso_middleware.py +808 -0
  81. distllm/api/streaming.py +479 -0
  82. distllm/api/token_estimator.py +86 -0
  83. distllm/api/tool_calling.py +240 -0
  84. distllm/api/tracing_middleware.py +579 -0
  85. distllm/api/validation.py +58 -0
  86. distllm/api/waf.py +344 -0
  87. distllm/api/webhooks/__init__.py +17 -0
  88. distllm/api/webhooks/delivery.py +452 -0
  89. distllm/api/ws_chat.py +495 -0
  90. distllm/backends/__init__.py +121 -0
  91. distllm/backends/config.py +85 -0
  92. distllm/backends/exllama_backend.py +229 -0
  93. distllm/backends/llamacpp_backend.py +266 -0
  94. distllm/backends/mlx_backend.py +199 -0
  95. distllm/backends/nim_backend.py +838 -0
  96. distllm/backends/ollama_backend.py +48 -0
  97. distllm/backends/onnx_backend.py +239 -0
  98. distllm/backends/paged_attention.py +547 -0
  99. distllm/backends/paged_attention_quantized.py +319 -0
  100. distllm/backends/protocol.py +203 -0
  101. distllm/backends/pytorch_backend.py +157 -0
  102. distllm/backends/registry.py +428 -0
  103. distllm/backends/tensorrt_backend.py +333 -0
  104. distllm/backends/tgi_backend.py +120 -0
  105. distllm/backends/triton_backend.py +875 -0
  106. distllm/backends/vllm_backend.py +489 -0
  107. distllm/backends/webgpu_backend.py +255 -0
  108. distllm/benchmarks/__init__.py +10 -0
  109. distllm/benchmarks/evaluation_harness.py +69 -0
  110. distllm/benchmarks/scaling.py +298 -0
  111. distllm/cache/__init__.py +19 -0
  112. distllm/cache/crdt.py +507 -0
  113. distllm/cache/cross_cluster_prefix_index.py +709 -0
  114. distllm/cli/__init__.py +17 -0
  115. distllm/cli/adapters.py +93 -0
  116. distllm/cli/autopsy.py +287 -0
  117. distllm/cli/backup.py +118 -0
  118. distllm/cli/benchmark.py +241 -0
  119. distllm/cli/cert.py +86 -0
  120. distllm/cli/chat.py +79 -0
  121. distllm/cli/chat_stream.py +494 -0
  122. distllm/cli/client.py +233 -0
  123. distllm/cli/cluster.py +405 -0
  124. distllm/cli/completion.py +79 -0
  125. distllm/cli/compress.py +98 -0
  126. distllm/cli/config_commands.py +71 -0
  127. distllm/cli/cost_avoid.py +168 -0
  128. distllm/cli/daas_commands.py +123 -0
  129. distllm/cli/defrag_commands.py +128 -0
  130. distllm/cli/deploy.py +252 -0
  131. distllm/cli/doctor.py +688 -0
  132. distllm/cli/draft_commands.py +82 -0
  133. distllm/cli/error_handler.py +192 -0
  134. distllm/cli/eval.py +168 -0
  135. distllm/cli/federate_commands.py +125 -0
  136. distllm/cli/install.py +198 -0
  137. distllm/cli/logs.py +85 -0
  138. distllm/cli/main.py +1795 -0
  139. distllm/cli/models.py +74 -0
  140. distllm/cli/notify.py +91 -0
  141. distllm/cli/observe.py +314 -0
  142. distllm/cli/onboard.py +671 -0
  143. distllm/cli/output.py +437 -0
  144. distllm/cli/profile.py +152 -0
  145. distllm/cli/profile_presets.py +129 -0
  146. distllm/cli/prompts.py +139 -0
  147. distllm/cli/quota.py +227 -0
  148. distllm/cli/router.py +131 -0
  149. distllm/cli/run.py +59 -0
  150. distllm/cli/setup.py +104 -0
  151. distllm/cli/setup_wizard.py +191 -0
  152. distllm/cli/status.py +65 -0
  153. distllm/cli/system_commands.py +255 -0
  154. distllm/cli/tune.py +488 -0
  155. distllm/cli/tutorial.py +128 -0
  156. distllm/cli/verify.py +37 -0
  157. distllm/cli/webhook.py +85 -0
  158. distllm/client/__init__.py +38 -0
  159. distllm/client/client.py +503 -0
  160. distllm/cloud/__init__.py +53 -0
  161. distllm/cloud/aws.py +199 -0
  162. distllm/cloud/azure.py +179 -0
  163. distllm/cloud/common.py +217 -0
  164. distllm/cloud/gcp.py +238 -0
  165. distllm/cloud/spot_orchestrator.py +1642 -0
  166. distllm/cloud/worker_agent.py +344 -0
  167. distllm/compliance/__init__.py +10 -0
  168. distllm/compliance/evidence_pack.py +699 -0
  169. distllm/config/__init__.py +81 -0
  170. distllm/config/_application.py +41 -0
  171. distllm/config/_backends.py +80 -0
  172. distllm/config/_cache.py +121 -0
  173. distllm/config/_deployment.py +95 -0
  174. distllm/config/_generation.py +36 -0
  175. distllm/config/_hardware.py +42 -0
  176. distllm/config/_model.py +270 -0
  177. distllm/config/_network.py +235 -0
  178. distllm/config/_observability.py +42 -0
  179. distllm/config/_parallelism.py +172 -0
  180. distllm/config/_performance.py +47 -0
  181. distllm/config/loader.py +168 -0
  182. distllm/config/model_heuristics.py +169 -0
  183. distllm/config/profiles.py +168 -0
  184. distllm/config/reference.py +104 -0
  185. distllm/config/resolver.py +240 -0
  186. distllm/config/settings.py +463 -0
  187. distllm/constants.py +117 -0
  188. distllm/core/__init__.py +111 -0
  189. distllm/core/_advanced_scheduling_original.py +1892 -0
  190. distllm/core/ab_test_coordinator.py +178 -0
  191. distllm/core/activation_profiler.py +172 -0
  192. distllm/core/adaptive_batching.py +234 -0
  193. distllm/core/adaptive_cache_compressor.py +168 -0
  194. distllm/core/adaptive_compression.py +427 -0
  195. distllm/core/adaptive_compression_hierarchy.py +402 -0
  196. distllm/core/advanced_scheduling/__init__.py +39 -0
  197. distllm/core/advanced_scheduling/cost_aware.py +55 -0
  198. distllm/core/advanced_scheduling/disaggregated.py +352 -0
  199. distllm/core/advanced_scheduling/energy.py +58 -0
  200. distllm/core/advanced_scheduling/federated.py +59 -0
  201. distllm/core/advanced_scheduling/heterogeneous.py +84 -0
  202. distllm/core/advanced_scheduling/multi_objective.py +372 -0
  203. distllm/core/advanced_scheduling/policy.py +95 -0
  204. distllm/core/advanced_scheduling/predictive.py +47 -0
  205. distllm/core/advanced_scheduling/preemption.py +43 -0
  206. distllm/core/advanced_scheduling/tiered_store.py +604 -0
  207. distllm/core/advanced_scheduling/token_bank.py +62 -0
  208. distllm/core/advanced_scheduling/wan.py +85 -0
  209. distllm/core/aegis_compliance.py +890 -0
  210. distllm/core/aether_federated.py +1326 -0
  211. distllm/core/agentic_router.py +749 -0
  212. distllm/core/api_key_store.py +478 -0
  213. distllm/core/arbitrage_engine.py +613 -0
  214. distllm/core/aria_autoscaler.py +1339 -0
  215. distllm/core/async_connection_pool.py +187 -0
  216. distllm/core/async_pipelined_speculative.py +543 -0
  217. distllm/core/atlas_mesh.py +1437 -0
  218. distllm/core/auto_discovery.py +351 -0
  219. distllm/core/auto_partitioner.py +396 -0
  220. distllm/core/autonomous_healer.py +466 -0
  221. distllm/core/autoq.py +1545 -0
  222. distllm/core/backup_manager.py +344 -0
  223. distllm/core/bargaining_engine.py +450 -0
  224. distllm/core/batch_builder.py +480 -0
  225. distllm/core/batch_scheduler.py +1342 -0
  226. distllm/core/block_affinity_tracker.py +162 -0
  227. distllm/core/block_eviction_policy.py +363 -0
  228. distllm/core/cache_aware_router.py +116 -0
  229. distllm/core/cache_bench.py +239 -0
  230. distllm/core/cache_coherence.py +139 -0
  231. distllm/core/cache_doctor.py +220 -0
  232. distllm/core/cache_eviction.py +181 -0
  233. distllm/core/cache_index.py +91 -0
  234. distllm/core/cache_manager.py +703 -0
  235. distllm/core/cache_migration.py +134 -0
  236. distllm/core/cache_persistence.py +200 -0
  237. distllm/core/cache_query_log.py +195 -0
  238. distllm/core/cache_snapshot.py +178 -0
  239. distllm/core/cache_template_warmer.py +112 -0
  240. distllm/core/cache_warming.py +188 -0
  241. distllm/core/calibration.py +223 -0
  242. distllm/core/carbon_migration.py +338 -0
  243. distllm/core/cert_rotation.py +402 -0
  244. distllm/core/certificate_manager.py +414 -0
  245. distllm/core/cluster_manager.py +212 -0
  246. distllm/core/cluster_predictive_prefetcher.py +239 -0
  247. distllm/core/cluster_registry.py +227 -0
  248. distllm/core/cluster_state_store.py +442 -0
  249. distllm/core/compliance_evidence.py +503 -0
  250. distllm/core/compressed_speculative.py +399 -0
  251. distllm/core/connection_pool.py +387 -0
  252. distllm/core/constrained_decoder.py +678 -0
  253. distllm/core/coordinator.py +1594 -0
  254. distllm/core/coordinator_cli.py +130 -0
  255. distllm/core/coordinator_config.py +203 -0
  256. distllm/core/coordinator_config_wiring.py +316 -0
  257. distllm/core/coordinator_election.py +246 -0
  258. distllm/core/coordinator_failover.py +197 -0
  259. distllm/core/coordinator_health.py +135 -0
  260. distllm/core/coordinator_lifecycle.py +272 -0
  261. distllm/core/coordinator_metrics.py +155 -0
  262. distllm/core/coordinator_multi_model.py +41 -0
  263. distllm/core/coordinator_request.py +246 -0
  264. distllm/core/coordinator_state.py +215 -0
  265. distllm/core/coordinator_subsystem.py +614 -0
  266. distllm/core/correctness_cert.py +133 -0
  267. distllm/core/correctness_harness.py +421 -0
  268. distllm/core/cortex_multimodel.py +939 -0
  269. distllm/core/cost_comparison.py +250 -0
  270. distllm/core/cost_dashboard.py +363 -0
  271. distllm/core/cost_optimizer.py +317 -0
  272. distllm/core/cost_tracker.py +569 -0
  273. distllm/core/cross_cloud_router.py +924 -0
  274. distllm/core/cross_model_prefix_sharing.py +237 -0
  275. distllm/core/cuda_graph.py +165 -0
  276. distllm/core/debug.py +84 -0
  277. distllm/core/device_registry.py +289 -0
  278. distllm/core/di.py +153 -0
  279. distllm/core/differential_privacy.py +183 -0
  280. distllm/core/distributed_distillation.py +402 -0
  281. distllm/core/distributed_speculative.py +1563 -0
  282. distllm/core/dp_inference/__init__.py +657 -0
  283. distllm/core/dp_inference/accounting.py +144 -0
  284. distllm/core/dp_inference/config.py +139 -0
  285. distllm/core/dp_inference/mechanisms.py +137 -0
  286. distllm/core/dp_inference.py +1016 -0
  287. distllm/core/draft_model_router.py +435 -0
  288. distllm/core/draft_quality_scorer.py +157 -0
  289. distllm/core/draft_tree.py +232 -0
  290. distllm/core/dynamic_memory_budget.py +179 -0
  291. distllm/core/dynamic_sharder.py +364 -0
  292. distllm/core/dynamic_speculation.py +131 -0
  293. distllm/core/evaluation/__init__.py +57 -0
  294. distllm/core/evaluation/constants.py +86 -0
  295. distllm/core/evaluation/db.py +185 -0
  296. distllm/core/evaluation/formatters.py +82 -0
  297. distllm/core/evaluation/loaders.py +320 -0
  298. distllm/core/evaluation/models.py +66 -0
  299. distllm/core/evaluation/report.py +79 -0
  300. distllm/core/evaluation/runner.py +518 -0
  301. distllm/core/evaluation/scorers.py +278 -0
  302. distllm/core/evaluation/worker.py +123 -0
  303. distllm/core/evaluation_harness.py +1530 -0
  304. distllm/core/event_bus.py +516 -0
  305. distllm/core/faas_7b.py +201 -0
  306. distllm/core/feature_flags.py +245 -0
  307. distllm/core/federated_finetuner.py +377 -0
  308. distllm/core/federated_incentives.py +267 -0
  309. distllm/core/framework_adapters.py +390 -0
  310. distllm/core/gaia_cache.py +1330 -0
  311. distllm/core/gossip_cache_bridge.py +202 -0
  312. distllm/core/gpu_power_manager.py +173 -0
  313. distllm/core/gpu_profiler.py +59 -0
  314. distllm/core/gpu_resource_manager.py +338 -0
  315. distllm/core/graceful_degradation.py +264 -0
  316. distllm/core/grammar_constrained.py +549 -0
  317. distllm/core/grammar_constrained_draft.py +91 -0
  318. distllm/core/grammar_decoder.py +285 -0
  319. distllm/core/ha_coordinator.py +452 -0
  320. distllm/core/hardware_plugin_registry.py +126 -0
  321. distllm/core/hashing.py +29 -0
  322. distllm/core/health_manager.py +320 -0
  323. distllm/core/heterogeneous_scheduler.py +663 -0
  324. distllm/core/hibernation_manager.py +250 -0
  325. distllm/core/hierarchical_digest.py +168 -0
  326. distllm/core/hybrid_cache.py +253 -0
  327. distllm/core/hydra_diffusion.py +177 -0
  328. distllm/core/inference_engine.py +1072 -0
  329. distllm/core/intelligent_autoscaler.py +219 -0
  330. distllm/core/kraken_chaos.py +1407 -0
  331. distllm/core/kv_backup.py +217 -0
  332. distllm/core/kv_cache.py +1271 -0
  333. distllm/core/kv_cache_adaptive_quantizer.py +168 -0
  334. distllm/core/kv_cache_compressor.py +290 -0
  335. distllm/core/kv_cache_manager.py +142 -0
  336. distllm/core/kv_cache_marketplace.py +319 -0
  337. distllm/core/kv_cache_metrics.py +205 -0
  338. distllm/core/kv_cache_migration.py +137 -0
  339. distllm/core/kv_cache_paged.py +78 -0
  340. distllm/core/kv_cache_replication.py +107 -0
  341. distllm/core/kv_cache_serialization.py +182 -0
  342. distllm/core/latency_tracker.py +17 -0
  343. distllm/core/leaky_bucket_limiter.py +194 -0
  344. distllm/core/learning_router.py +652 -0
  345. distllm/core/llamacpp_backend.py +23 -0
  346. distllm/core/load_balancer.py +286 -0
  347. distllm/core/media_pipeline.py +387 -0
  348. distllm/core/memory_defragmenter.py +666 -0
  349. distllm/core/metering.py +449 -0
  350. distllm/core/metrics_collector.py +39 -0
  351. distllm/core/micro_batch_scheduler.py +170 -0
  352. distllm/core/model_hub_enhanced.py +382 -0
  353. distllm/core/model_registry.py +9 -0
  354. distllm/core/model_router.py +930 -0
  355. distllm/core/model_selector.py +245 -0
  356. distllm/core/model_sizing.py +334 -0
  357. distllm/core/model_version_manager.py +414 -0
  358. distllm/core/moe_orchestrator.py +205 -0
  359. distllm/core/money.py +53 -0
  360. distllm/core/monitor.py +141 -0
  361. distllm/core/mtp_head.py +444 -0
  362. distllm/core/multi_draft_verifier.py +346 -0
  363. distllm/core/multi_model_serving.py +624 -0
  364. distllm/core/multimodal_engine.py +194 -0
  365. distllm/core/neural_partition_optimizer.py +1398 -0
  366. distllm/core/node_recovery.py +27 -0
  367. distllm/core/notification_manager.py +317 -0
  368. distllm/core/param_update_channel.py +104 -0
  369. distllm/core/pareto_optimizer.py +227 -0
  370. distllm/core/performance_alerts.py +198 -0
  371. distllm/core/performance_baseline.py +252 -0
  372. distllm/core/persistence.py +947 -0
  373. distllm/core/persistent_store.py +485 -0
  374. distllm/core/pipeline_composer.py +211 -0
  375. distllm/core/pipeline_executor.py +500 -0
  376. distllm/core/pipeline_orchestrator.py +17 -0
  377. distllm/core/pipeline_overlap.py +177 -0
  378. distllm/core/placement.py +442 -0
  379. distllm/core/plugin_marketplace.py +483 -0
  380. distllm/core/plugin_sandbox.py +751 -0
  381. distllm/core/plugin_system.py +601 -0
  382. distllm/core/plugins/__init__.py +26 -0
  383. distllm/core/plugins/sandbox.py +38 -0
  384. distllm/core/predictive_cache.py +29 -0
  385. distllm/core/predictive_cache_warming.py +323 -0
  386. distllm/core/predictive_failure.py +163 -0
  387. distllm/core/preemptible_scheduler.py +343 -0
  388. distllm/core/preference_learning.py +606 -0
  389. distllm/core/prefix_cache.py +17 -0
  390. distllm/core/pricing_providers.py +576 -0
  391. distllm/core/priority_heap.py +62 -0
  392. distllm/core/privacy_budget.py +177 -0
  393. distllm/core/prompt_caching_service.py +228 -0
  394. distllm/core/prompt_library.py +838 -0
  395. distllm/core/protocols.py +213 -0
  396. distllm/core/provider_health.py +234 -0
  397. distllm/core/provisioning.py +329 -0
  398. distllm/core/pulse_performance_model.py +1558 -0
  399. distllm/core/quantization_selector.py +210 -0
  400. distllm/core/radix_tree_cache.py +306 -0
  401. distllm/core/rebalancer.py +25 -0
  402. distllm/core/redis_prompt_cache.py +311 -0
  403. distllm/core/replication_controller.py +128 -0
  404. distllm/core/request_auditor.py +215 -0
  405. distllm/core/request_fingerprinting.py +231 -0
  406. distllm/core/request_latency.py +211 -0
  407. distllm/core/request_pipeline.py +810 -0
  408. distllm/core/request_replay.py +253 -0
  409. distllm/core/request_tracker.py +184 -0
  410. distllm/core/resource_manager.py +657 -0
  411. distllm/core/route_audit.py +175 -0
  412. distllm/core/routing_extensions.py +604 -0
  413. distllm/core/schedule_simulator.py +289 -0
  414. distllm/core/schedule_viz.py +233 -0
  415. distllm/core/scheduler/__init__.py +34 -0
  416. distllm/core/scheduler/budget.py +25 -0
  417. distllm/core/scheduler/budget_computer.py +322 -0
  418. distllm/core/scheduler/chunked_prefill.py +52 -0
  419. distllm/core/scheduler/kv_cache_manager.py +313 -0
  420. distllm/core/scheduler/preemption_manager.py +251 -0
  421. distllm/core/scheduler/pressure.py +48 -0
  422. distllm/core/scheduler/sequence.py +179 -0
  423. distllm/core/secret_manager.py +315 -0
  424. distllm/core/semantic_cache.py +369 -0
  425. distllm/core/sentinel_qos.py +833 -0
  426. distllm/core/shadow_eval_runner.py +194 -0
  427. distllm/core/shared_layer_pool.py +279 -0
  428. distllm/core/smart_model_router.py +309 -0
  429. distllm/core/spec_equivalence.py +140 -0
  430. distllm/core/spec_verify.py +225 -0
  431. distllm/core/speculative_adaptor.py +108 -0
  432. distllm/core/speculative_dashboard.py +134 -0
  433. distllm/core/speculative_decoder.py +1286 -0
  434. distllm/core/speculative_profiler.py +132 -0
  435. distllm/core/split_brain.py +236 -0
  436. distllm/core/spot_failover.py +292 -0
  437. distllm/core/spot_forecasting.py +463 -0
  438. distllm/core/starvation_monitor.py +70 -0
  439. distllm/core/state_replication.py +261 -0
  440. distllm/core/stats_collector.py +64 -0
  441. distllm/core/step_processor.py +115 -0
  442. distllm/core/straggler_alerts.py +174 -0
  443. distllm/core/straggler_aware_scheduler.py +255 -0
  444. distllm/core/straggler_detector.py +29 -0
  445. distllm/core/streaming_cost.py +286 -0
  446. distllm/core/streaming_generator.py +228 -0
  447. distllm/core/structured_output/__init__.py +348 -0
  448. distllm/core/structured_output/config.py +38 -0
  449. distllm/core/structured_output/engine.py +367 -0
  450. distllm/core/structured_output/streaming.py +183 -0
  451. distllm/core/structured_output/validator.py +199 -0
  452. distllm/core/subsystem_registry.py +171 -0
  453. distllm/core/synapse_debugger.py +226 -0
  454. distllm/core/synth_data_generator.py +122 -0
  455. distllm/core/telemetry.py +240 -0
  456. distllm/core/tenant_billing.py +446 -0
  457. distllm/core/tenant_cost_attribution.py +314 -0
  458. distllm/core/token_generator.py +368 -0
  459. distllm/core/token_streaming_buffer.py +147 -0
  460. distllm/core/topology_aware_lb.py +324 -0
  461. distllm/core/topology_aware_tiering.py +344 -0
  462. distllm/core/tree_speculative_decoder.py +405 -0
  463. distllm/core/types.py +17 -0
  464. distllm/core/unified_router.py +439 -0
  465. distllm/core/unified_sla_router.py +242 -0
  466. distllm/core/usage_meter.py +915 -0
  467. distllm/core/vectorstore/__init__.py +30 -0
  468. distllm/core/vectorstore/base.py +106 -0
  469. distllm/core/vectorstore/chroma_store.py +99 -0
  470. distllm/core/vectorstore/factory.py +97 -0
  471. distllm/core/vectorstore/pgvector_store.py +123 -0
  472. distllm/core/vectorstore/providers/__init__.py +7 -0
  473. distllm/core/vectorstore/providers/milvus.py +219 -0
  474. distllm/core/vectorstore/providers/pinecone.py +142 -0
  475. distllm/core/vectorstore/providers/qdrant.py +176 -0
  476. distllm/core/vectorstore/providers/weaviate.py +185 -0
  477. distllm/core/vectorstore/qdrant_store.py +148 -0
  478. distllm/core/vectorstore/rag.py +151 -0
  479. distllm/core/vllm_backend.py +21 -0
  480. distllm/core/vllm_node.py +174 -0
  481. distllm/core/voyager_multimodal.py +1354 -0
  482. distllm/core/webgpu_manager.py +382 -0
  483. distllm/core/webhook_formatters.py +243 -0
  484. distllm/core/webhook_manager.py +433 -0
  485. distllm/core/wisp_wasm.py +934 -0
  486. distllm/core/workload_classifier.py +137 -0
  487. distllm/dashboard/__init__.py +17 -0
  488. distllm/dashboard/verification_history.py +192 -0
  489. distllm/dashboard/ws_handler.py +586 -0
  490. distllm/dist/__init__.py +85 -0
  491. distllm/dist/__init__.pyi +132 -0
  492. distllm/dist/_pipeline_original.py +2916 -0
  493. distllm/dist/adaptive_speculator.py +384 -0
  494. distllm/dist/admin_cli.py +293 -0
  495. distllm/dist/api_docs.py +594 -0
  496. distllm/dist/async_pipeline.py +578 -0
  497. distllm/dist/attention.py +1699 -0
  498. distllm/dist/attention_block_pool.py +634 -0
  499. distllm/dist/attention_distributed.py +138 -0
  500. distllm/dist/autoscaler.py +196 -0
  501. distllm/dist/backends/__init__.py +29 -0
  502. distllm/dist/backends/backend_profiles.py +357 -0
  503. distllm/dist/backends/graceful_degradation.py +409 -0
  504. distllm/dist/backends/health_monitor.py +518 -0
  505. distllm/dist/backends/llamacpp.py +125 -0
  506. distllm/dist/backends/ray.py +89 -0
  507. distllm/dist/backends/vllm.py +177 -0
  508. distllm/dist/block_pool.py +662 -0
  509. distllm/dist/block_transfer_service.py +267 -0
  510. distllm/dist/byzantine.py +2193 -0
  511. distllm/dist/cache.py +305 -0
  512. distllm/dist/cache_digest.py +405 -0
  513. distllm/dist/chaos/__init__.py +52 -0
  514. distllm/dist/chaos/chaos_orchestrator.py +229 -0
  515. distllm/dist/chaos/fault_injector.py +1273 -0
  516. distllm/dist/chunked_prefill.py +136 -0
  517. distllm/dist/cloud_selector.py +257 -0
  518. distllm/dist/config.py +70 -0
  519. distllm/dist/cross_cluster.py +231 -0
  520. distllm/dist/daas/__init__.py +46 -0
  521. distllm/dist/daas/load_balancer.py +275 -0
  522. distllm/dist/daas/marketplace_integration.py +374 -0
  523. distllm/dist/daas/resource_isolation.py +280 -0
  524. distllm/dist/daas/tenant_dispatcher.py +252 -0
  525. distllm/dist/daas/usage_meter.py +360 -0
  526. distllm/dist/daas_server.py +514 -0
  527. distllm/dist/disagg/__init__.py +162 -0
  528. distllm/dist/disagg/pool.py +214 -0
  529. distllm/dist/disagg/transfer.py +77 -0
  530. distllm/dist/disagg/transfer_compression.py +793 -0
  531. distllm/dist/disagg/transfer_engine.py +998 -0
  532. distllm/dist/discovery.py +166 -0
  533. distllm/dist/draft_bank.py +344 -0
  534. distllm/dist/draft_migration.py +382 -0
  535. distllm/dist/ebpf_transport.py +364 -0
  536. distllm/dist/edge_cloud.py +544 -0
  537. distllm/dist/edge_federation.py +426 -0
  538. distllm/dist/execution_planner.py +148 -0
  539. distllm/dist/federated_merge.py +560 -0
  540. distllm/dist/federation.py +1114 -0
  541. distllm/dist/flash_attention.py +237 -0
  542. distllm/dist/fsdp.py +295 -0
  543. distllm/dist/geo.py +361 -0
  544. distllm/dist/ice_transport.py +2968 -0
  545. distllm/dist/kv_migration.py +237 -0
  546. distllm/dist/latency.py +60 -0
  547. distllm/dist/lifecycle.py +467 -0
  548. distllm/dist/marketplace.py +891 -0
  549. distllm/dist/merkle.py +178 -0
  550. distllm/dist/metrics.py +456 -0
  551. distllm/dist/model_store.py +107 -0
  552. distllm/dist/multi_tenant.py +339 -0
  553. distllm/dist/multimodal.py +392 -0
  554. distllm/dist/nat.py +934 -0
  555. distllm/dist/nccl.py +733 -0
  556. distllm/dist/network.py +126 -0
  557. distllm/dist/node_client.py +568 -0
  558. distllm/dist/node_pb2.py +66 -0
  559. distllm/dist/node_pb2.pyi +236 -0
  560. distllm/dist/node_pb2_grpc.py +312 -0
  561. distllm/dist/node_pb2_grpc.pyi +41 -0
  562. distllm/dist/node_registrar.py +247 -0
  563. distllm/dist/node_service.py +572 -0
  564. distllm/dist/observability.py +258 -0
  565. distllm/dist/otel.py +158 -0
  566. distllm/dist/p2p/__init__.py +22 -0
  567. distllm/dist/p2p/discovery.py +132 -0
  568. distllm/dist/p2p/gossip.py +1339 -0
  569. distllm/dist/p2p/kademlia_dht.py +1058 -0
  570. distllm/dist/p2p/load_balancer.py +131 -0
  571. distllm/dist/p2p/quic_transport.py +700 -0
  572. distllm/dist/p2p/router.py +607 -0
  573. distllm/dist/p2p/transport.py +399 -0
  574. distllm/dist/p2p_model_distributor.py +160 -0
  575. distllm/dist/paged_attention_kernel.py +281 -0
  576. distllm/dist/parallel.py +1061 -0
  577. distllm/dist/parallel_planner.py +586 -0
  578. distllm/dist/parallel_topology.py +231 -0
  579. distllm/dist/partition/__init__.py +146 -0
  580. distllm/dist/partition/adaptive.py +532 -0
  581. distllm/dist/partition/benchmark_suite.py +479 -0
  582. distllm/dist/partition/cli.py +189 -0
  583. distllm/dist/partition/cloud_arbitrage.py +523 -0
  584. distllm/dist/partition/config.py +97 -0
  585. distllm/dist/partition/cost_model.py +553 -0
  586. distllm/dist/partition/learned_cost.py +653 -0
  587. distllm/dist/partition/mixed_precision_tuner.py +310 -0
  588. distllm/dist/partition/network_cost_model.py +664 -0
  589. distllm/dist/partition/optimizer.py +650 -0
  590. distllm/dist/partition/pareto_optimizer.py +466 -0
  591. distllm/dist/partition/partitioner.py +363 -0
  592. distllm/dist/partition/persistence.py +392 -0
  593. distllm/dist/partition/plugins.py +270 -0
  594. distllm/dist/partition/profiles.py +659 -0
  595. distllm/dist/partition/quant_bench.py +373 -0
  596. distllm/dist/partition/quant_calibrate.py +229 -0
  597. distllm/dist/partition/quant_coordinator.py +260 -0
  598. distllm/dist/partition/quant_cost.py +293 -0
  599. distllm/dist/partition/quant_partition.py +432 -0
  600. distllm/dist/partition/quant_report.py +301 -0
  601. distllm/dist/partition/quantization_metrics.py +355 -0
  602. distllm/dist/partition/quantization_search.py +290 -0
  603. distllm/dist/partition/quantization_tuner.py +1383 -0
  604. distllm/dist/partition/topology.py +218 -0
  605. distllm/dist/partition/validator.py +405 -0
  606. distllm/dist/partition/visualizer.py +275 -0
  607. distllm/dist/pipeline/__init__.py +71 -0
  608. distllm/dist/pipeline/bandwidth_controller.py +780 -0
  609. distllm/dist/pipeline/compression_negotiation.py +771 -0
  610. distllm/dist/pipeline/context.py +80 -0
  611. distllm/dist/pipeline/continuous_batching.py +435 -0
  612. distllm/dist/pipeline/disagg_orchestrator.py +284 -0
  613. distllm/dist/pipeline/orchestrator.py +727 -0
  614. distllm/dist/pipeline/pipeline_reconfig.py +1395 -0
  615. distllm/dist/pipeline/profiler.py +84 -0
  616. distllm/dist/pipeline/serialization.py +176 -0
  617. distllm/dist/pipeline/simulator.py +172 -0
  618. distllm/dist/pipeline/strategy.py +193 -0
  619. distllm/dist/pipeline/token_accumulator.py +211 -0
  620. distllm/dist/pipeline/transport.py +230 -0
  621. distllm/dist/pipeline/wan_disagg_orchestrator.py +564 -0
  622. distllm/dist/power_cap.py +358 -0
  623. distllm/dist/predictive_cache.py +399 -0
  624. distllm/dist/preemption.py +198 -0
  625. distllm/dist/prefix_cache.py +461 -0
  626. distllm/dist/prefix_clustering.py +305 -0
  627. distllm/dist/privacy/__init__.py +411 -0
  628. distllm/dist/privacy/privacy_accountant.py +344 -0
  629. distllm/dist/privacy.py +240 -0
  630. distllm/dist/provisioning.py +338 -0
  631. distllm/dist/quality.py +118 -0
  632. distllm/dist/quic_transport.py +335 -0
  633. distllm/dist/quota_enforcer.py +265 -0
  634. distllm/dist/rebalancer.py +234 -0
  635. distllm/dist/recovery.py +827 -0
  636. distllm/dist/recovery_drill.py +281 -0
  637. distllm/dist/redis_cache.py +265 -0
  638. distllm/dist/redundant.py +308 -0
  639. distllm/dist/redundant_executor.py +1536 -0
  640. distllm/dist/reputation.py +231 -0
  641. distllm/dist/router.py +382 -0
  642. distllm/dist/routing/__init__.py +19 -0
  643. distllm/dist/routing/arbitrage_engine.py +575 -0
  644. distllm/dist/routing/composite.py +179 -0
  645. distllm/dist/routing/consistent_hash.py +144 -0
  646. distllm/dist/routing/latency_aware.py +116 -0
  647. distllm/dist/routing/load_aware.py +114 -0
  648. distllm/dist/scheduling/__init__.py +38 -0
  649. distllm/dist/scheduling/batcher.py +95 -0
  650. distllm/dist/scheduling/classifier.py +153 -0
  651. distllm/dist/scheduling/deadline_scheduler.py +958 -0
  652. distllm/dist/scheduling/iteration.py +404 -0
  653. distllm/dist/scheduling/profiler.py +57 -0
  654. distllm/dist/simulation/__init__.py +54 -0
  655. distllm/dist/simulation/chaos_simulator.py +901 -0
  656. distllm/dist/simulation/cluster_simulator.py +667 -0
  657. distllm/dist/simulation/cost_aware_provisioning.py +370 -0
  658. distllm/dist/simulation/digital_twin.py +617 -0
  659. distllm/dist/simulation/topology_optimizer.py +747 -0
  660. distllm/dist/simulation/what_if.py +492 -0
  661. distllm/dist/speculative/__init__.py +47 -0
  662. distllm/dist/speculative/adaptive_spec.py +291 -0
  663. distllm/dist/speculative/draft_cache.py +246 -0
  664. distllm/dist/speculative/draft_orchestrator.py +634 -0
  665. distllm/dist/speculative/draft_registry.py +439 -0
  666. distllm/dist/speculative/multi_draft.py +304 -0
  667. distllm/dist/speculative/online_sisd.py +523 -0
  668. distllm/dist/straggler.py +819 -0
  669. distllm/dist/streaming_kv_transfer.py +174 -0
  670. distllm/dist/structured_output/__init__.py +7 -0
  671. distllm/dist/structured_output/engine.py +144 -0
  672. distllm/dist/tensor_pool.py +120 -0
  673. distllm/dist/tgi_compat.py +254 -0
  674. distllm/dist/topology.py +141 -0
  675. distllm/dist/topology_consensus.py +1961 -0
  676. distllm/dist/topology_dynamic.py +270 -0
  677. distllm/dist/tp_inprocess.py +214 -0
  678. distllm/dist/tp_launcher.py +310 -0
  679. distllm/dist/tracing.py +325 -0
  680. distllm/dist/transport.py +493 -0
  681. distllm/dist/wan_speculative.py +295 -0
  682. distllm/dist/wandb_integration.py +252 -0
  683. distllm/dist/webrtc.py +846 -0
  684. distllm/dist/wide_area.py +515 -0
  685. distllm/dist/worker.py +635 -0
  686. distllm/dist/zero_copy.py +307 -0
  687. distllm/errors/__init__.py +105 -0
  688. distllm/errors/policies.py +106 -0
  689. distllm/errors/retry.py +168 -0
  690. distllm/errors/types.py +721 -0
  691. distllm/health/__init__.py +6 -0
  692. distllm/health/failover.py +81 -0
  693. distllm/health/prober.py +31 -0
  694. distllm/health/service.py +173 -0
  695. distllm/health/state.py +76 -0
  696. distllm/integrations/__init__.py +72 -0
  697. distllm/integrations/_common/__init__.py +7 -0
  698. distllm/integrations/_common/base_tool_provider.py +189 -0
  699. distllm/integrations/_common/cost_tracker.py +144 -0
  700. distllm/integrations/_common/model_router.py +148 -0
  701. distllm/integrations/ci/__init__.py +24 -0
  702. distllm/integrations/ci/_common.py +92 -0
  703. distllm/integrations/ci/gitlab.py +345 -0
  704. distllm/integrations/ci/jenkins.py +273 -0
  705. distllm/integrations/mlflow_tracking.py +372 -0
  706. distllm/integrations/spark_connector.py +909 -0
  707. distllm/integrations/wandb/__init__.py +81 -0
  708. distllm/models/__init__.py +40 -0
  709. distllm/models/_trust_remote.py +48 -0
  710. distllm/models/adapter.py +768 -0
  711. distllm/models/adapter_router.py +198 -0
  712. distllm/models/cache.py +172 -0
  713. distllm/models/model_hub.py +650 -0
  714. distllm/models/partition_planner.py +384 -0
  715. distllm/models/partitioner.py +784 -0
  716. distllm/models/rope_scaling.py +119 -0
  717. distllm/models/safetensors_index.py +179 -0
  718. distllm/observability/__init__.py +107 -0
  719. distllm/observability/alerting_config.py +881 -0
  720. distllm/observability/capacity_models.py +398 -0
  721. distllm/observability/capacity_planning.py +2870 -0
  722. distllm/observability/exporter.py +228 -0
  723. distllm/observability/exporter_datadog.py +1362 -0
  724. distllm/observability/health_config.py +1282 -0
  725. distllm/observability/incident_response.py +779 -0
  726. distllm/observability/logging.py +211 -0
  727. distllm/observability/logging_config.py +864 -0
  728. distllm/observability/loki_sink.py +178 -0
  729. distllm/observability/metrics.py +111 -0
  730. distllm/observability/metrics_completeness.py +393 -0
  731. distllm/observability/otel_logging.py +290 -0
  732. distllm/observability/self_healing_config.py +1308 -0
  733. distllm/observability/slo_config.py +1520 -0
  734. distllm/observability/spans.py +235 -0
  735. distllm/observability/tracing.py +268 -0
  736. distllm/observability/tracing_config.py +676 -0
  737. distllm/observability/wandb_monitor.py +66 -0
  738. distllm/plugins/__init__.py +14 -0
  739. distllm/plugins/airflow.py +110 -0
  740. distllm/plugins/auth_plugin.py +576 -0
  741. distllm/plugins/builtin.py +376 -0
  742. distllm/plugins/cache_plugin.py +576 -0
  743. distllm/plugins/health_plugin.py +396 -0
  744. distllm/plugins/kubeflow.py +84 -0
  745. distllm/plugins/mlflow_plugin.py +71 -0
  746. distllm/plugins/registry.py +135 -0
  747. distllm/prompts/__init__.py +39 -0
  748. distllm/prompts/analysis.py +103 -0
  749. distllm/prompts/code.py +172 -0
  750. distllm/prompts/creative.py +115 -0
  751. distllm/prompts/education.py +91 -0
  752. distllm/prompts/engine.py +119 -0
  753. distllm/prompts/exchange.py +705 -0
  754. distllm/prompts/language.py +81 -0
  755. distllm/prompts/library.py +1198 -0
  756. distllm/prompts/management.py +35 -0
  757. distllm/prompts/productivity.py +86 -0
  758. distllm/prompts/professional.py +111 -0
  759. distllm/prompts/prompt_def.py +33 -0
  760. distllm/prompts/reasoning.py +108 -0
  761. distllm/prompts/specialized.py +140 -0
  762. distllm/prompts/templates.py +128 -0
  763. distllm/prompts/writing.py +140 -0
  764. distllm/py.typed +0 -0
  765. distllm/sdk/__init__.py +82 -0
  766. distllm/sdk/circuit_breaker.py +139 -0
  767. distllm/sdk/client.py +1191 -0
  768. distllm/sdk/observability.py +42 -0
  769. distllm/sdk/streaming.py +228 -0
  770. distllm/sdk/transport.py +224 -0
  771. distllm/sdk/types.py +288 -0
  772. distllm/security/__init__.py +63 -0
  773. distllm/security/content_moderation/__init__.py +44 -0
  774. distllm/security/content_moderation/base.py +405 -0
  775. distllm/security/content_moderation/jailbreak.py +192 -0
  776. distllm/security/content_moderation/pii.py +262 -0
  777. distllm/security/content_moderation/pipeline.py +272 -0
  778. distllm/security/content_moderation/topics.py +153 -0
  779. distllm/security/content_moderation/toxicity.py +131 -0
  780. distllm/security/e2e.py +437 -0
  781. distllm/security/edge_attestation.py +381 -0
  782. distllm/security/log_redaction.py +244 -0
  783. distllm/security/quantum_safe_tls.py +314 -0
  784. distllm/security/spiffe.py +347 -0
  785. distllm/security/tee.py +461 -0
  786. distllm/security/utils.py +139 -0
  787. distllm/security/watermark.py +779 -0
  788. distllm/terraform/__init__.py +260 -0
  789. distllm/ui/__init__.py +7 -0
  790. distllm/ui/app.py +98 -0
  791. distllm/utils/__init__.py +0 -0
  792. distllm/utils/gbnf_grammar.py +99 -0
  793. distllm/utils/lazy_imports.py +90 -0
  794. distllm/utils/scheduling.py +38 -0
  795. distllm/verification/__init__.py +49 -0
  796. distllm/verification/comparator.py +263 -0
  797. distllm/verification/hash_registry.py +220 -0
  798. distllm/verification/report.py +240 -0
  799. distllm/verification/runner.py +822 -0
  800. distllm/verification/test_verify.py +205 -0
  801. distributed_llm-0.4.1.dist-info/METADATA +315 -0
  802. distributed_llm-0.4.1.dist-info/RECORD +806 -0
  803. distributed_llm-0.4.1.dist-info/WHEEL +5 -0
  804. distributed_llm-0.4.1.dist-info/entry_points.txt +5 -0
  805. distributed_llm-0.4.1.dist-info/licenses/LICENSE +97 -0
  806. distributed_llm-0.4.1.dist-info/top_level.txt +1 -0
@@ -0,0 +1,338 @@
1
+ """Carbon-Aware Load Migration.
2
+
3
+ Monitors carbon intensity in active regions and triggers migration
4
+ to cleaner regions when carbon spikes above a threshold. Uses KV cache
5
+ shipping from cross_cluster.py for live migration.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import os
11
+ import threading
12
+ import time
13
+ from dataclasses import dataclass, field
14
+ from typing import Any, Callable
15
+
16
+ from loguru import logger
17
+
18
+
19
+ @dataclass
20
+ class MigrationEvent:
21
+ """A carbon-triggered migration event."""
22
+ from_region: str
23
+ to_region: str
24
+ from_carbon: float
25
+ to_carbon: float
26
+ carbon_saved: float
27
+ request_ids: list[str] = field(default_factory=list)
28
+ timestamp: float = field(default_factory=time.time)
29
+ success: bool = False
30
+ duration_ms: float = 0.0
31
+
32
+
33
+ class CarbonIntensityClient:
34
+ """Client for real-time carbon intensity data via electricityMap API.
35
+
36
+ Uses the https://api.electricitymap.org endpoint. Set
37
+ ``DISTLLM_CARBON_API_KEY`` to your electricityMap API token.
38
+ """
39
+
40
+ def __init__(self, api_key: str | None = None):
41
+ self._api_key = api_key or os.environ.get("DISTLLM_CARBON_API_KEY", "")
42
+ self._base_url = "https://api.electricitymap.org/v3"
43
+ self._cache: dict[str, tuple[float, float]] = {} # zone -> (gco2/kWh, timestamp)
44
+ self._cache_ttl: float = 600.0 # 10 min
45
+ self._lock = threading.Lock()
46
+
47
+ def get_intensity(self, zone: str) -> float:
48
+ """Get current carbon intensity for a zone (gCO2/kWh).
49
+
50
+ Uses cached data when available (< 10 min old).
51
+ Returns 0.0 on failure.
52
+ """
53
+ now = time.time()
54
+ with self._lock:
55
+ cached = self._cache.get(zone)
56
+ if cached and (now - cached[1]) < self._cache_ttl:
57
+ return cached[0]
58
+
59
+ if not self._api_key:
60
+ # Fallback to static regional data
61
+ from distllm.core.cross_cloud_router import _REGIONAL_CARBON_INTENSITY
62
+ val = _REGIONAL_CARBON_INTENSITY.get(zone, 0.0)
63
+ with self._lock:
64
+ self._cache[zone] = (val, now)
65
+ return val
66
+
67
+ try:
68
+ import httpx
69
+ resp = httpx.get(
70
+ f"{self._base_url}/carbon-intensity/latest",
71
+ params={"zone": zone},
72
+ headers={"auth-token": self._api_key},
73
+ timeout=10.0,
74
+ )
75
+ if resp.status_code == 200:
76
+ data = resp.json()
77
+ val = float(data.get("carbonIntensity", 0.0))
78
+ with self._lock:
79
+ self._cache[zone] = (val, now)
80
+ return val
81
+ except Exception as e:
82
+ logger.debug(f"Carbon API request failed for {zone}: {e}")
83
+
84
+ return 0.0
85
+
86
+ def get_all_intensities(self, zones: list[str]) -> dict[str, float]:
87
+ """Fetch intensities for multiple zones."""
88
+ return {z: self.get_intensity(z) for z in zones}
89
+
90
+
91
+ class CarbonMigrationEngine:
92
+ """Monitors carbon intensity and triggers live migration to cleaner regions.
93
+
94
+ Uses the electricityMap API when configured, with static fallback.
95
+
96
+ Usage::
97
+
98
+ engine = CarbonMigrationEngine(threshold=400)
99
+ engine.set_active_region("us-east-1", ["req-1", "req-2"])
100
+ engine.start(migration_callback=my_migrate_fn)
101
+ """
102
+
103
+ def __init__(
104
+ self,
105
+ carbon_provider: Any = None,
106
+ threshold: float = 400.0,
107
+ check_interval_s: float = 300.0,
108
+ migration_cooldown_s: float = 900.0,
109
+ min_savings_pct: float = 20.0,
110
+ ):
111
+ if carbon_provider is None:
112
+ carbon_provider = CarbonIntensityClient()
113
+ self._carbon_provider = carbon_provider
114
+ self._threshold = threshold
115
+ self._check_interval = check_interval_s
116
+ self._cooldown = migration_cooldown_s
117
+ self._min_savings_pct = min_savings_pct / 100.0
118
+ self._active_region: str = ""
119
+ self._active_requests: list[str] = []
120
+ self._migration_callback: Callable[[str, str, list[str]], bool] | None = None
121
+ self._running = False
122
+ self._thread: threading.Thread | None = None
123
+ self._events: list[MigrationEvent] = []
124
+ self._last_migration: float = 0.0
125
+ self._lock = threading.Lock()
126
+
127
+ def set_active_region(self, region: str, request_ids: list[str] | None = None) -> None:
128
+ """Set the currently active region and tracked requests."""
129
+ with self._lock:
130
+ self._active_region = region
131
+ if request_ids is not None:
132
+ self._active_requests = request_ids
133
+
134
+ def set_migration_callback(self, callback: Callable[[str, str, list[str]], bool]) -> None:
135
+ """Set the migration function.
136
+
137
+ The callback receives (from_region, to_region, request_ids) and
138
+ returns True if migration succeeded.
139
+ """
140
+ self._migration_callback = callback
141
+
142
+ def start(self) -> None:
143
+ """Start the carbon monitoring loop."""
144
+ if self._running:
145
+ return
146
+ self._running = True
147
+ self._thread = threading.Thread(
148
+ target=self._monitor_loop,
149
+ daemon=True,
150
+ name="carbon-migration",
151
+ )
152
+ self._thread.start()
153
+ logger.info(f"Carbon migration engine started (threshold={self._threshold} gCO2/kWh)")
154
+
155
+ def stop(self) -> None:
156
+ """Stop the monitoring loop."""
157
+ self._running = False
158
+ if self._thread:
159
+ self._thread.join(timeout=5)
160
+ self._thread = None
161
+
162
+ def _monitor_loop(self) -> None:
163
+ while self._running:
164
+ time.sleep(self._check_interval)
165
+ try:
166
+ self._check_and_migrate()
167
+ except Exception as e:
168
+ logger.debug(f"Carbon migration check failed: {e}")
169
+
170
+ def _check_and_migrate(self) -> None:
171
+ """Check if current region exceeds carbon threshold and migrate if needed."""
172
+ if not self._carbon_provider or not self._active_region:
173
+ return
174
+ # Check cooldown
175
+ if time.time() - self._last_migration < self._cooldown:
176
+ return
177
+ # Get current carbon intensity via real-time API (with static fallback)
178
+ current_val = self._carbon_provider.get_intensity(self._active_region) if hasattr(self._carbon_provider, 'get_intensity') else 0.0
179
+ if isinstance(current_val, (int, float)):
180
+ current_gco2 = current_val
181
+ else:
182
+ current_gco2 = current_val.gco2_per_kwh if hasattr(current_val, 'gco2_per_kwh') else 0.0
183
+
184
+ if current_gco2 <= self._threshold:
185
+ return
186
+
187
+ # Find a cleaner region using carbon provider + fallback
188
+ regions_to_check = getattr(self._carbon_provider, 'get_all_intensities', None)
189
+ if regions_to_check:
190
+ from distllm.core.cross_cloud_router import _REGIONAL_CARBON_INTENSITY
191
+ intensities = regions_to_check(list(_REGIONAL_CARBON_INTENSITY.keys()))
192
+ else:
193
+ from distllm.core.cross_cloud_router import _REGIONAL_CARBON_INTENSITY
194
+ intensities = dict(_REGIONAL_CARBON_INTENSITY)
195
+
196
+ best_region = None
197
+ best_carbon = current_gco2
198
+ for region, carbon in intensities.items():
199
+ if carbon > 0 and carbon < best_carbon * (1.0 - self._min_savings_pct):
200
+ best_region = region
201
+ best_carbon = carbon
202
+ if not best_region:
203
+ return
204
+ # Trigger migration
205
+ logger.warning(
206
+ f"Carbon spike in {self._active_region}: {current_gco2:.0f} gCO2/kWh "
207
+ f"(threshold: {self._threshold}). Migrating to {best_region} ({best_carbon:.0f})"
208
+ )
209
+ event = MigrationEvent(
210
+ from_region=self._active_region,
211
+ to_region=best_region,
212
+ from_carbon=current_gco2,
213
+ to_carbon=best_carbon,
214
+ carbon_saved=current_gco2 - best_carbon,
215
+ request_ids=list(self._active_requests),
216
+ )
217
+ if self._migration_callback:
218
+ try:
219
+ success = self._migration_callback(
220
+ self._active_region, best_region, self._active_requests
221
+ )
222
+ event.success = success
223
+ if success:
224
+ self._active_region = best_region
225
+ self._last_migration = time.time()
226
+ except Exception as e:
227
+ logger.error(f"Migration callback failed: {e}")
228
+ event.success = False
229
+ with self._lock:
230
+ self._events.append(event)
231
+
232
+ def get_events(self, limit: int = 50) -> list[dict[str, Any]]:
233
+ """Get recent migration events."""
234
+ with self._lock:
235
+ return [
236
+ {
237
+ "from": e.from_region,
238
+ "to": e.to_region,
239
+ "from_carbon": e.from_carbon,
240
+ "to_carbon": e.to_carbon,
241
+ "carbon_saved": e.carbon_saved,
242
+ "success": e.success,
243
+ "timestamp": e.timestamp,
244
+ }
245
+ for e in self._events[-limit:]
246
+ ]
247
+
248
+
249
+ class SLATier:
250
+ """An SLA tier with routing constraints."""
251
+
252
+ def __init__(
253
+ self,
254
+ name: str,
255
+ prefer_spot: bool = True,
256
+ allow_on_demand: bool = True,
257
+ max_latency_ms: float = 200.0,
258
+ max_carbon_intensity: float = float("inf"),
259
+ max_price_per_hour: float = float("inf"),
260
+ carbon_weight: float = 0.3,
261
+ ):
262
+ self.name = name
263
+ self.prefer_spot = prefer_spot
264
+ self.allow_on_demand = allow_on_demand
265
+ self.max_latency_ms = max_latency_ms
266
+ self.max_carbon_intensity = max_carbon_intensity
267
+ self.max_price_per_hour = max_price_per_hour
268
+ self.carbon_weight = carbon_weight
269
+
270
+ def to_routing_kwargs(self) -> dict[str, Any]:
271
+ """Convert to kwargs for CrossCloudRouter.select_provider_carbon_aware()."""
272
+ return {
273
+ "prefer_spot": self.prefer_spot,
274
+ "max_latency_ms": self.max_latency_ms,
275
+ "max_price": self.max_price_per_hour,
276
+ "max_carbon_intensity": self.max_carbon_intensity,
277
+ "carbon_weight": self.carbon_weight,
278
+ }
279
+
280
+
281
+ # Predefined SLA tiers
282
+ SLA_TIERS: dict[str, SLATier] = {
283
+ "critical": SLATier(
284
+ name="Critical",
285
+ prefer_spot=False,
286
+ allow_on_demand=True,
287
+ max_latency_ms=50.0,
288
+ max_carbon_intensity=float("inf"),
289
+ max_price_per_hour=float("inf"),
290
+ carbon_weight=0.0,
291
+ ),
292
+ "standard": SLATier(
293
+ name="Standard",
294
+ prefer_spot=True,
295
+ allow_on_demand=True,
296
+ max_latency_ms=200.0,
297
+ max_carbon_intensity=500.0,
298
+ max_price_per_hour=50.0,
299
+ carbon_weight=0.15,
300
+ ),
301
+ "batch": SLATier(
302
+ name="Batch",
303
+ prefer_spot=True,
304
+ allow_on_demand=False,
305
+ max_latency_ms=5000.0,
306
+ max_carbon_intensity=300.0,
307
+ max_price_per_hour=10.0,
308
+ carbon_weight=0.5,
309
+ ),
310
+ "green": SLATier(
311
+ name="Green",
312
+ prefer_spot=True,
313
+ allow_on_demand=False,
314
+ max_latency_ms=5000.0,
315
+ max_carbon_intensity=200.0,
316
+ max_price_per_hour=float("inf"),
317
+ carbon_weight=1.0,
318
+ ),
319
+ }
320
+
321
+
322
+ def get_sla_tier(name: str) -> SLATier:
323
+ """Get a predefined SLA tier by name."""
324
+ return SLA_TIERS.get(name.lower(), SLA_TIERS["standard"])
325
+
326
+
327
+ def list_sla_tiers() -> list[dict[str, Any]]:
328
+ """List all predefined SLA tiers."""
329
+ return [
330
+ {
331
+ "name": tier.name,
332
+ "prefer_spot": tier.prefer_spot,
333
+ "max_latency_ms": tier.max_latency_ms,
334
+ "max_carbon_intensity": tier.max_carbon_intensity,
335
+ "carbon_weight": tier.carbon_weight,
336
+ }
337
+ for tier in SLA_TIERS.values()
338
+ ]