distributed-llm 0.4.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (806) hide show
  1. distllm/__init__.py +47 -0
  2. distllm/api/__init__.py +11 -0
  3. distllm/api/api_settings.py +126 -0
  4. distllm/api/api_state.py +85 -0
  5. distllm/api/api_versioning.py +984 -0
  6. distllm/api/app_state.py +29 -0
  7. distllm/api/auth/__init__.py +181 -0
  8. distllm/api/auth/models.py +37 -0
  9. distllm/api/auth/oauth2.py +123 -0
  10. distllm/api/auth/oidc.py +317 -0
  11. distllm/api/auth/saml.py +107 -0
  12. distllm/api/auth/store.py +39 -0
  13. distllm/api/auth_deps.py +68 -0
  14. distllm/api/authz/__init__.py +30 -0
  15. distllm/api/authz/opa.py +297 -0
  16. distllm/api/background_tasks.py +295 -0
  17. distllm/api/body_cache_middleware.py +117 -0
  18. distllm/api/circuit_breaker_middleware.py +175 -0
  19. distllm/api/cost_middleware.py +186 -0
  20. distllm/api/csrf_middleware.py +196 -0
  21. distllm/api/dedup.py +180 -0
  22. distllm/api/errors.py +256 -0
  23. distllm/api/grpc_bridge.py +296 -0
  24. distllm/api/ip_utils.py +58 -0
  25. distllm/api/middleware.py +706 -0
  26. distllm/api/observability_middleware.py +169 -0
  27. distllm/api/persistent_store.py +295 -0
  28. distllm/api/prompt_injection.py +418 -0
  29. distllm/api/quota_middleware.py +186 -0
  30. distllm/api/rate_limit_middleware.py +13 -0
  31. distllm/api/rate_limiter.py +250 -0
  32. distllm/api/rate_limiter_unified.py +975 -0
  33. distllm/api/redis_rate_limiter.py +164 -0
  34. distllm/api/routes/__init__.py +42 -0
  35. distllm/api/routes/admin.py +599 -0
  36. distllm/api/routes/api_keys.py +115 -0
  37. distllm/api/routes/assistants.py +443 -0
  38. distllm/api/routes/batch.py +1094 -0
  39. distllm/api/routes/chat.py +875 -0
  40. distllm/api/routes/completion.py +141 -0
  41. distllm/api/routes/debug.py +185 -0
  42. distllm/api/routes/defrag.py +66 -0
  43. distllm/api/routes/embeddings.py +399 -0
  44. distllm/api/routes/eval.py +254 -0
  45. distllm/api/routes/exchange.py +367 -0
  46. distllm/api/routes/experiments.py +293 -0
  47. distllm/api/routes/federated.py +155 -0
  48. distllm/api/routes/files.py +243 -0
  49. distllm/api/routes/fine_tuning.py +381 -0
  50. distllm/api/routes/gossip.py +72 -0
  51. distllm/api/routes/health.py +395 -0
  52. distllm/api/routes/leaderboard.py +377 -0
  53. distllm/api/routes/marketplace.py +320 -0
  54. distllm/api/routes/metrics_history.py +415 -0
  55. distllm/api/routes/model_registry.py +255 -0
  56. distllm/api/routes/moderation.py +420 -0
  57. distllm/api/routes/plugins.py +184 -0
  58. distllm/api/routes/prompts.py +347 -0
  59. distllm/api/routes/router_admin.py +187 -0
  60. distllm/api/routes/scheduler.py +168 -0
  61. distllm/api/routes/tokenize.py +77 -0
  62. distllm/api/routes/tools.py +89 -0
  63. distllm/api/routes/webhooks.py +336 -0
  64. distllm/api/routes/webrtc.py +194 -0
  65. distllm/api/semantic_cache.py +337 -0
  66. distllm/api/server.py +2038 -0
  67. distllm/api/server_config.py +140 -0
  68. distllm/api/server_coordinator.py +125 -0
  69. distllm/api/server_errors.py +79 -0
  70. distllm/api/server_lifespan.py +184 -0
  71. distllm/api/server_routes_api.py +456 -0
  72. distllm/api/server_routes_dashboard.py +246 -0
  73. distllm/api/server_state.py +29 -0
  74. distllm/api/services/__init__.py +2 -0
  75. distllm/api/services/chat_service.py +677 -0
  76. distllm/api/services/completion_service.py +227 -0
  77. distllm/api/services/embedding_service.py +451 -0
  78. distllm/api/services/eval_service.py +198 -0
  79. distllm/api/sso_auth.py +644 -0
  80. distllm/api/sso_middleware.py +808 -0
  81. distllm/api/streaming.py +479 -0
  82. distllm/api/token_estimator.py +86 -0
  83. distllm/api/tool_calling.py +240 -0
  84. distllm/api/tracing_middleware.py +579 -0
  85. distllm/api/validation.py +58 -0
  86. distllm/api/waf.py +344 -0
  87. distllm/api/webhooks/__init__.py +17 -0
  88. distllm/api/webhooks/delivery.py +452 -0
  89. distllm/api/ws_chat.py +495 -0
  90. distllm/backends/__init__.py +121 -0
  91. distllm/backends/config.py +85 -0
  92. distllm/backends/exllama_backend.py +229 -0
  93. distllm/backends/llamacpp_backend.py +266 -0
  94. distllm/backends/mlx_backend.py +199 -0
  95. distllm/backends/nim_backend.py +838 -0
  96. distllm/backends/ollama_backend.py +48 -0
  97. distllm/backends/onnx_backend.py +239 -0
  98. distllm/backends/paged_attention.py +547 -0
  99. distllm/backends/paged_attention_quantized.py +319 -0
  100. distllm/backends/protocol.py +203 -0
  101. distllm/backends/pytorch_backend.py +157 -0
  102. distllm/backends/registry.py +428 -0
  103. distllm/backends/tensorrt_backend.py +333 -0
  104. distllm/backends/tgi_backend.py +120 -0
  105. distllm/backends/triton_backend.py +875 -0
  106. distllm/backends/vllm_backend.py +489 -0
  107. distllm/backends/webgpu_backend.py +255 -0
  108. distllm/benchmarks/__init__.py +10 -0
  109. distllm/benchmarks/evaluation_harness.py +69 -0
  110. distllm/benchmarks/scaling.py +298 -0
  111. distllm/cache/__init__.py +19 -0
  112. distllm/cache/crdt.py +507 -0
  113. distllm/cache/cross_cluster_prefix_index.py +709 -0
  114. distllm/cli/__init__.py +17 -0
  115. distllm/cli/adapters.py +93 -0
  116. distllm/cli/autopsy.py +287 -0
  117. distllm/cli/backup.py +118 -0
  118. distllm/cli/benchmark.py +241 -0
  119. distllm/cli/cert.py +86 -0
  120. distllm/cli/chat.py +79 -0
  121. distllm/cli/chat_stream.py +494 -0
  122. distllm/cli/client.py +233 -0
  123. distllm/cli/cluster.py +405 -0
  124. distllm/cli/completion.py +79 -0
  125. distllm/cli/compress.py +98 -0
  126. distllm/cli/config_commands.py +71 -0
  127. distllm/cli/cost_avoid.py +168 -0
  128. distllm/cli/daas_commands.py +123 -0
  129. distllm/cli/defrag_commands.py +128 -0
  130. distllm/cli/deploy.py +252 -0
  131. distllm/cli/doctor.py +688 -0
  132. distllm/cli/draft_commands.py +82 -0
  133. distllm/cli/error_handler.py +192 -0
  134. distllm/cli/eval.py +168 -0
  135. distllm/cli/federate_commands.py +125 -0
  136. distllm/cli/install.py +198 -0
  137. distllm/cli/logs.py +85 -0
  138. distllm/cli/main.py +1795 -0
  139. distllm/cli/models.py +74 -0
  140. distllm/cli/notify.py +91 -0
  141. distllm/cli/observe.py +314 -0
  142. distllm/cli/onboard.py +671 -0
  143. distllm/cli/output.py +437 -0
  144. distllm/cli/profile.py +152 -0
  145. distllm/cli/profile_presets.py +129 -0
  146. distllm/cli/prompts.py +139 -0
  147. distllm/cli/quota.py +227 -0
  148. distllm/cli/router.py +131 -0
  149. distllm/cli/run.py +59 -0
  150. distllm/cli/setup.py +104 -0
  151. distllm/cli/setup_wizard.py +191 -0
  152. distllm/cli/status.py +65 -0
  153. distllm/cli/system_commands.py +255 -0
  154. distllm/cli/tune.py +488 -0
  155. distllm/cli/tutorial.py +128 -0
  156. distllm/cli/verify.py +37 -0
  157. distllm/cli/webhook.py +85 -0
  158. distllm/client/__init__.py +38 -0
  159. distllm/client/client.py +503 -0
  160. distllm/cloud/__init__.py +53 -0
  161. distllm/cloud/aws.py +199 -0
  162. distllm/cloud/azure.py +179 -0
  163. distllm/cloud/common.py +217 -0
  164. distllm/cloud/gcp.py +238 -0
  165. distllm/cloud/spot_orchestrator.py +1642 -0
  166. distllm/cloud/worker_agent.py +344 -0
  167. distllm/compliance/__init__.py +10 -0
  168. distllm/compliance/evidence_pack.py +699 -0
  169. distllm/config/__init__.py +81 -0
  170. distllm/config/_application.py +41 -0
  171. distllm/config/_backends.py +80 -0
  172. distllm/config/_cache.py +121 -0
  173. distllm/config/_deployment.py +95 -0
  174. distllm/config/_generation.py +36 -0
  175. distllm/config/_hardware.py +42 -0
  176. distllm/config/_model.py +270 -0
  177. distllm/config/_network.py +235 -0
  178. distllm/config/_observability.py +42 -0
  179. distllm/config/_parallelism.py +172 -0
  180. distllm/config/_performance.py +47 -0
  181. distllm/config/loader.py +168 -0
  182. distllm/config/model_heuristics.py +169 -0
  183. distllm/config/profiles.py +168 -0
  184. distllm/config/reference.py +104 -0
  185. distllm/config/resolver.py +240 -0
  186. distllm/config/settings.py +463 -0
  187. distllm/constants.py +117 -0
  188. distllm/core/__init__.py +111 -0
  189. distllm/core/_advanced_scheduling_original.py +1892 -0
  190. distllm/core/ab_test_coordinator.py +178 -0
  191. distllm/core/activation_profiler.py +172 -0
  192. distllm/core/adaptive_batching.py +234 -0
  193. distllm/core/adaptive_cache_compressor.py +168 -0
  194. distllm/core/adaptive_compression.py +427 -0
  195. distllm/core/adaptive_compression_hierarchy.py +402 -0
  196. distllm/core/advanced_scheduling/__init__.py +39 -0
  197. distllm/core/advanced_scheduling/cost_aware.py +55 -0
  198. distllm/core/advanced_scheduling/disaggregated.py +352 -0
  199. distllm/core/advanced_scheduling/energy.py +58 -0
  200. distllm/core/advanced_scheduling/federated.py +59 -0
  201. distllm/core/advanced_scheduling/heterogeneous.py +84 -0
  202. distllm/core/advanced_scheduling/multi_objective.py +372 -0
  203. distllm/core/advanced_scheduling/policy.py +95 -0
  204. distllm/core/advanced_scheduling/predictive.py +47 -0
  205. distllm/core/advanced_scheduling/preemption.py +43 -0
  206. distllm/core/advanced_scheduling/tiered_store.py +604 -0
  207. distllm/core/advanced_scheduling/token_bank.py +62 -0
  208. distllm/core/advanced_scheduling/wan.py +85 -0
  209. distllm/core/aegis_compliance.py +890 -0
  210. distllm/core/aether_federated.py +1326 -0
  211. distllm/core/agentic_router.py +749 -0
  212. distllm/core/api_key_store.py +478 -0
  213. distllm/core/arbitrage_engine.py +613 -0
  214. distllm/core/aria_autoscaler.py +1339 -0
  215. distllm/core/async_connection_pool.py +187 -0
  216. distllm/core/async_pipelined_speculative.py +543 -0
  217. distllm/core/atlas_mesh.py +1437 -0
  218. distllm/core/auto_discovery.py +351 -0
  219. distllm/core/auto_partitioner.py +396 -0
  220. distllm/core/autonomous_healer.py +466 -0
  221. distllm/core/autoq.py +1545 -0
  222. distllm/core/backup_manager.py +344 -0
  223. distllm/core/bargaining_engine.py +450 -0
  224. distllm/core/batch_builder.py +480 -0
  225. distllm/core/batch_scheduler.py +1342 -0
  226. distllm/core/block_affinity_tracker.py +162 -0
  227. distllm/core/block_eviction_policy.py +363 -0
  228. distllm/core/cache_aware_router.py +116 -0
  229. distllm/core/cache_bench.py +239 -0
  230. distllm/core/cache_coherence.py +139 -0
  231. distllm/core/cache_doctor.py +220 -0
  232. distllm/core/cache_eviction.py +181 -0
  233. distllm/core/cache_index.py +91 -0
  234. distllm/core/cache_manager.py +703 -0
  235. distllm/core/cache_migration.py +134 -0
  236. distllm/core/cache_persistence.py +200 -0
  237. distllm/core/cache_query_log.py +195 -0
  238. distllm/core/cache_snapshot.py +178 -0
  239. distllm/core/cache_template_warmer.py +112 -0
  240. distllm/core/cache_warming.py +188 -0
  241. distllm/core/calibration.py +223 -0
  242. distllm/core/carbon_migration.py +338 -0
  243. distllm/core/cert_rotation.py +402 -0
  244. distllm/core/certificate_manager.py +414 -0
  245. distllm/core/cluster_manager.py +212 -0
  246. distllm/core/cluster_predictive_prefetcher.py +239 -0
  247. distllm/core/cluster_registry.py +227 -0
  248. distllm/core/cluster_state_store.py +442 -0
  249. distllm/core/compliance_evidence.py +503 -0
  250. distllm/core/compressed_speculative.py +399 -0
  251. distllm/core/connection_pool.py +387 -0
  252. distllm/core/constrained_decoder.py +678 -0
  253. distllm/core/coordinator.py +1594 -0
  254. distllm/core/coordinator_cli.py +130 -0
  255. distllm/core/coordinator_config.py +203 -0
  256. distllm/core/coordinator_config_wiring.py +316 -0
  257. distllm/core/coordinator_election.py +246 -0
  258. distllm/core/coordinator_failover.py +197 -0
  259. distllm/core/coordinator_health.py +135 -0
  260. distllm/core/coordinator_lifecycle.py +272 -0
  261. distllm/core/coordinator_metrics.py +155 -0
  262. distllm/core/coordinator_multi_model.py +41 -0
  263. distllm/core/coordinator_request.py +246 -0
  264. distllm/core/coordinator_state.py +215 -0
  265. distllm/core/coordinator_subsystem.py +614 -0
  266. distllm/core/correctness_cert.py +133 -0
  267. distllm/core/correctness_harness.py +421 -0
  268. distllm/core/cortex_multimodel.py +939 -0
  269. distllm/core/cost_comparison.py +250 -0
  270. distllm/core/cost_dashboard.py +363 -0
  271. distllm/core/cost_optimizer.py +317 -0
  272. distllm/core/cost_tracker.py +569 -0
  273. distllm/core/cross_cloud_router.py +924 -0
  274. distllm/core/cross_model_prefix_sharing.py +237 -0
  275. distllm/core/cuda_graph.py +165 -0
  276. distllm/core/debug.py +84 -0
  277. distllm/core/device_registry.py +289 -0
  278. distllm/core/di.py +153 -0
  279. distllm/core/differential_privacy.py +183 -0
  280. distllm/core/distributed_distillation.py +402 -0
  281. distllm/core/distributed_speculative.py +1563 -0
  282. distllm/core/dp_inference/__init__.py +657 -0
  283. distllm/core/dp_inference/accounting.py +144 -0
  284. distllm/core/dp_inference/config.py +139 -0
  285. distllm/core/dp_inference/mechanisms.py +137 -0
  286. distllm/core/dp_inference.py +1016 -0
  287. distllm/core/draft_model_router.py +435 -0
  288. distllm/core/draft_quality_scorer.py +157 -0
  289. distllm/core/draft_tree.py +232 -0
  290. distllm/core/dynamic_memory_budget.py +179 -0
  291. distllm/core/dynamic_sharder.py +364 -0
  292. distllm/core/dynamic_speculation.py +131 -0
  293. distllm/core/evaluation/__init__.py +57 -0
  294. distllm/core/evaluation/constants.py +86 -0
  295. distllm/core/evaluation/db.py +185 -0
  296. distllm/core/evaluation/formatters.py +82 -0
  297. distllm/core/evaluation/loaders.py +320 -0
  298. distllm/core/evaluation/models.py +66 -0
  299. distllm/core/evaluation/report.py +79 -0
  300. distllm/core/evaluation/runner.py +518 -0
  301. distllm/core/evaluation/scorers.py +278 -0
  302. distllm/core/evaluation/worker.py +123 -0
  303. distllm/core/evaluation_harness.py +1530 -0
  304. distllm/core/event_bus.py +516 -0
  305. distllm/core/faas_7b.py +201 -0
  306. distllm/core/feature_flags.py +245 -0
  307. distllm/core/federated_finetuner.py +377 -0
  308. distllm/core/federated_incentives.py +267 -0
  309. distllm/core/framework_adapters.py +390 -0
  310. distllm/core/gaia_cache.py +1330 -0
  311. distllm/core/gossip_cache_bridge.py +202 -0
  312. distllm/core/gpu_power_manager.py +173 -0
  313. distllm/core/gpu_profiler.py +59 -0
  314. distllm/core/gpu_resource_manager.py +338 -0
  315. distllm/core/graceful_degradation.py +264 -0
  316. distllm/core/grammar_constrained.py +549 -0
  317. distllm/core/grammar_constrained_draft.py +91 -0
  318. distllm/core/grammar_decoder.py +285 -0
  319. distllm/core/ha_coordinator.py +452 -0
  320. distllm/core/hardware_plugin_registry.py +126 -0
  321. distllm/core/hashing.py +29 -0
  322. distllm/core/health_manager.py +320 -0
  323. distllm/core/heterogeneous_scheduler.py +663 -0
  324. distllm/core/hibernation_manager.py +250 -0
  325. distllm/core/hierarchical_digest.py +168 -0
  326. distllm/core/hybrid_cache.py +253 -0
  327. distllm/core/hydra_diffusion.py +177 -0
  328. distllm/core/inference_engine.py +1072 -0
  329. distllm/core/intelligent_autoscaler.py +219 -0
  330. distllm/core/kraken_chaos.py +1407 -0
  331. distllm/core/kv_backup.py +217 -0
  332. distllm/core/kv_cache.py +1271 -0
  333. distllm/core/kv_cache_adaptive_quantizer.py +168 -0
  334. distllm/core/kv_cache_compressor.py +290 -0
  335. distllm/core/kv_cache_manager.py +142 -0
  336. distllm/core/kv_cache_marketplace.py +319 -0
  337. distllm/core/kv_cache_metrics.py +205 -0
  338. distllm/core/kv_cache_migration.py +137 -0
  339. distllm/core/kv_cache_paged.py +78 -0
  340. distllm/core/kv_cache_replication.py +107 -0
  341. distllm/core/kv_cache_serialization.py +182 -0
  342. distllm/core/latency_tracker.py +17 -0
  343. distllm/core/leaky_bucket_limiter.py +194 -0
  344. distllm/core/learning_router.py +652 -0
  345. distllm/core/llamacpp_backend.py +23 -0
  346. distllm/core/load_balancer.py +286 -0
  347. distllm/core/media_pipeline.py +387 -0
  348. distllm/core/memory_defragmenter.py +666 -0
  349. distllm/core/metering.py +449 -0
  350. distllm/core/metrics_collector.py +39 -0
  351. distllm/core/micro_batch_scheduler.py +170 -0
  352. distllm/core/model_hub_enhanced.py +382 -0
  353. distllm/core/model_registry.py +9 -0
  354. distllm/core/model_router.py +930 -0
  355. distllm/core/model_selector.py +245 -0
  356. distllm/core/model_sizing.py +334 -0
  357. distllm/core/model_version_manager.py +414 -0
  358. distllm/core/moe_orchestrator.py +205 -0
  359. distllm/core/money.py +53 -0
  360. distllm/core/monitor.py +141 -0
  361. distllm/core/mtp_head.py +444 -0
  362. distllm/core/multi_draft_verifier.py +346 -0
  363. distllm/core/multi_model_serving.py +624 -0
  364. distllm/core/multimodal_engine.py +194 -0
  365. distllm/core/neural_partition_optimizer.py +1398 -0
  366. distllm/core/node_recovery.py +27 -0
  367. distllm/core/notification_manager.py +317 -0
  368. distllm/core/param_update_channel.py +104 -0
  369. distllm/core/pareto_optimizer.py +227 -0
  370. distllm/core/performance_alerts.py +198 -0
  371. distllm/core/performance_baseline.py +252 -0
  372. distllm/core/persistence.py +947 -0
  373. distllm/core/persistent_store.py +485 -0
  374. distllm/core/pipeline_composer.py +211 -0
  375. distllm/core/pipeline_executor.py +500 -0
  376. distllm/core/pipeline_orchestrator.py +17 -0
  377. distllm/core/pipeline_overlap.py +177 -0
  378. distllm/core/placement.py +442 -0
  379. distllm/core/plugin_marketplace.py +483 -0
  380. distllm/core/plugin_sandbox.py +751 -0
  381. distllm/core/plugin_system.py +601 -0
  382. distllm/core/plugins/__init__.py +26 -0
  383. distllm/core/plugins/sandbox.py +38 -0
  384. distllm/core/predictive_cache.py +29 -0
  385. distllm/core/predictive_cache_warming.py +323 -0
  386. distllm/core/predictive_failure.py +163 -0
  387. distllm/core/preemptible_scheduler.py +343 -0
  388. distllm/core/preference_learning.py +606 -0
  389. distllm/core/prefix_cache.py +17 -0
  390. distllm/core/pricing_providers.py +576 -0
  391. distllm/core/priority_heap.py +62 -0
  392. distllm/core/privacy_budget.py +177 -0
  393. distllm/core/prompt_caching_service.py +228 -0
  394. distllm/core/prompt_library.py +838 -0
  395. distllm/core/protocols.py +213 -0
  396. distllm/core/provider_health.py +234 -0
  397. distllm/core/provisioning.py +329 -0
  398. distllm/core/pulse_performance_model.py +1558 -0
  399. distllm/core/quantization_selector.py +210 -0
  400. distllm/core/radix_tree_cache.py +306 -0
  401. distllm/core/rebalancer.py +25 -0
  402. distllm/core/redis_prompt_cache.py +311 -0
  403. distllm/core/replication_controller.py +128 -0
  404. distllm/core/request_auditor.py +215 -0
  405. distllm/core/request_fingerprinting.py +231 -0
  406. distllm/core/request_latency.py +211 -0
  407. distllm/core/request_pipeline.py +810 -0
  408. distllm/core/request_replay.py +253 -0
  409. distllm/core/request_tracker.py +184 -0
  410. distllm/core/resource_manager.py +657 -0
  411. distllm/core/route_audit.py +175 -0
  412. distllm/core/routing_extensions.py +604 -0
  413. distllm/core/schedule_simulator.py +289 -0
  414. distllm/core/schedule_viz.py +233 -0
  415. distllm/core/scheduler/__init__.py +34 -0
  416. distllm/core/scheduler/budget.py +25 -0
  417. distllm/core/scheduler/budget_computer.py +322 -0
  418. distllm/core/scheduler/chunked_prefill.py +52 -0
  419. distllm/core/scheduler/kv_cache_manager.py +313 -0
  420. distllm/core/scheduler/preemption_manager.py +251 -0
  421. distllm/core/scheduler/pressure.py +48 -0
  422. distllm/core/scheduler/sequence.py +179 -0
  423. distllm/core/secret_manager.py +315 -0
  424. distllm/core/semantic_cache.py +369 -0
  425. distllm/core/sentinel_qos.py +833 -0
  426. distllm/core/shadow_eval_runner.py +194 -0
  427. distllm/core/shared_layer_pool.py +279 -0
  428. distllm/core/smart_model_router.py +309 -0
  429. distllm/core/spec_equivalence.py +140 -0
  430. distllm/core/spec_verify.py +225 -0
  431. distllm/core/speculative_adaptor.py +108 -0
  432. distllm/core/speculative_dashboard.py +134 -0
  433. distllm/core/speculative_decoder.py +1286 -0
  434. distllm/core/speculative_profiler.py +132 -0
  435. distllm/core/split_brain.py +236 -0
  436. distllm/core/spot_failover.py +292 -0
  437. distllm/core/spot_forecasting.py +463 -0
  438. distllm/core/starvation_monitor.py +70 -0
  439. distllm/core/state_replication.py +261 -0
  440. distllm/core/stats_collector.py +64 -0
  441. distllm/core/step_processor.py +115 -0
  442. distllm/core/straggler_alerts.py +174 -0
  443. distllm/core/straggler_aware_scheduler.py +255 -0
  444. distllm/core/straggler_detector.py +29 -0
  445. distllm/core/streaming_cost.py +286 -0
  446. distllm/core/streaming_generator.py +228 -0
  447. distllm/core/structured_output/__init__.py +348 -0
  448. distllm/core/structured_output/config.py +38 -0
  449. distllm/core/structured_output/engine.py +367 -0
  450. distllm/core/structured_output/streaming.py +183 -0
  451. distllm/core/structured_output/validator.py +199 -0
  452. distllm/core/subsystem_registry.py +171 -0
  453. distllm/core/synapse_debugger.py +226 -0
  454. distllm/core/synth_data_generator.py +122 -0
  455. distllm/core/telemetry.py +240 -0
  456. distllm/core/tenant_billing.py +446 -0
  457. distllm/core/tenant_cost_attribution.py +314 -0
  458. distllm/core/token_generator.py +368 -0
  459. distllm/core/token_streaming_buffer.py +147 -0
  460. distllm/core/topology_aware_lb.py +324 -0
  461. distllm/core/topology_aware_tiering.py +344 -0
  462. distllm/core/tree_speculative_decoder.py +405 -0
  463. distllm/core/types.py +17 -0
  464. distllm/core/unified_router.py +439 -0
  465. distllm/core/unified_sla_router.py +242 -0
  466. distllm/core/usage_meter.py +915 -0
  467. distllm/core/vectorstore/__init__.py +30 -0
  468. distllm/core/vectorstore/base.py +106 -0
  469. distllm/core/vectorstore/chroma_store.py +99 -0
  470. distllm/core/vectorstore/factory.py +97 -0
  471. distllm/core/vectorstore/pgvector_store.py +123 -0
  472. distllm/core/vectorstore/providers/__init__.py +7 -0
  473. distllm/core/vectorstore/providers/milvus.py +219 -0
  474. distllm/core/vectorstore/providers/pinecone.py +142 -0
  475. distllm/core/vectorstore/providers/qdrant.py +176 -0
  476. distllm/core/vectorstore/providers/weaviate.py +185 -0
  477. distllm/core/vectorstore/qdrant_store.py +148 -0
  478. distllm/core/vectorstore/rag.py +151 -0
  479. distllm/core/vllm_backend.py +21 -0
  480. distllm/core/vllm_node.py +174 -0
  481. distllm/core/voyager_multimodal.py +1354 -0
  482. distllm/core/webgpu_manager.py +382 -0
  483. distllm/core/webhook_formatters.py +243 -0
  484. distllm/core/webhook_manager.py +433 -0
  485. distllm/core/wisp_wasm.py +934 -0
  486. distllm/core/workload_classifier.py +137 -0
  487. distllm/dashboard/__init__.py +17 -0
  488. distllm/dashboard/verification_history.py +192 -0
  489. distllm/dashboard/ws_handler.py +586 -0
  490. distllm/dist/__init__.py +85 -0
  491. distllm/dist/__init__.pyi +132 -0
  492. distllm/dist/_pipeline_original.py +2916 -0
  493. distllm/dist/adaptive_speculator.py +384 -0
  494. distllm/dist/admin_cli.py +293 -0
  495. distllm/dist/api_docs.py +594 -0
  496. distllm/dist/async_pipeline.py +578 -0
  497. distllm/dist/attention.py +1699 -0
  498. distllm/dist/attention_block_pool.py +634 -0
  499. distllm/dist/attention_distributed.py +138 -0
  500. distllm/dist/autoscaler.py +196 -0
  501. distllm/dist/backends/__init__.py +29 -0
  502. distllm/dist/backends/backend_profiles.py +357 -0
  503. distllm/dist/backends/graceful_degradation.py +409 -0
  504. distllm/dist/backends/health_monitor.py +518 -0
  505. distllm/dist/backends/llamacpp.py +125 -0
  506. distllm/dist/backends/ray.py +89 -0
  507. distllm/dist/backends/vllm.py +177 -0
  508. distllm/dist/block_pool.py +662 -0
  509. distllm/dist/block_transfer_service.py +267 -0
  510. distllm/dist/byzantine.py +2193 -0
  511. distllm/dist/cache.py +305 -0
  512. distllm/dist/cache_digest.py +405 -0
  513. distllm/dist/chaos/__init__.py +52 -0
  514. distllm/dist/chaos/chaos_orchestrator.py +229 -0
  515. distllm/dist/chaos/fault_injector.py +1273 -0
  516. distllm/dist/chunked_prefill.py +136 -0
  517. distllm/dist/cloud_selector.py +257 -0
  518. distllm/dist/config.py +70 -0
  519. distllm/dist/cross_cluster.py +231 -0
  520. distllm/dist/daas/__init__.py +46 -0
  521. distllm/dist/daas/load_balancer.py +275 -0
  522. distllm/dist/daas/marketplace_integration.py +374 -0
  523. distllm/dist/daas/resource_isolation.py +280 -0
  524. distllm/dist/daas/tenant_dispatcher.py +252 -0
  525. distllm/dist/daas/usage_meter.py +360 -0
  526. distllm/dist/daas_server.py +514 -0
  527. distllm/dist/disagg/__init__.py +162 -0
  528. distllm/dist/disagg/pool.py +214 -0
  529. distllm/dist/disagg/transfer.py +77 -0
  530. distllm/dist/disagg/transfer_compression.py +793 -0
  531. distllm/dist/disagg/transfer_engine.py +998 -0
  532. distllm/dist/discovery.py +166 -0
  533. distllm/dist/draft_bank.py +344 -0
  534. distllm/dist/draft_migration.py +382 -0
  535. distllm/dist/ebpf_transport.py +364 -0
  536. distllm/dist/edge_cloud.py +544 -0
  537. distllm/dist/edge_federation.py +426 -0
  538. distllm/dist/execution_planner.py +148 -0
  539. distllm/dist/federated_merge.py +560 -0
  540. distllm/dist/federation.py +1114 -0
  541. distllm/dist/flash_attention.py +237 -0
  542. distllm/dist/fsdp.py +295 -0
  543. distllm/dist/geo.py +361 -0
  544. distllm/dist/ice_transport.py +2968 -0
  545. distllm/dist/kv_migration.py +237 -0
  546. distllm/dist/latency.py +60 -0
  547. distllm/dist/lifecycle.py +467 -0
  548. distllm/dist/marketplace.py +891 -0
  549. distllm/dist/merkle.py +178 -0
  550. distllm/dist/metrics.py +456 -0
  551. distllm/dist/model_store.py +107 -0
  552. distllm/dist/multi_tenant.py +339 -0
  553. distllm/dist/multimodal.py +392 -0
  554. distllm/dist/nat.py +934 -0
  555. distllm/dist/nccl.py +733 -0
  556. distllm/dist/network.py +126 -0
  557. distllm/dist/node_client.py +568 -0
  558. distllm/dist/node_pb2.py +66 -0
  559. distllm/dist/node_pb2.pyi +236 -0
  560. distllm/dist/node_pb2_grpc.py +312 -0
  561. distllm/dist/node_pb2_grpc.pyi +41 -0
  562. distllm/dist/node_registrar.py +247 -0
  563. distllm/dist/node_service.py +572 -0
  564. distllm/dist/observability.py +258 -0
  565. distllm/dist/otel.py +158 -0
  566. distllm/dist/p2p/__init__.py +22 -0
  567. distllm/dist/p2p/discovery.py +132 -0
  568. distllm/dist/p2p/gossip.py +1339 -0
  569. distllm/dist/p2p/kademlia_dht.py +1058 -0
  570. distllm/dist/p2p/load_balancer.py +131 -0
  571. distllm/dist/p2p/quic_transport.py +700 -0
  572. distllm/dist/p2p/router.py +607 -0
  573. distllm/dist/p2p/transport.py +399 -0
  574. distllm/dist/p2p_model_distributor.py +160 -0
  575. distllm/dist/paged_attention_kernel.py +281 -0
  576. distllm/dist/parallel.py +1061 -0
  577. distllm/dist/parallel_planner.py +586 -0
  578. distllm/dist/parallel_topology.py +231 -0
  579. distllm/dist/partition/__init__.py +146 -0
  580. distllm/dist/partition/adaptive.py +532 -0
  581. distllm/dist/partition/benchmark_suite.py +479 -0
  582. distllm/dist/partition/cli.py +189 -0
  583. distllm/dist/partition/cloud_arbitrage.py +523 -0
  584. distllm/dist/partition/config.py +97 -0
  585. distllm/dist/partition/cost_model.py +553 -0
  586. distllm/dist/partition/learned_cost.py +653 -0
  587. distllm/dist/partition/mixed_precision_tuner.py +310 -0
  588. distllm/dist/partition/network_cost_model.py +664 -0
  589. distllm/dist/partition/optimizer.py +650 -0
  590. distllm/dist/partition/pareto_optimizer.py +466 -0
  591. distllm/dist/partition/partitioner.py +363 -0
  592. distllm/dist/partition/persistence.py +392 -0
  593. distllm/dist/partition/plugins.py +270 -0
  594. distllm/dist/partition/profiles.py +659 -0
  595. distllm/dist/partition/quant_bench.py +373 -0
  596. distllm/dist/partition/quant_calibrate.py +229 -0
  597. distllm/dist/partition/quant_coordinator.py +260 -0
  598. distllm/dist/partition/quant_cost.py +293 -0
  599. distllm/dist/partition/quant_partition.py +432 -0
  600. distllm/dist/partition/quant_report.py +301 -0
  601. distllm/dist/partition/quantization_metrics.py +355 -0
  602. distllm/dist/partition/quantization_search.py +290 -0
  603. distllm/dist/partition/quantization_tuner.py +1383 -0
  604. distllm/dist/partition/topology.py +218 -0
  605. distllm/dist/partition/validator.py +405 -0
  606. distllm/dist/partition/visualizer.py +275 -0
  607. distllm/dist/pipeline/__init__.py +71 -0
  608. distllm/dist/pipeline/bandwidth_controller.py +780 -0
  609. distllm/dist/pipeline/compression_negotiation.py +771 -0
  610. distllm/dist/pipeline/context.py +80 -0
  611. distllm/dist/pipeline/continuous_batching.py +435 -0
  612. distllm/dist/pipeline/disagg_orchestrator.py +284 -0
  613. distllm/dist/pipeline/orchestrator.py +727 -0
  614. distllm/dist/pipeline/pipeline_reconfig.py +1395 -0
  615. distllm/dist/pipeline/profiler.py +84 -0
  616. distllm/dist/pipeline/serialization.py +176 -0
  617. distllm/dist/pipeline/simulator.py +172 -0
  618. distllm/dist/pipeline/strategy.py +193 -0
  619. distllm/dist/pipeline/token_accumulator.py +211 -0
  620. distllm/dist/pipeline/transport.py +230 -0
  621. distllm/dist/pipeline/wan_disagg_orchestrator.py +564 -0
  622. distllm/dist/power_cap.py +358 -0
  623. distllm/dist/predictive_cache.py +399 -0
  624. distllm/dist/preemption.py +198 -0
  625. distllm/dist/prefix_cache.py +461 -0
  626. distllm/dist/prefix_clustering.py +305 -0
  627. distllm/dist/privacy/__init__.py +411 -0
  628. distllm/dist/privacy/privacy_accountant.py +344 -0
  629. distllm/dist/privacy.py +240 -0
  630. distllm/dist/provisioning.py +338 -0
  631. distllm/dist/quality.py +118 -0
  632. distllm/dist/quic_transport.py +335 -0
  633. distllm/dist/quota_enforcer.py +265 -0
  634. distllm/dist/rebalancer.py +234 -0
  635. distllm/dist/recovery.py +827 -0
  636. distllm/dist/recovery_drill.py +281 -0
  637. distllm/dist/redis_cache.py +265 -0
  638. distllm/dist/redundant.py +308 -0
  639. distllm/dist/redundant_executor.py +1536 -0
  640. distllm/dist/reputation.py +231 -0
  641. distllm/dist/router.py +382 -0
  642. distllm/dist/routing/__init__.py +19 -0
  643. distllm/dist/routing/arbitrage_engine.py +575 -0
  644. distllm/dist/routing/composite.py +179 -0
  645. distllm/dist/routing/consistent_hash.py +144 -0
  646. distllm/dist/routing/latency_aware.py +116 -0
  647. distllm/dist/routing/load_aware.py +114 -0
  648. distllm/dist/scheduling/__init__.py +38 -0
  649. distllm/dist/scheduling/batcher.py +95 -0
  650. distllm/dist/scheduling/classifier.py +153 -0
  651. distllm/dist/scheduling/deadline_scheduler.py +958 -0
  652. distllm/dist/scheduling/iteration.py +404 -0
  653. distllm/dist/scheduling/profiler.py +57 -0
  654. distllm/dist/simulation/__init__.py +54 -0
  655. distllm/dist/simulation/chaos_simulator.py +901 -0
  656. distllm/dist/simulation/cluster_simulator.py +667 -0
  657. distllm/dist/simulation/cost_aware_provisioning.py +370 -0
  658. distllm/dist/simulation/digital_twin.py +617 -0
  659. distllm/dist/simulation/topology_optimizer.py +747 -0
  660. distllm/dist/simulation/what_if.py +492 -0
  661. distllm/dist/speculative/__init__.py +47 -0
  662. distllm/dist/speculative/adaptive_spec.py +291 -0
  663. distllm/dist/speculative/draft_cache.py +246 -0
  664. distllm/dist/speculative/draft_orchestrator.py +634 -0
  665. distllm/dist/speculative/draft_registry.py +439 -0
  666. distllm/dist/speculative/multi_draft.py +304 -0
  667. distllm/dist/speculative/online_sisd.py +523 -0
  668. distllm/dist/straggler.py +819 -0
  669. distllm/dist/streaming_kv_transfer.py +174 -0
  670. distllm/dist/structured_output/__init__.py +7 -0
  671. distllm/dist/structured_output/engine.py +144 -0
  672. distllm/dist/tensor_pool.py +120 -0
  673. distllm/dist/tgi_compat.py +254 -0
  674. distllm/dist/topology.py +141 -0
  675. distllm/dist/topology_consensus.py +1961 -0
  676. distllm/dist/topology_dynamic.py +270 -0
  677. distllm/dist/tp_inprocess.py +214 -0
  678. distllm/dist/tp_launcher.py +310 -0
  679. distllm/dist/tracing.py +325 -0
  680. distllm/dist/transport.py +493 -0
  681. distllm/dist/wan_speculative.py +295 -0
  682. distllm/dist/wandb_integration.py +252 -0
  683. distllm/dist/webrtc.py +846 -0
  684. distllm/dist/wide_area.py +515 -0
  685. distllm/dist/worker.py +635 -0
  686. distllm/dist/zero_copy.py +307 -0
  687. distllm/errors/__init__.py +105 -0
  688. distllm/errors/policies.py +106 -0
  689. distllm/errors/retry.py +168 -0
  690. distllm/errors/types.py +721 -0
  691. distllm/health/__init__.py +6 -0
  692. distllm/health/failover.py +81 -0
  693. distllm/health/prober.py +31 -0
  694. distllm/health/service.py +173 -0
  695. distllm/health/state.py +76 -0
  696. distllm/integrations/__init__.py +72 -0
  697. distllm/integrations/_common/__init__.py +7 -0
  698. distllm/integrations/_common/base_tool_provider.py +189 -0
  699. distllm/integrations/_common/cost_tracker.py +144 -0
  700. distllm/integrations/_common/model_router.py +148 -0
  701. distllm/integrations/ci/__init__.py +24 -0
  702. distllm/integrations/ci/_common.py +92 -0
  703. distllm/integrations/ci/gitlab.py +345 -0
  704. distllm/integrations/ci/jenkins.py +273 -0
  705. distllm/integrations/mlflow_tracking.py +372 -0
  706. distllm/integrations/spark_connector.py +909 -0
  707. distllm/integrations/wandb/__init__.py +81 -0
  708. distllm/models/__init__.py +40 -0
  709. distllm/models/_trust_remote.py +48 -0
  710. distllm/models/adapter.py +768 -0
  711. distllm/models/adapter_router.py +198 -0
  712. distllm/models/cache.py +172 -0
  713. distllm/models/model_hub.py +650 -0
  714. distllm/models/partition_planner.py +384 -0
  715. distllm/models/partitioner.py +784 -0
  716. distllm/models/rope_scaling.py +119 -0
  717. distllm/models/safetensors_index.py +179 -0
  718. distllm/observability/__init__.py +107 -0
  719. distllm/observability/alerting_config.py +881 -0
  720. distllm/observability/capacity_models.py +398 -0
  721. distllm/observability/capacity_planning.py +2870 -0
  722. distllm/observability/exporter.py +228 -0
  723. distllm/observability/exporter_datadog.py +1362 -0
  724. distllm/observability/health_config.py +1282 -0
  725. distllm/observability/incident_response.py +779 -0
  726. distllm/observability/logging.py +211 -0
  727. distllm/observability/logging_config.py +864 -0
  728. distllm/observability/loki_sink.py +178 -0
  729. distllm/observability/metrics.py +111 -0
  730. distllm/observability/metrics_completeness.py +393 -0
  731. distllm/observability/otel_logging.py +290 -0
  732. distllm/observability/self_healing_config.py +1308 -0
  733. distllm/observability/slo_config.py +1520 -0
  734. distllm/observability/spans.py +235 -0
  735. distllm/observability/tracing.py +268 -0
  736. distllm/observability/tracing_config.py +676 -0
  737. distllm/observability/wandb_monitor.py +66 -0
  738. distllm/plugins/__init__.py +14 -0
  739. distllm/plugins/airflow.py +110 -0
  740. distllm/plugins/auth_plugin.py +576 -0
  741. distllm/plugins/builtin.py +376 -0
  742. distllm/plugins/cache_plugin.py +576 -0
  743. distllm/plugins/health_plugin.py +396 -0
  744. distllm/plugins/kubeflow.py +84 -0
  745. distllm/plugins/mlflow_plugin.py +71 -0
  746. distllm/plugins/registry.py +135 -0
  747. distllm/prompts/__init__.py +39 -0
  748. distllm/prompts/analysis.py +103 -0
  749. distllm/prompts/code.py +172 -0
  750. distllm/prompts/creative.py +115 -0
  751. distllm/prompts/education.py +91 -0
  752. distllm/prompts/engine.py +119 -0
  753. distllm/prompts/exchange.py +705 -0
  754. distllm/prompts/language.py +81 -0
  755. distllm/prompts/library.py +1198 -0
  756. distllm/prompts/management.py +35 -0
  757. distllm/prompts/productivity.py +86 -0
  758. distllm/prompts/professional.py +111 -0
  759. distllm/prompts/prompt_def.py +33 -0
  760. distllm/prompts/reasoning.py +108 -0
  761. distllm/prompts/specialized.py +140 -0
  762. distllm/prompts/templates.py +128 -0
  763. distllm/prompts/writing.py +140 -0
  764. distllm/py.typed +0 -0
  765. distllm/sdk/__init__.py +82 -0
  766. distllm/sdk/circuit_breaker.py +139 -0
  767. distllm/sdk/client.py +1191 -0
  768. distllm/sdk/observability.py +42 -0
  769. distllm/sdk/streaming.py +228 -0
  770. distllm/sdk/transport.py +224 -0
  771. distllm/sdk/types.py +288 -0
  772. distllm/security/__init__.py +63 -0
  773. distllm/security/content_moderation/__init__.py +44 -0
  774. distllm/security/content_moderation/base.py +405 -0
  775. distllm/security/content_moderation/jailbreak.py +192 -0
  776. distllm/security/content_moderation/pii.py +262 -0
  777. distllm/security/content_moderation/pipeline.py +272 -0
  778. distllm/security/content_moderation/topics.py +153 -0
  779. distllm/security/content_moderation/toxicity.py +131 -0
  780. distllm/security/e2e.py +437 -0
  781. distllm/security/edge_attestation.py +381 -0
  782. distllm/security/log_redaction.py +244 -0
  783. distllm/security/quantum_safe_tls.py +314 -0
  784. distllm/security/spiffe.py +347 -0
  785. distllm/security/tee.py +461 -0
  786. distllm/security/utils.py +139 -0
  787. distllm/security/watermark.py +779 -0
  788. distllm/terraform/__init__.py +260 -0
  789. distllm/ui/__init__.py +7 -0
  790. distllm/ui/app.py +98 -0
  791. distllm/utils/__init__.py +0 -0
  792. distllm/utils/gbnf_grammar.py +99 -0
  793. distllm/utils/lazy_imports.py +90 -0
  794. distllm/utils/scheduling.py +38 -0
  795. distllm/verification/__init__.py +49 -0
  796. distllm/verification/comparator.py +263 -0
  797. distllm/verification/hash_registry.py +220 -0
  798. distllm/verification/report.py +240 -0
  799. distllm/verification/runner.py +822 -0
  800. distllm/verification/test_verify.py +205 -0
  801. distributed_llm-0.4.1.dist-info/METADATA +315 -0
  802. distributed_llm-0.4.1.dist-info/RECORD +806 -0
  803. distributed_llm-0.4.1.dist-info/WHEEL +5 -0
  804. distributed_llm-0.4.1.dist-info/entry_points.txt +5 -0
  805. distributed_llm-0.4.1.dist-info/licenses/LICENSE +97 -0
  806. distributed_llm-0.4.1.dist-info/top_level.txt +1 -0
@@ -0,0 +1,136 @@
1
+ """Chunked prefill for predictable latency with long prompts.
2
+
3
+ Splits very long prompts into manageable chunks and tracks progress
4
+ so the KV cache can be built incrementally across pipeline iterations.
5
+ """
6
+
7
+
8
+ from __future__ import annotations
9
+ import math
10
+ from dataclasses import dataclass
11
+ from typing import List, Optional
12
+
13
+ # Default max context — overridden by model config when available
14
+ DEFAULT_MAX_CONTEXT_TOKENS: int = 131072
15
+
16
+ # Known model context lengths (matched by substring in model name)
17
+ _MODEL_CONTEXT_LENGTHS: dict[str, int] = {
18
+ "llama-3": 131072,
19
+ "llama3": 131072,
20
+ "qwen2.5": 131072,
21
+ "qwen-2.5": 131072,
22
+ "mistral": 32768,
23
+ "mixtral": 32768,
24
+ "gemma": 8192,
25
+ "phi-3": 131072,
26
+ "phi3": 131072,
27
+ "command-r": 128000,
28
+ "deepseek": 131072,
29
+ "yi": 200000,
30
+ "internlm": 32768,
31
+ "codellama": 16384,
32
+ }
33
+
34
+
35
+ def get_max_context_tokens(
36
+ model_name: str = "",
37
+ model_info: dict | None = None,
38
+ ) -> int:
39
+ """Derive max context tokens from model name or config.
40
+
41
+
42
+ Checks (in order):
43
+ 1. model_info["max_position_embeddings"] if present
44
+ 2. model_info["max_seq_len"] if present
45
+ 3. Known model name substring match
46
+ 4. DEFAULT_MAX_CONTEXT_TOKENS (131072)
47
+ """
48
+
49
+ if model_info:
50
+ mpe = model_info.get("max_position_embeddings")
51
+ if mpe and mpe > 0:
52
+ return int(mpe)
53
+ msl = model_info.get("max_seq_len")
54
+ if msl and msl > 0:
55
+ return int(msl)
56
+
57
+ name_lower = model_name.lower()
58
+ for pattern, ctx_len in _MODEL_CONTEXT_LENGTHS.items():
59
+ if pattern in name_lower:
60
+ return ctx_len
61
+
62
+ return DEFAULT_MAX_CONTEXT_TOKENS
63
+
64
+
65
+ @dataclass
66
+ class ChunkState:
67
+ prompt_tokens: List[int]
68
+ chunk_size: int
69
+ current_offset: int = 0
70
+
71
+ @property
72
+ def remaining(self) -> List[int]:
73
+ return self.prompt_tokens[self.current_offset:]
74
+
75
+ @property
76
+ def is_done(self) -> bool:
77
+ return self.current_offset >= len(self.prompt_tokens)
78
+
79
+ def next_chunk(self) -> List[int]:
80
+ end = min(self.current_offset + self.chunk_size, len(self.prompt_tokens))
81
+ chunk = self.prompt_tokens[self.current_offset:end]
82
+ self.current_offset = end
83
+ return chunk
84
+
85
+ @property
86
+ def position_offset(self) -> int:
87
+ return self.current_offset
88
+
89
+ @property
90
+ def chunks_total(self) -> int:
91
+ return math.ceil(len(self.prompt_tokens) / self.chunk_size)
92
+
93
+ @property
94
+ def chunks_done(self) -> int:
95
+ pos = self.current_offset
96
+ return min(pos // self.chunk_size + (1 if pos % self.chunk_size > 0 else 0), self.chunks_total)
97
+
98
+
99
+ def maybe_chunk(
100
+ token_ids: List[int],
101
+ chunk_size: int,
102
+ enabled: bool = True,
103
+ max_context_tokens: int | None = None,
104
+ model_name: str = "",
105
+ model_info: dict | None = None,
106
+ ) -> Optional[ChunkState]:
107
+ """Optionally split a prompt into chunks for incremental prefill.
108
+
109
+
110
+ Args:
111
+ token_ids: Full prompt token IDs.
112
+ chunk_size: Maximum tokens per chunk.
113
+ enabled: If False, always returns None (no chunking).
114
+ max_context_tokens: Override for max context length. If None,
115
+ derived from model_name/model_info via get_max_context_tokens().
116
+ model_name: Model name for auto-detecting context length.
117
+ model_info: Model config dict for auto-detecting context length.
118
+
119
+ Returns:
120
+ ChunkState if chunking is needed, None if prompt fits in one chunk.
121
+
122
+ Raises:
123
+ ValueError: If prompt exceeds max context length.
124
+ """
125
+
126
+ if max_context_tokens is None:
127
+ max_context_tokens = get_max_context_tokens(model_name, model_info)
128
+
129
+ if not enabled or len(token_ids) <= chunk_size:
130
+ return None
131
+ if len(token_ids) > max_context_tokens:
132
+ raise ValueError(
133
+ f"Prompt length ({len(token_ids)} tokens) exceeds maximum "
134
+ f"context length ({max_context_tokens} tokens)"
135
+ )
136
+ return ChunkState(prompt_tokens=token_ids, chunk_size=chunk_size)
@@ -0,0 +1,257 @@
1
+ """Automatic cloud region selection — deploy to the cheapest available region.
2
+
3
+ Queries cloud provider pricing APIs (AWS, GCP, Azure) to find the region
4
+ with the lowest GPU instance cost that meets the model's hardware requirements.
5
+ Integrates with :class:`GeoRouter` so spillover decisions consider both
6
+ latency *and* cost.
7
+
8
+ Usage::
9
+
10
+ selector = CloudRegionSelector(providers=["aws", "gcp"])
11
+ best = selector.find_cheapest_region(
12
+ model_name="llama-70b",
13
+ required_gpu_memory_gb=80,
14
+ min_gpu_count=4,
15
+ )
16
+ # → {"region": "us-east-1", "provider": "aws",
17
+ # "price_per_hour": 32.44, "gpu_type": "A100"}
18
+ """
19
+
20
+ from __future__ import annotations
21
+
22
+ import json
23
+ import os
24
+ import threading
25
+ import time
26
+ from dataclasses import dataclass
27
+ from typing import Any
28
+
29
+ from loguru import logger
30
+
31
+
32
+ @dataclass
33
+ class RegionOffer:
34
+ """A region that has the required GPU capacity."""
35
+ provider: str # "aws", "gcp", "azure"
36
+ region: str # e.g. "us-east-1"
37
+ gpu_type: str # e.g. "A100", "H100", "L40S"
38
+ gpu_count: int # GPUs per instance
39
+ price_per_hour: float # USD
40
+ spot_price_per_hour: float # USD (spot/preemptible)
41
+ spot_available: bool = True
42
+ latency_ms: float = 0.0 # estimated from GeoRouter
43
+ carbon_intensity: float = 0.0 # gCO2/kWh
44
+
45
+ @property
46
+ def is_cost_effective(self) -> bool:
47
+ return self.price_per_hour > 0
48
+
49
+
50
+ # ── Static pricing data (fallback when APIs are unreachable) ──────────
51
+ # These are representative prices for popular GPU instances across
52
+ # providers as of 2026-06. In production the selector queries live APIs.
53
+
54
+ _FALLBACK_PRICING: dict[str, list[dict[str, Any]]] = {
55
+ "aws": [
56
+ {"region": "us-east-1", "gpu_type": "A100", "gpu_count": 8, "price": 32.44, "spot": 9.73},
57
+ {"region": "us-east-1", "gpu_type": "H100", "gpu_count": 8, "price": 40.96, "spot": 12.29},
58
+ {"region": "us-west-2", "gpu_type": "A100", "gpu_count": 8, "price": 34.56, "spot": 10.37},
59
+ {"region": "eu-west-1", "gpu_type": "A100", "gpu_count": 8, "price": 37.89, "spot": 11.37},
60
+ {"region": "ap-southeast-1", "gpu_type": "A100", "gpu_count": 8, "price": 39.12, "spot": 11.74},
61
+ {"region": "us-east-1", "gpu_type": "L40S", "gpu_count": 8, "price": 18.72, "spot": 5.62},
62
+ ],
63
+ "gcp": [
64
+ {"region": "us-central1", "gpu_type": "A100", "gpu_count": 8, "price": 29.52, "spot": 8.86},
65
+ {"region": "us-central1", "gpu_type": "H100", "gpu_count": 8, "price": 37.84, "spot": 11.35},
66
+ {"region": "europe-west4", "gpu_type": "A100", "gpu_count": 8, "price": 34.99, "spot": 10.50},
67
+ ],
68
+ "azure": [
69
+ {"region": "eastus", "gpu_type": "A100", "gpu_count": 8, "price": 35.28, "spot": 10.58},
70
+ {"region": "westus3", "gpu_type": "H100", "gpu_count": 8, "price": 45.67, "spot": 13.70},
71
+ ],
72
+ }
73
+
74
+
75
+ class CloudRegionSelector:
76
+ """Selects the cheapest cloud region for a given GPU requirement.
77
+
78
+ Caches pricing data with a TTL to avoid repeated API calls.
79
+ Falls back to static pricing when cloud APIs are unreachable.
80
+ """
81
+
82
+ def __init__(
83
+ self,
84
+ providers: list[str] | None = None,
85
+ prefer_spot: bool = True,
86
+ cache_ttl_s: float = 3600.0,
87
+ api_keys: dict[str, str] | None = None,
88
+ ):
89
+ self._providers = providers or ["aws", "gcp", "azure"]
90
+ self._prefer_spot = prefer_spot
91
+ self._cache_ttl_s = cache_ttl_s
92
+ self._api_keys = api_keys or {}
93
+ self._cached_offers: list[RegionOffer] | None = None
94
+ self._cache_time: float = 0.0
95
+ self._lock = threading.Lock()
96
+
97
+ def find_cheapest_region(
98
+ self,
99
+ model_name: str = "",
100
+ required_gpu_memory_gb: float = 80.0,
101
+ min_gpu_count: int = 1,
102
+ max_price_per_hour: float = float("inf"),
103
+ ) -> RegionOffer | None:
104
+ """Find the cheapest region meeting GPU requirements.
105
+
106
+ Args:
107
+ model_name: Optional model name for filtering compatible GPUs.
108
+ required_gpu_memory_gb: Minimum GPU memory per device.
109
+ min_gpu_count: Minimum number of GPUs.
110
+ max_price_per_hour: Maximum acceptable price per hour.
111
+
112
+ Returns:
113
+ The cheapest :class:`RegionOffer`, or ``None`` if no region
114
+ meets the constraints.
115
+ """
116
+ offers = self._get_offers()
117
+ candidates = [
118
+ o for o in offers
119
+ if o.gpu_count >= min_gpu_count
120
+ and self._gpu_matches(o.gpu_type, required_gpu_memory_gb)
121
+ and o.price_per_hour <= max_price_per_hour
122
+ ]
123
+
124
+ if not candidates:
125
+ logger.warning(f"No region found: {min_gpu_count}× GPU with "
126
+ f"{required_gpu_memory_gb}GB at <${max_price_per_hour}/hr")
127
+ return None
128
+
129
+ # Sort by effective hourly cost.
130
+ key = lambda o: o.spot_price_per_hour if self._prefer_spot and o.spot_available else o.price_per_hour
131
+ candidates.sort(key=key)
132
+ best = candidates[0]
133
+
134
+ logger.info(
135
+ f"Cheapest region: {best.provider}/{best.region} "
136
+ f"({best.gpu_type}×{best.gpu_count}) "
137
+ f"@ ${best.price_per_hour:.2f}/hr"
138
+ )
139
+ return best
140
+
141
+ def list_regions(
142
+ self,
143
+ required_gpu_memory_gb: float = 80.0,
144
+ ) -> list[dict[str, Any]]:
145
+ """List all available regions with pricing (for dashboard)."""
146
+ offers = self._get_offers()
147
+ return [
148
+ {
149
+ "provider": o.provider,
150
+ "region": o.region,
151
+ "gpu_type": o.gpu_type,
152
+ "gpu_count": o.gpu_count,
153
+ "price_per_hour": o.price_per_hour,
154
+ "spot_price": o.spot_price_per_hour,
155
+ }
156
+ for o in offers
157
+ if self._gpu_matches(o.gpu_type, required_gpu_memory_gb)
158
+ ]
159
+
160
+ # ── Internal ──────────────────────────────────────────────────────
161
+
162
+ def _get_offers(self) -> list[RegionOffer]:
163
+ """Return cached or freshly-fetched offers."""
164
+ now = time.time()
165
+ with self._lock:
166
+ if self._cached_offers and now - self._cache_time < self._cache_ttl_s:
167
+ return self._cached_offers
168
+
169
+ offers = self._fetch_offers()
170
+ self._cached_offers = offers
171
+ self._cache_time = now
172
+ return offers
173
+
174
+ def _fetch_offers(self) -> list[RegionOffer]:
175
+ """Fetch pricing from cloud provider APIs, falling back to static data."""
176
+ all_offers: list[RegionOffer] = []
177
+
178
+ for provider in self._providers:
179
+ offers = self._fetch_provider(provider)
180
+ all_offers.extend(offers)
181
+
182
+ if not all_offers:
183
+ logger.info("Cloud APIs unreachable — using static pricing data")
184
+ for provider in self._providers:
185
+ all_offers.extend(self._static_offers(provider))
186
+
187
+ return all_offers
188
+
189
+ def _fetch_provider(self, provider: str) -> list[RegionOffer]:
190
+ """Attempt to fetch live pricing from a cloud provider.
191
+
192
+ Returns an empty list on failure (caller falls back to static).
193
+ """
194
+ fetcher = {
195
+ "aws": self._fetch_aws,
196
+ "gcp": self._fetch_gcp,
197
+ "azure": self._fetch_azure,
198
+ }.get(provider)
199
+
200
+ if fetcher is None:
201
+ return []
202
+
203
+ try:
204
+ return fetcher()
205
+ except Exception as e:
206
+ logger.debug(f"Failed to fetch {provider} pricing: {e}")
207
+ return []
208
+
209
+ def _fetch_aws(self) -> list[RegionOffer]:
210
+ """Fetch AWS GPU pricing via the Price List API."""
211
+ api_key = self._api_keys.get("aws", os.environ.get("AWS_PRICING_API_KEY", ""))
212
+ if not api_key:
213
+ return []
214
+ # Placeholder — real implementation would call
215
+ # https://api.pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonEC2/current/index.json
216
+ # and filter for GPU instance types.
217
+ return []
218
+
219
+ def _fetch_gcp(self) -> list[RegionOffer]:
220
+ api_key = self._api_keys.get("gcp", os.environ.get("GCP_PRICING_API_KEY", ""))
221
+ if not api_key:
222
+ return []
223
+ return []
224
+
225
+ def _fetch_azure(self) -> list[RegionOffer]:
226
+ api_key = self._api_keys.get("azure", os.environ.get("AZURE_PRICING_API_KEY", ""))
227
+ if not api_key:
228
+ return []
229
+ return []
230
+
231
+ @staticmethod
232
+ def _static_offers(provider: str) -> list[RegionOffer]:
233
+ """Return fallback pricing data for a provider."""
234
+ data = _FALLBACK_PRICING.get(provider, [])
235
+ return [
236
+ RegionOffer(
237
+ provider=provider,
238
+ region=entry["region"],
239
+ gpu_type=entry["gpu_type"],
240
+ gpu_count=entry["gpu_count"],
241
+ price_per_hour=entry["price"],
242
+ spot_price_per_hour=entry["spot"],
243
+ )
244
+ for entry in data
245
+ ]
246
+
247
+ @staticmethod
248
+ def _gpu_matches(gpu_type: str, required_gb: float) -> bool:
249
+ """Rough GPU memory lookup by type."""
250
+ memory_map = {
251
+ "A100": 80.0, "A100-40GB": 40.0,
252
+ "H100": 80.0, "H200": 141.0,
253
+ "L40S": 48.0, "L4": 24.0,
254
+ "V100": 32.0, "T4": 16.0,
255
+ }
256
+ available = memory_map.get(gpu_type, 80.0)
257
+ return available >= required_gb
distllm/dist/config.py ADDED
@@ -0,0 +1,70 @@
1
+ """Configuration for wide-area distributed inference.
2
+
3
+ Uses pydantic.BaseSettings so every field can be overridden via environment
4
+ variable with the ``WIDE_AREA_`` prefix — e.g. ``WIDE_AREA_TIMEOUT=60``
5
+ overrides :attr:`WideAreaConfig.wan_timeout_seconds`.
6
+ """
7
+
8
+ from __future__ import annotations
9
+ from typing import Literal
10
+
11
+ from pydantic import Field, field_validator
12
+ from pydantic_settings import BaseSettings, SettingsConfigDict
13
+
14
+ class WideAreaConfig(BaseSettings):
15
+ """Configuration for wide-area (high-latency) inference.
16
+
17
+ Controls how the pipeline handles nodes separated by WAN links
18
+ (high latency, limited bandwidth, potential packet loss).
19
+
20
+ .. rubric:: 12-factor overrides
21
+
22
+ Every field can be set via environment variable with the ``WIDE_AREA_``
23
+ prefix. Examples::
24
+
25
+ export WIDE_AREA_ENABLED=true
26
+ export WIDE_AREA_TIMEOUT=60
27
+ export WIDE_AREA_TRANSPORT=quic
28
+
29
+ Attributes:
30
+ enabled: Enable WAN-aware pipeline execution.
31
+ p2p_forwarding: Nodes forward hidden_states directly (bypass coordinator).
32
+ token_accumulation: Buffer N decode steps before sending across WAN.
33
+ accumulation_window: Max tokens to accumulate when token_accumulation=True.
34
+ wan_timeout_seconds: Per-hop gRPC timeout for WAN links (much larger than LAN).
35
+ max_accumulation_retries: Max retries per accumulated batch.
36
+ adaptive_batching: Dynamically adjust accumulation window based on measured RTT.
37
+ latency_sample_interval: Seconds between latency measurements.
38
+ fallback_to_local: Fall back to local-only inference if WAN link fails.
39
+ compression_level: gRPC compression level (0=off, 1=fast, 2=gzip).
40
+ heartbeat_interval_seconds: Seconds between WAN link health checks.
41
+ transport: Transport backend for WAN links. ``"auto"`` selects QUIC when
42
+ available, falling back to gRPC. ``"quic"`` forces QUIC (fails if
43
+ aioquic is not installed). ``"grpc"`` forces gRPC.
44
+ """
45
+
46
+ model_config = SettingsConfigDict(
47
+ env_prefix="WIDE_AREA_",
48
+ extra="ignore",
49
+ frozen=True,
50
+ )
51
+
52
+ enabled: bool = False
53
+ p2p_forwarding: bool = True
54
+ token_accumulation: bool = True
55
+ accumulation_window: int = Field(default=3, ge=1)
56
+ wan_timeout_seconds: float = Field(default=120.0, gt=0)
57
+ max_accumulation_retries: int = Field(default=3, ge=0)
58
+ adaptive_batching: bool = True
59
+ latency_sample_interval: float = Field(default=10.0, gt=0)
60
+ fallback_to_local: bool = True
61
+ compression_level: int = 2
62
+ heartbeat_interval_seconds: float = Field(default=5.0, gt=0)
63
+ transport: Literal["auto", "quic", "grpc"] = "auto"
64
+
65
+ @field_validator("compression_level")
66
+ @classmethod
67
+ def _validate_compression(cls, v: int) -> int:
68
+ if v not in (0, 1, 2):
69
+ raise ValueError(f"compression_level must be 0, 1, or 2, got {v}")
70
+ return v
@@ -0,0 +1,231 @@
1
+ """Cross-cluster request forwarder for federated inference."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import base64
6
+ import json
7
+ import time
8
+ from collections.abc import AsyncGenerator
9
+ from typing import Any
10
+
11
+ import httpx
12
+
13
+ try:
14
+ import ray
15
+ except ImportError:
16
+ ray = None
17
+ from loguru import logger
18
+
19
+ from distllm.dist import node_pb2
20
+ from distllm.security import safe_urlopen
21
+
22
+
23
+ class CrossClusterForwarder:
24
+ def __init__(
25
+ self,
26
+ timeout_s: float = 120.0,
27
+ max_retries: int = 2,
28
+ retry_delay_s: float = 1.0,
29
+ ) -> None:
30
+ self.timeout_s = timeout_s
31
+ self.max_retries = max_retries
32
+ self.retry_delay_s = retry_delay_s
33
+ self._ray_workers: dict[str, list[ray.actor.ActorHandle]] = {}
34
+
35
+ def set_ray_workers(
36
+ self,
37
+ workers: dict[str, list[ray.actor.ActorHandle]],
38
+ ) -> None:
39
+ self._ray_workers.update(workers)
40
+
41
+ def _call_ray_worker(
42
+ self,
43
+ cluster_id: str,
44
+ request: dict[str, Any],
45
+ ) -> dict[str, Any] | None:
46
+ workers = self._ray_workers.get(cluster_id)
47
+ if not workers:
48
+ return None
49
+
50
+ worker = workers[hash(json.dumps(request, sort_keys=True)) % len(workers)]
51
+ try:
52
+ ref = worker.process.remote(request)
53
+ return ray.get(ref, timeout=self.timeout_s)
54
+ except Exception as e:
55
+ logger.warning(f"Ray forward to cluster '{cluster_id}' failed: {e}")
56
+ return None
57
+
58
+ def forward_request(
59
+ self,
60
+ remote_coord_url: str,
61
+ request: dict[str, Any],
62
+ timeout_s: float | None = None,
63
+ cluster_id: str | None = None,
64
+ ) -> dict[str, Any]:
65
+ if cluster_id:
66
+ result = self._call_ray_worker(cluster_id, request)
67
+ if result is not None:
68
+ logger.debug(
69
+ f"Ray cross-cluster forward to '{cluster_id}': "
70
+ f"{len(result.get('choices', []))} choices"
71
+ )
72
+ return result
73
+
74
+ import urllib.request
75
+
76
+ timeout = timeout_s or self.timeout_s
77
+ payload = json.dumps(request).encode()
78
+ url = f"{remote_coord_url.rstrip('/')}/v1/completions"
79
+
80
+ for attempt in range(self.max_retries + 1):
81
+ try:
82
+ req = urllib.request.Request(
83
+ url,
84
+ data=payload,
85
+ headers={
86
+ "Content-Type": "application/json",
87
+ "X-Forwarded-From": "federated",
88
+ },
89
+ method="POST",
90
+ )
91
+ with safe_urlopen(req, timeout=timeout, allow_private_hosts=True) as resp:
92
+ data = json.loads(resp.read())
93
+ logger.debug(
94
+ f"Cross-cluster forward: {url} -> {len(data.get('choices', []))} choices"
95
+ )
96
+ return data
97
+ except Exception as e:
98
+ if attempt < self.max_retries:
99
+ # Exponential backoff with jitter to prevent thundering herd
100
+ import random
101
+ base_delay = self.retry_delay_s * (2 ** attempt)
102
+ jitter = random.uniform(0, base_delay * 0.5)
103
+ delay = base_delay + jitter
104
+ logger.warning(
105
+ f"Cross-cluster forward attempt {attempt + 1} failed: {e}, "
106
+ f"retrying in {delay:.2f}s..."
107
+ )
108
+ time.sleep(delay)
109
+ else:
110
+ logger.error(
111
+ f"Cross-cluster forward failed after {self.max_retries + 1} attempts: {e}"
112
+ )
113
+ raise
114
+
115
+ async def forward_streaming(
116
+ self,
117
+ remote_coord_url: str,
118
+ request: dict[str, Any],
119
+ timeout_s: float | None = None,
120
+ ) -> AsyncGenerator[str, None]:
121
+ timeout = timeout_s or self.timeout_s
122
+ request["stream"] = True
123
+ url = f"{remote_coord_url.rstrip('/')}/v1/chat/completions"
124
+
125
+ try:
126
+ async with httpx.AsyncClient(timeout=httpx.Timeout(timeout)) as client:
127
+ async with client.stream(
128
+ "POST",
129
+ url,
130
+ json=request,
131
+ headers={"X-Forwarded-From": "federated"},
132
+ ) as resp:
133
+ async for line in resp.aiter_lines():
134
+ if line.startswith("data: "):
135
+ data = line[6:].strip()
136
+ if data == "[DONE]":
137
+ break
138
+ yield data
139
+ except Exception as e:
140
+ logger.error(f"Cross-cluster streaming forward failed: {e}")
141
+ yield json.dumps({"error": str(e)})
142
+
143
+ def _kv_to_protobuf(self, kv_data: dict[str, Any]) -> bytes:
144
+ """Serialize KV cache data to protobuf binary for efficient transfer.
145
+
146
+ Uses ``KVCacheProto`` from the gRPC schema for 10-50x faster
147
+ serialization vs JSON for large tensor payloads.
148
+
149
+ Args:
150
+ kv_data: Dict with ``key`` and ``value`` tensor lists.
151
+
152
+ Returns:
153
+ Serialized protobuf bytes (base64-encoded str for JSON transport).
154
+ """
155
+ cache_pb = node_pb2.KVCacheProto()
156
+ from distllm.dist.pipeline.serialization import to_proto_tensor, set_kv_cache_proto
157
+
158
+ layers = kv_data if isinstance(kv_data, list) else kv_data.get("layers", [])
159
+ for layer in layers:
160
+ layer_pb = cache_pb.layers.add()
161
+ if "key" in layer and "value" in layer:
162
+ k = layer["key"]
163
+ v = layer["value"]
164
+ if hasattr(k, "shape"):
165
+ layer_pb.key_states.CopyFrom(to_proto_tensor(k))
166
+ layer_pb.value_states.CopyFrom(to_proto_tensor(v))
167
+ else:
168
+ # Already serialized as list — skip tensor conversion
169
+ pass
170
+ return base64.b64encode(cache_pb.SerializeToString()).decode("ascii")
171
+
172
+ def forward_kv_cache(
173
+ self,
174
+ remote_node_url: str,
175
+ prefix_hash: str,
176
+ kv_data: dict[str, Any],
177
+ ) -> bool:
178
+ import urllib.request
179
+
180
+ kv_binary = self._kv_to_protobuf(kv_data)
181
+ payload = json.dumps({
182
+ "prefix_hash": prefix_hash,
183
+ "kv_data": {"__proto__": "KVCacheProto", "data": kv_binary},
184
+ }).encode()
185
+
186
+ url = f"{remote_node_url.rstrip('/')}/api/v1/cache/warm"
187
+ try:
188
+ req = urllib.request.Request(
189
+ url,
190
+ data=payload,
191
+ headers={"Content-Type": "application/json"},
192
+ method="POST",
193
+ )
194
+ with safe_urlopen(req, timeout=30, allow_private_hosts=True) as resp:
195
+ return resp.status == 200
196
+ except Exception as e:
197
+ logger.debug(f"KV cache forward failed: {e}")
198
+ return False
199
+
200
+ def replicate_kv_batch(
201
+ self,
202
+ entries: list[dict[str, Any]],
203
+ remote_node_urls: list[str],
204
+ ) -> int:
205
+ import urllib.request
206
+
207
+ success = 0
208
+ for entry in entries:
209
+ prefix_hash = entry.get("prefix_hash", "")
210
+ kv_data = entry.get("kv_data", {})
211
+ kv_binary = self._kv_to_protobuf(kv_data)
212
+ payload = json.dumps({
213
+ "prefix_hash": prefix_hash,
214
+ "kv_data": {"__proto__": "KVCacheProto", "data": kv_binary},
215
+ }).encode()
216
+
217
+ for url in remote_node_urls:
218
+ target = f"{url.rstrip('/')}/api/v1/cache/warm"
219
+ try:
220
+ req = urllib.request.Request(
221
+ target,
222
+ data=payload,
223
+ headers={"Content-Type": "application/json"},
224
+ method="POST",
225
+ )
226
+ with safe_urlopen(req, timeout=30, allow_private_hosts=True) as resp:
227
+ if resp.status == 200:
228
+ success += 1
229
+ except Exception as e:
230
+ logger.debug(f"KV batch replicate to {target} failed: {e}")
231
+ return success