distributed-llm 0.4.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- distllm/__init__.py +47 -0
- distllm/api/__init__.py +11 -0
- distllm/api/api_settings.py +126 -0
- distllm/api/api_state.py +85 -0
- distllm/api/api_versioning.py +984 -0
- distllm/api/app_state.py +29 -0
- distllm/api/auth/__init__.py +181 -0
- distllm/api/auth/models.py +37 -0
- distllm/api/auth/oauth2.py +123 -0
- distllm/api/auth/oidc.py +317 -0
- distllm/api/auth/saml.py +107 -0
- distllm/api/auth/store.py +39 -0
- distllm/api/auth_deps.py +68 -0
- distllm/api/authz/__init__.py +30 -0
- distllm/api/authz/opa.py +297 -0
- distllm/api/background_tasks.py +295 -0
- distllm/api/body_cache_middleware.py +117 -0
- distllm/api/circuit_breaker_middleware.py +175 -0
- distllm/api/cost_middleware.py +186 -0
- distllm/api/csrf_middleware.py +196 -0
- distllm/api/dedup.py +180 -0
- distllm/api/errors.py +256 -0
- distllm/api/grpc_bridge.py +296 -0
- distllm/api/ip_utils.py +58 -0
- distllm/api/middleware.py +706 -0
- distllm/api/observability_middleware.py +169 -0
- distllm/api/persistent_store.py +295 -0
- distllm/api/prompt_injection.py +418 -0
- distllm/api/quota_middleware.py +186 -0
- distllm/api/rate_limit_middleware.py +13 -0
- distllm/api/rate_limiter.py +250 -0
- distllm/api/rate_limiter_unified.py +975 -0
- distllm/api/redis_rate_limiter.py +164 -0
- distllm/api/routes/__init__.py +42 -0
- distllm/api/routes/admin.py +599 -0
- distllm/api/routes/api_keys.py +115 -0
- distllm/api/routes/assistants.py +443 -0
- distllm/api/routes/batch.py +1094 -0
- distllm/api/routes/chat.py +875 -0
- distllm/api/routes/completion.py +141 -0
- distllm/api/routes/debug.py +185 -0
- distllm/api/routes/defrag.py +66 -0
- distllm/api/routes/embeddings.py +399 -0
- distllm/api/routes/eval.py +254 -0
- distllm/api/routes/exchange.py +367 -0
- distllm/api/routes/experiments.py +293 -0
- distllm/api/routes/federated.py +155 -0
- distllm/api/routes/files.py +243 -0
- distllm/api/routes/fine_tuning.py +381 -0
- distllm/api/routes/gossip.py +72 -0
- distllm/api/routes/health.py +395 -0
- distllm/api/routes/leaderboard.py +377 -0
- distllm/api/routes/marketplace.py +320 -0
- distllm/api/routes/metrics_history.py +415 -0
- distllm/api/routes/model_registry.py +255 -0
- distllm/api/routes/moderation.py +420 -0
- distllm/api/routes/plugins.py +184 -0
- distllm/api/routes/prompts.py +347 -0
- distllm/api/routes/router_admin.py +187 -0
- distllm/api/routes/scheduler.py +168 -0
- distllm/api/routes/tokenize.py +77 -0
- distllm/api/routes/tools.py +89 -0
- distllm/api/routes/webhooks.py +336 -0
- distllm/api/routes/webrtc.py +194 -0
- distllm/api/semantic_cache.py +337 -0
- distllm/api/server.py +2038 -0
- distllm/api/server_config.py +140 -0
- distllm/api/server_coordinator.py +125 -0
- distllm/api/server_errors.py +79 -0
- distllm/api/server_lifespan.py +184 -0
- distllm/api/server_routes_api.py +456 -0
- distllm/api/server_routes_dashboard.py +246 -0
- distllm/api/server_state.py +29 -0
- distllm/api/services/__init__.py +2 -0
- distllm/api/services/chat_service.py +677 -0
- distllm/api/services/completion_service.py +227 -0
- distllm/api/services/embedding_service.py +451 -0
- distllm/api/services/eval_service.py +198 -0
- distllm/api/sso_auth.py +644 -0
- distllm/api/sso_middleware.py +808 -0
- distllm/api/streaming.py +479 -0
- distllm/api/token_estimator.py +86 -0
- distllm/api/tool_calling.py +240 -0
- distllm/api/tracing_middleware.py +579 -0
- distllm/api/validation.py +58 -0
- distllm/api/waf.py +344 -0
- distllm/api/webhooks/__init__.py +17 -0
- distllm/api/webhooks/delivery.py +452 -0
- distllm/api/ws_chat.py +495 -0
- distllm/backends/__init__.py +121 -0
- distllm/backends/config.py +85 -0
- distllm/backends/exllama_backend.py +229 -0
- distllm/backends/llamacpp_backend.py +266 -0
- distllm/backends/mlx_backend.py +199 -0
- distllm/backends/nim_backend.py +838 -0
- distllm/backends/ollama_backend.py +48 -0
- distllm/backends/onnx_backend.py +239 -0
- distllm/backends/paged_attention.py +547 -0
- distllm/backends/paged_attention_quantized.py +319 -0
- distllm/backends/protocol.py +203 -0
- distllm/backends/pytorch_backend.py +157 -0
- distllm/backends/registry.py +428 -0
- distllm/backends/tensorrt_backend.py +333 -0
- distllm/backends/tgi_backend.py +120 -0
- distllm/backends/triton_backend.py +875 -0
- distllm/backends/vllm_backend.py +489 -0
- distllm/backends/webgpu_backend.py +255 -0
- distllm/benchmarks/__init__.py +10 -0
- distllm/benchmarks/evaluation_harness.py +69 -0
- distllm/benchmarks/scaling.py +298 -0
- distllm/cache/__init__.py +19 -0
- distllm/cache/crdt.py +507 -0
- distllm/cache/cross_cluster_prefix_index.py +709 -0
- distllm/cli/__init__.py +17 -0
- distllm/cli/adapters.py +93 -0
- distllm/cli/autopsy.py +287 -0
- distllm/cli/backup.py +118 -0
- distllm/cli/benchmark.py +241 -0
- distllm/cli/cert.py +86 -0
- distllm/cli/chat.py +79 -0
- distllm/cli/chat_stream.py +494 -0
- distllm/cli/client.py +233 -0
- distllm/cli/cluster.py +405 -0
- distllm/cli/completion.py +79 -0
- distllm/cli/compress.py +98 -0
- distllm/cli/config_commands.py +71 -0
- distllm/cli/cost_avoid.py +168 -0
- distllm/cli/daas_commands.py +123 -0
- distllm/cli/defrag_commands.py +128 -0
- distllm/cli/deploy.py +252 -0
- distllm/cli/doctor.py +688 -0
- distllm/cli/draft_commands.py +82 -0
- distllm/cli/error_handler.py +192 -0
- distllm/cli/eval.py +168 -0
- distllm/cli/federate_commands.py +125 -0
- distllm/cli/install.py +198 -0
- distllm/cli/logs.py +85 -0
- distllm/cli/main.py +1795 -0
- distllm/cli/models.py +74 -0
- distllm/cli/notify.py +91 -0
- distllm/cli/observe.py +314 -0
- distllm/cli/onboard.py +671 -0
- distllm/cli/output.py +437 -0
- distllm/cli/profile.py +152 -0
- distllm/cli/profile_presets.py +129 -0
- distllm/cli/prompts.py +139 -0
- distllm/cli/quota.py +227 -0
- distllm/cli/router.py +131 -0
- distllm/cli/run.py +59 -0
- distllm/cli/setup.py +104 -0
- distllm/cli/setup_wizard.py +191 -0
- distllm/cli/status.py +65 -0
- distllm/cli/system_commands.py +255 -0
- distllm/cli/tune.py +488 -0
- distllm/cli/tutorial.py +128 -0
- distllm/cli/verify.py +37 -0
- distllm/cli/webhook.py +85 -0
- distllm/client/__init__.py +38 -0
- distllm/client/client.py +503 -0
- distllm/cloud/__init__.py +53 -0
- distllm/cloud/aws.py +199 -0
- distllm/cloud/azure.py +179 -0
- distllm/cloud/common.py +217 -0
- distllm/cloud/gcp.py +238 -0
- distllm/cloud/spot_orchestrator.py +1642 -0
- distllm/cloud/worker_agent.py +344 -0
- distllm/compliance/__init__.py +10 -0
- distllm/compliance/evidence_pack.py +699 -0
- distllm/config/__init__.py +81 -0
- distllm/config/_application.py +41 -0
- distllm/config/_backends.py +80 -0
- distllm/config/_cache.py +121 -0
- distllm/config/_deployment.py +95 -0
- distllm/config/_generation.py +36 -0
- distllm/config/_hardware.py +42 -0
- distllm/config/_model.py +270 -0
- distllm/config/_network.py +235 -0
- distllm/config/_observability.py +42 -0
- distllm/config/_parallelism.py +172 -0
- distllm/config/_performance.py +47 -0
- distllm/config/loader.py +168 -0
- distllm/config/model_heuristics.py +169 -0
- distllm/config/profiles.py +168 -0
- distllm/config/reference.py +104 -0
- distllm/config/resolver.py +240 -0
- distllm/config/settings.py +463 -0
- distllm/constants.py +117 -0
- distllm/core/__init__.py +111 -0
- distllm/core/_advanced_scheduling_original.py +1892 -0
- distllm/core/ab_test_coordinator.py +178 -0
- distllm/core/activation_profiler.py +172 -0
- distllm/core/adaptive_batching.py +234 -0
- distllm/core/adaptive_cache_compressor.py +168 -0
- distllm/core/adaptive_compression.py +427 -0
- distllm/core/adaptive_compression_hierarchy.py +402 -0
- distllm/core/advanced_scheduling/__init__.py +39 -0
- distllm/core/advanced_scheduling/cost_aware.py +55 -0
- distllm/core/advanced_scheduling/disaggregated.py +352 -0
- distllm/core/advanced_scheduling/energy.py +58 -0
- distllm/core/advanced_scheduling/federated.py +59 -0
- distllm/core/advanced_scheduling/heterogeneous.py +84 -0
- distllm/core/advanced_scheduling/multi_objective.py +372 -0
- distllm/core/advanced_scheduling/policy.py +95 -0
- distllm/core/advanced_scheduling/predictive.py +47 -0
- distllm/core/advanced_scheduling/preemption.py +43 -0
- distllm/core/advanced_scheduling/tiered_store.py +604 -0
- distllm/core/advanced_scheduling/token_bank.py +62 -0
- distllm/core/advanced_scheduling/wan.py +85 -0
- distllm/core/aegis_compliance.py +890 -0
- distllm/core/aether_federated.py +1326 -0
- distllm/core/agentic_router.py +749 -0
- distllm/core/api_key_store.py +478 -0
- distllm/core/arbitrage_engine.py +613 -0
- distllm/core/aria_autoscaler.py +1339 -0
- distllm/core/async_connection_pool.py +187 -0
- distllm/core/async_pipelined_speculative.py +543 -0
- distllm/core/atlas_mesh.py +1437 -0
- distllm/core/auto_discovery.py +351 -0
- distllm/core/auto_partitioner.py +396 -0
- distllm/core/autonomous_healer.py +466 -0
- distllm/core/autoq.py +1545 -0
- distllm/core/backup_manager.py +344 -0
- distllm/core/bargaining_engine.py +450 -0
- distllm/core/batch_builder.py +480 -0
- distllm/core/batch_scheduler.py +1342 -0
- distllm/core/block_affinity_tracker.py +162 -0
- distllm/core/block_eviction_policy.py +363 -0
- distllm/core/cache_aware_router.py +116 -0
- distllm/core/cache_bench.py +239 -0
- distllm/core/cache_coherence.py +139 -0
- distllm/core/cache_doctor.py +220 -0
- distllm/core/cache_eviction.py +181 -0
- distllm/core/cache_index.py +91 -0
- distllm/core/cache_manager.py +703 -0
- distllm/core/cache_migration.py +134 -0
- distllm/core/cache_persistence.py +200 -0
- distllm/core/cache_query_log.py +195 -0
- distllm/core/cache_snapshot.py +178 -0
- distllm/core/cache_template_warmer.py +112 -0
- distllm/core/cache_warming.py +188 -0
- distllm/core/calibration.py +223 -0
- distllm/core/carbon_migration.py +338 -0
- distllm/core/cert_rotation.py +402 -0
- distllm/core/certificate_manager.py +414 -0
- distllm/core/cluster_manager.py +212 -0
- distllm/core/cluster_predictive_prefetcher.py +239 -0
- distllm/core/cluster_registry.py +227 -0
- distllm/core/cluster_state_store.py +442 -0
- distllm/core/compliance_evidence.py +503 -0
- distllm/core/compressed_speculative.py +399 -0
- distllm/core/connection_pool.py +387 -0
- distllm/core/constrained_decoder.py +678 -0
- distllm/core/coordinator.py +1594 -0
- distllm/core/coordinator_cli.py +130 -0
- distllm/core/coordinator_config.py +203 -0
- distllm/core/coordinator_config_wiring.py +316 -0
- distllm/core/coordinator_election.py +246 -0
- distllm/core/coordinator_failover.py +197 -0
- distllm/core/coordinator_health.py +135 -0
- distllm/core/coordinator_lifecycle.py +272 -0
- distllm/core/coordinator_metrics.py +155 -0
- distllm/core/coordinator_multi_model.py +41 -0
- distllm/core/coordinator_request.py +246 -0
- distllm/core/coordinator_state.py +215 -0
- distllm/core/coordinator_subsystem.py +614 -0
- distllm/core/correctness_cert.py +133 -0
- distllm/core/correctness_harness.py +421 -0
- distllm/core/cortex_multimodel.py +939 -0
- distllm/core/cost_comparison.py +250 -0
- distllm/core/cost_dashboard.py +363 -0
- distllm/core/cost_optimizer.py +317 -0
- distllm/core/cost_tracker.py +569 -0
- distllm/core/cross_cloud_router.py +924 -0
- distllm/core/cross_model_prefix_sharing.py +237 -0
- distllm/core/cuda_graph.py +165 -0
- distllm/core/debug.py +84 -0
- distllm/core/device_registry.py +289 -0
- distllm/core/di.py +153 -0
- distllm/core/differential_privacy.py +183 -0
- distllm/core/distributed_distillation.py +402 -0
- distllm/core/distributed_speculative.py +1563 -0
- distllm/core/dp_inference/__init__.py +657 -0
- distllm/core/dp_inference/accounting.py +144 -0
- distllm/core/dp_inference/config.py +139 -0
- distllm/core/dp_inference/mechanisms.py +137 -0
- distllm/core/dp_inference.py +1016 -0
- distllm/core/draft_model_router.py +435 -0
- distllm/core/draft_quality_scorer.py +157 -0
- distllm/core/draft_tree.py +232 -0
- distllm/core/dynamic_memory_budget.py +179 -0
- distllm/core/dynamic_sharder.py +364 -0
- distllm/core/dynamic_speculation.py +131 -0
- distllm/core/evaluation/__init__.py +57 -0
- distllm/core/evaluation/constants.py +86 -0
- distllm/core/evaluation/db.py +185 -0
- distllm/core/evaluation/formatters.py +82 -0
- distllm/core/evaluation/loaders.py +320 -0
- distllm/core/evaluation/models.py +66 -0
- distllm/core/evaluation/report.py +79 -0
- distllm/core/evaluation/runner.py +518 -0
- distllm/core/evaluation/scorers.py +278 -0
- distllm/core/evaluation/worker.py +123 -0
- distllm/core/evaluation_harness.py +1530 -0
- distllm/core/event_bus.py +516 -0
- distllm/core/faas_7b.py +201 -0
- distllm/core/feature_flags.py +245 -0
- distllm/core/federated_finetuner.py +377 -0
- distllm/core/federated_incentives.py +267 -0
- distllm/core/framework_adapters.py +390 -0
- distllm/core/gaia_cache.py +1330 -0
- distllm/core/gossip_cache_bridge.py +202 -0
- distllm/core/gpu_power_manager.py +173 -0
- distllm/core/gpu_profiler.py +59 -0
- distllm/core/gpu_resource_manager.py +338 -0
- distllm/core/graceful_degradation.py +264 -0
- distllm/core/grammar_constrained.py +549 -0
- distllm/core/grammar_constrained_draft.py +91 -0
- distllm/core/grammar_decoder.py +285 -0
- distllm/core/ha_coordinator.py +452 -0
- distllm/core/hardware_plugin_registry.py +126 -0
- distllm/core/hashing.py +29 -0
- distllm/core/health_manager.py +320 -0
- distllm/core/heterogeneous_scheduler.py +663 -0
- distllm/core/hibernation_manager.py +250 -0
- distllm/core/hierarchical_digest.py +168 -0
- distllm/core/hybrid_cache.py +253 -0
- distllm/core/hydra_diffusion.py +177 -0
- distllm/core/inference_engine.py +1072 -0
- distllm/core/intelligent_autoscaler.py +219 -0
- distllm/core/kraken_chaos.py +1407 -0
- distllm/core/kv_backup.py +217 -0
- distllm/core/kv_cache.py +1271 -0
- distllm/core/kv_cache_adaptive_quantizer.py +168 -0
- distllm/core/kv_cache_compressor.py +290 -0
- distllm/core/kv_cache_manager.py +142 -0
- distllm/core/kv_cache_marketplace.py +319 -0
- distllm/core/kv_cache_metrics.py +205 -0
- distllm/core/kv_cache_migration.py +137 -0
- distllm/core/kv_cache_paged.py +78 -0
- distllm/core/kv_cache_replication.py +107 -0
- distllm/core/kv_cache_serialization.py +182 -0
- distllm/core/latency_tracker.py +17 -0
- distllm/core/leaky_bucket_limiter.py +194 -0
- distllm/core/learning_router.py +652 -0
- distllm/core/llamacpp_backend.py +23 -0
- distllm/core/load_balancer.py +286 -0
- distllm/core/media_pipeline.py +387 -0
- distllm/core/memory_defragmenter.py +666 -0
- distllm/core/metering.py +449 -0
- distllm/core/metrics_collector.py +39 -0
- distllm/core/micro_batch_scheduler.py +170 -0
- distllm/core/model_hub_enhanced.py +382 -0
- distllm/core/model_registry.py +9 -0
- distllm/core/model_router.py +930 -0
- distllm/core/model_selector.py +245 -0
- distllm/core/model_sizing.py +334 -0
- distllm/core/model_version_manager.py +414 -0
- distllm/core/moe_orchestrator.py +205 -0
- distllm/core/money.py +53 -0
- distllm/core/monitor.py +141 -0
- distllm/core/mtp_head.py +444 -0
- distllm/core/multi_draft_verifier.py +346 -0
- distllm/core/multi_model_serving.py +624 -0
- distllm/core/multimodal_engine.py +194 -0
- distllm/core/neural_partition_optimizer.py +1398 -0
- distllm/core/node_recovery.py +27 -0
- distllm/core/notification_manager.py +317 -0
- distllm/core/param_update_channel.py +104 -0
- distllm/core/pareto_optimizer.py +227 -0
- distllm/core/performance_alerts.py +198 -0
- distllm/core/performance_baseline.py +252 -0
- distllm/core/persistence.py +947 -0
- distllm/core/persistent_store.py +485 -0
- distllm/core/pipeline_composer.py +211 -0
- distllm/core/pipeline_executor.py +500 -0
- distllm/core/pipeline_orchestrator.py +17 -0
- distllm/core/pipeline_overlap.py +177 -0
- distllm/core/placement.py +442 -0
- distllm/core/plugin_marketplace.py +483 -0
- distllm/core/plugin_sandbox.py +751 -0
- distllm/core/plugin_system.py +601 -0
- distllm/core/plugins/__init__.py +26 -0
- distllm/core/plugins/sandbox.py +38 -0
- distllm/core/predictive_cache.py +29 -0
- distllm/core/predictive_cache_warming.py +323 -0
- distllm/core/predictive_failure.py +163 -0
- distllm/core/preemptible_scheduler.py +343 -0
- distllm/core/preference_learning.py +606 -0
- distllm/core/prefix_cache.py +17 -0
- distllm/core/pricing_providers.py +576 -0
- distllm/core/priority_heap.py +62 -0
- distllm/core/privacy_budget.py +177 -0
- distllm/core/prompt_caching_service.py +228 -0
- distllm/core/prompt_library.py +838 -0
- distllm/core/protocols.py +213 -0
- distllm/core/provider_health.py +234 -0
- distllm/core/provisioning.py +329 -0
- distllm/core/pulse_performance_model.py +1558 -0
- distllm/core/quantization_selector.py +210 -0
- distllm/core/radix_tree_cache.py +306 -0
- distllm/core/rebalancer.py +25 -0
- distllm/core/redis_prompt_cache.py +311 -0
- distllm/core/replication_controller.py +128 -0
- distllm/core/request_auditor.py +215 -0
- distllm/core/request_fingerprinting.py +231 -0
- distllm/core/request_latency.py +211 -0
- distllm/core/request_pipeline.py +810 -0
- distllm/core/request_replay.py +253 -0
- distllm/core/request_tracker.py +184 -0
- distllm/core/resource_manager.py +657 -0
- distllm/core/route_audit.py +175 -0
- distllm/core/routing_extensions.py +604 -0
- distllm/core/schedule_simulator.py +289 -0
- distllm/core/schedule_viz.py +233 -0
- distllm/core/scheduler/__init__.py +34 -0
- distllm/core/scheduler/budget.py +25 -0
- distllm/core/scheduler/budget_computer.py +322 -0
- distllm/core/scheduler/chunked_prefill.py +52 -0
- distllm/core/scheduler/kv_cache_manager.py +313 -0
- distllm/core/scheduler/preemption_manager.py +251 -0
- distllm/core/scheduler/pressure.py +48 -0
- distllm/core/scheduler/sequence.py +179 -0
- distllm/core/secret_manager.py +315 -0
- distllm/core/semantic_cache.py +369 -0
- distllm/core/sentinel_qos.py +833 -0
- distllm/core/shadow_eval_runner.py +194 -0
- distllm/core/shared_layer_pool.py +279 -0
- distllm/core/smart_model_router.py +309 -0
- distllm/core/spec_equivalence.py +140 -0
- distllm/core/spec_verify.py +225 -0
- distllm/core/speculative_adaptor.py +108 -0
- distllm/core/speculative_dashboard.py +134 -0
- distllm/core/speculative_decoder.py +1286 -0
- distllm/core/speculative_profiler.py +132 -0
- distllm/core/split_brain.py +236 -0
- distllm/core/spot_failover.py +292 -0
- distllm/core/spot_forecasting.py +463 -0
- distllm/core/starvation_monitor.py +70 -0
- distllm/core/state_replication.py +261 -0
- distllm/core/stats_collector.py +64 -0
- distllm/core/step_processor.py +115 -0
- distllm/core/straggler_alerts.py +174 -0
- distllm/core/straggler_aware_scheduler.py +255 -0
- distllm/core/straggler_detector.py +29 -0
- distllm/core/streaming_cost.py +286 -0
- distllm/core/streaming_generator.py +228 -0
- distllm/core/structured_output/__init__.py +348 -0
- distllm/core/structured_output/config.py +38 -0
- distllm/core/structured_output/engine.py +367 -0
- distllm/core/structured_output/streaming.py +183 -0
- distllm/core/structured_output/validator.py +199 -0
- distllm/core/subsystem_registry.py +171 -0
- distllm/core/synapse_debugger.py +226 -0
- distllm/core/synth_data_generator.py +122 -0
- distllm/core/telemetry.py +240 -0
- distllm/core/tenant_billing.py +446 -0
- distllm/core/tenant_cost_attribution.py +314 -0
- distllm/core/token_generator.py +368 -0
- distllm/core/token_streaming_buffer.py +147 -0
- distllm/core/topology_aware_lb.py +324 -0
- distllm/core/topology_aware_tiering.py +344 -0
- distllm/core/tree_speculative_decoder.py +405 -0
- distllm/core/types.py +17 -0
- distllm/core/unified_router.py +439 -0
- distllm/core/unified_sla_router.py +242 -0
- distllm/core/usage_meter.py +915 -0
- distllm/core/vectorstore/__init__.py +30 -0
- distllm/core/vectorstore/base.py +106 -0
- distllm/core/vectorstore/chroma_store.py +99 -0
- distllm/core/vectorstore/factory.py +97 -0
- distllm/core/vectorstore/pgvector_store.py +123 -0
- distllm/core/vectorstore/providers/__init__.py +7 -0
- distllm/core/vectorstore/providers/milvus.py +219 -0
- distllm/core/vectorstore/providers/pinecone.py +142 -0
- distllm/core/vectorstore/providers/qdrant.py +176 -0
- distllm/core/vectorstore/providers/weaviate.py +185 -0
- distllm/core/vectorstore/qdrant_store.py +148 -0
- distllm/core/vectorstore/rag.py +151 -0
- distllm/core/vllm_backend.py +21 -0
- distllm/core/vllm_node.py +174 -0
- distllm/core/voyager_multimodal.py +1354 -0
- distllm/core/webgpu_manager.py +382 -0
- distllm/core/webhook_formatters.py +243 -0
- distllm/core/webhook_manager.py +433 -0
- distllm/core/wisp_wasm.py +934 -0
- distllm/core/workload_classifier.py +137 -0
- distllm/dashboard/__init__.py +17 -0
- distllm/dashboard/verification_history.py +192 -0
- distllm/dashboard/ws_handler.py +586 -0
- distllm/dist/__init__.py +85 -0
- distllm/dist/__init__.pyi +132 -0
- distllm/dist/_pipeline_original.py +2916 -0
- distllm/dist/adaptive_speculator.py +384 -0
- distllm/dist/admin_cli.py +293 -0
- distllm/dist/api_docs.py +594 -0
- distllm/dist/async_pipeline.py +578 -0
- distllm/dist/attention.py +1699 -0
- distllm/dist/attention_block_pool.py +634 -0
- distllm/dist/attention_distributed.py +138 -0
- distllm/dist/autoscaler.py +196 -0
- distllm/dist/backends/__init__.py +29 -0
- distllm/dist/backends/backend_profiles.py +357 -0
- distllm/dist/backends/graceful_degradation.py +409 -0
- distllm/dist/backends/health_monitor.py +518 -0
- distllm/dist/backends/llamacpp.py +125 -0
- distllm/dist/backends/ray.py +89 -0
- distllm/dist/backends/vllm.py +177 -0
- distllm/dist/block_pool.py +662 -0
- distllm/dist/block_transfer_service.py +267 -0
- distllm/dist/byzantine.py +2193 -0
- distllm/dist/cache.py +305 -0
- distllm/dist/cache_digest.py +405 -0
- distllm/dist/chaos/__init__.py +52 -0
- distllm/dist/chaos/chaos_orchestrator.py +229 -0
- distllm/dist/chaos/fault_injector.py +1273 -0
- distllm/dist/chunked_prefill.py +136 -0
- distllm/dist/cloud_selector.py +257 -0
- distllm/dist/config.py +70 -0
- distllm/dist/cross_cluster.py +231 -0
- distllm/dist/daas/__init__.py +46 -0
- distllm/dist/daas/load_balancer.py +275 -0
- distllm/dist/daas/marketplace_integration.py +374 -0
- distllm/dist/daas/resource_isolation.py +280 -0
- distllm/dist/daas/tenant_dispatcher.py +252 -0
- distllm/dist/daas/usage_meter.py +360 -0
- distllm/dist/daas_server.py +514 -0
- distllm/dist/disagg/__init__.py +162 -0
- distllm/dist/disagg/pool.py +214 -0
- distllm/dist/disagg/transfer.py +77 -0
- distllm/dist/disagg/transfer_compression.py +793 -0
- distllm/dist/disagg/transfer_engine.py +998 -0
- distllm/dist/discovery.py +166 -0
- distllm/dist/draft_bank.py +344 -0
- distllm/dist/draft_migration.py +382 -0
- distllm/dist/ebpf_transport.py +364 -0
- distllm/dist/edge_cloud.py +544 -0
- distllm/dist/edge_federation.py +426 -0
- distllm/dist/execution_planner.py +148 -0
- distllm/dist/federated_merge.py +560 -0
- distllm/dist/federation.py +1114 -0
- distllm/dist/flash_attention.py +237 -0
- distllm/dist/fsdp.py +295 -0
- distllm/dist/geo.py +361 -0
- distllm/dist/ice_transport.py +2968 -0
- distllm/dist/kv_migration.py +237 -0
- distllm/dist/latency.py +60 -0
- distllm/dist/lifecycle.py +467 -0
- distllm/dist/marketplace.py +891 -0
- distllm/dist/merkle.py +178 -0
- distllm/dist/metrics.py +456 -0
- distllm/dist/model_store.py +107 -0
- distllm/dist/multi_tenant.py +339 -0
- distllm/dist/multimodal.py +392 -0
- distllm/dist/nat.py +934 -0
- distllm/dist/nccl.py +733 -0
- distllm/dist/network.py +126 -0
- distllm/dist/node_client.py +568 -0
- distllm/dist/node_pb2.py +66 -0
- distllm/dist/node_pb2.pyi +236 -0
- distllm/dist/node_pb2_grpc.py +312 -0
- distllm/dist/node_pb2_grpc.pyi +41 -0
- distllm/dist/node_registrar.py +247 -0
- distllm/dist/node_service.py +572 -0
- distllm/dist/observability.py +258 -0
- distllm/dist/otel.py +158 -0
- distllm/dist/p2p/__init__.py +22 -0
- distllm/dist/p2p/discovery.py +132 -0
- distllm/dist/p2p/gossip.py +1339 -0
- distllm/dist/p2p/kademlia_dht.py +1058 -0
- distllm/dist/p2p/load_balancer.py +131 -0
- distllm/dist/p2p/quic_transport.py +700 -0
- distllm/dist/p2p/router.py +607 -0
- distllm/dist/p2p/transport.py +399 -0
- distllm/dist/p2p_model_distributor.py +160 -0
- distllm/dist/paged_attention_kernel.py +281 -0
- distllm/dist/parallel.py +1061 -0
- distllm/dist/parallel_planner.py +586 -0
- distllm/dist/parallel_topology.py +231 -0
- distllm/dist/partition/__init__.py +146 -0
- distllm/dist/partition/adaptive.py +532 -0
- distllm/dist/partition/benchmark_suite.py +479 -0
- distllm/dist/partition/cli.py +189 -0
- distllm/dist/partition/cloud_arbitrage.py +523 -0
- distllm/dist/partition/config.py +97 -0
- distllm/dist/partition/cost_model.py +553 -0
- distllm/dist/partition/learned_cost.py +653 -0
- distllm/dist/partition/mixed_precision_tuner.py +310 -0
- distllm/dist/partition/network_cost_model.py +664 -0
- distllm/dist/partition/optimizer.py +650 -0
- distllm/dist/partition/pareto_optimizer.py +466 -0
- distllm/dist/partition/partitioner.py +363 -0
- distllm/dist/partition/persistence.py +392 -0
- distllm/dist/partition/plugins.py +270 -0
- distllm/dist/partition/profiles.py +659 -0
- distllm/dist/partition/quant_bench.py +373 -0
- distllm/dist/partition/quant_calibrate.py +229 -0
- distllm/dist/partition/quant_coordinator.py +260 -0
- distllm/dist/partition/quant_cost.py +293 -0
- distllm/dist/partition/quant_partition.py +432 -0
- distllm/dist/partition/quant_report.py +301 -0
- distllm/dist/partition/quantization_metrics.py +355 -0
- distllm/dist/partition/quantization_search.py +290 -0
- distllm/dist/partition/quantization_tuner.py +1383 -0
- distllm/dist/partition/topology.py +218 -0
- distllm/dist/partition/validator.py +405 -0
- distllm/dist/partition/visualizer.py +275 -0
- distllm/dist/pipeline/__init__.py +71 -0
- distllm/dist/pipeline/bandwidth_controller.py +780 -0
- distllm/dist/pipeline/compression_negotiation.py +771 -0
- distllm/dist/pipeline/context.py +80 -0
- distllm/dist/pipeline/continuous_batching.py +435 -0
- distllm/dist/pipeline/disagg_orchestrator.py +284 -0
- distllm/dist/pipeline/orchestrator.py +727 -0
- distllm/dist/pipeline/pipeline_reconfig.py +1395 -0
- distllm/dist/pipeline/profiler.py +84 -0
- distllm/dist/pipeline/serialization.py +176 -0
- distllm/dist/pipeline/simulator.py +172 -0
- distllm/dist/pipeline/strategy.py +193 -0
- distllm/dist/pipeline/token_accumulator.py +211 -0
- distllm/dist/pipeline/transport.py +230 -0
- distllm/dist/pipeline/wan_disagg_orchestrator.py +564 -0
- distllm/dist/power_cap.py +358 -0
- distllm/dist/predictive_cache.py +399 -0
- distllm/dist/preemption.py +198 -0
- distllm/dist/prefix_cache.py +461 -0
- distllm/dist/prefix_clustering.py +305 -0
- distllm/dist/privacy/__init__.py +411 -0
- distllm/dist/privacy/privacy_accountant.py +344 -0
- distllm/dist/privacy.py +240 -0
- distllm/dist/provisioning.py +338 -0
- distllm/dist/quality.py +118 -0
- distllm/dist/quic_transport.py +335 -0
- distllm/dist/quota_enforcer.py +265 -0
- distllm/dist/rebalancer.py +234 -0
- distllm/dist/recovery.py +827 -0
- distllm/dist/recovery_drill.py +281 -0
- distllm/dist/redis_cache.py +265 -0
- distllm/dist/redundant.py +308 -0
- distllm/dist/redundant_executor.py +1536 -0
- distllm/dist/reputation.py +231 -0
- distllm/dist/router.py +382 -0
- distllm/dist/routing/__init__.py +19 -0
- distllm/dist/routing/arbitrage_engine.py +575 -0
- distllm/dist/routing/composite.py +179 -0
- distllm/dist/routing/consistent_hash.py +144 -0
- distllm/dist/routing/latency_aware.py +116 -0
- distllm/dist/routing/load_aware.py +114 -0
- distllm/dist/scheduling/__init__.py +38 -0
- distllm/dist/scheduling/batcher.py +95 -0
- distllm/dist/scheduling/classifier.py +153 -0
- distllm/dist/scheduling/deadline_scheduler.py +958 -0
- distllm/dist/scheduling/iteration.py +404 -0
- distllm/dist/scheduling/profiler.py +57 -0
- distllm/dist/simulation/__init__.py +54 -0
- distllm/dist/simulation/chaos_simulator.py +901 -0
- distllm/dist/simulation/cluster_simulator.py +667 -0
- distllm/dist/simulation/cost_aware_provisioning.py +370 -0
- distllm/dist/simulation/digital_twin.py +617 -0
- distllm/dist/simulation/topology_optimizer.py +747 -0
- distllm/dist/simulation/what_if.py +492 -0
- distllm/dist/speculative/__init__.py +47 -0
- distllm/dist/speculative/adaptive_spec.py +291 -0
- distllm/dist/speculative/draft_cache.py +246 -0
- distllm/dist/speculative/draft_orchestrator.py +634 -0
- distllm/dist/speculative/draft_registry.py +439 -0
- distllm/dist/speculative/multi_draft.py +304 -0
- distllm/dist/speculative/online_sisd.py +523 -0
- distllm/dist/straggler.py +819 -0
- distllm/dist/streaming_kv_transfer.py +174 -0
- distllm/dist/structured_output/__init__.py +7 -0
- distllm/dist/structured_output/engine.py +144 -0
- distllm/dist/tensor_pool.py +120 -0
- distllm/dist/tgi_compat.py +254 -0
- distllm/dist/topology.py +141 -0
- distllm/dist/topology_consensus.py +1961 -0
- distllm/dist/topology_dynamic.py +270 -0
- distllm/dist/tp_inprocess.py +214 -0
- distllm/dist/tp_launcher.py +310 -0
- distllm/dist/tracing.py +325 -0
- distllm/dist/transport.py +493 -0
- distllm/dist/wan_speculative.py +295 -0
- distllm/dist/wandb_integration.py +252 -0
- distllm/dist/webrtc.py +846 -0
- distllm/dist/wide_area.py +515 -0
- distllm/dist/worker.py +635 -0
- distllm/dist/zero_copy.py +307 -0
- distllm/errors/__init__.py +105 -0
- distllm/errors/policies.py +106 -0
- distllm/errors/retry.py +168 -0
- distllm/errors/types.py +721 -0
- distllm/health/__init__.py +6 -0
- distllm/health/failover.py +81 -0
- distllm/health/prober.py +31 -0
- distllm/health/service.py +173 -0
- distllm/health/state.py +76 -0
- distllm/integrations/__init__.py +72 -0
- distllm/integrations/_common/__init__.py +7 -0
- distllm/integrations/_common/base_tool_provider.py +189 -0
- distllm/integrations/_common/cost_tracker.py +144 -0
- distllm/integrations/_common/model_router.py +148 -0
- distllm/integrations/ci/__init__.py +24 -0
- distllm/integrations/ci/_common.py +92 -0
- distllm/integrations/ci/gitlab.py +345 -0
- distllm/integrations/ci/jenkins.py +273 -0
- distllm/integrations/mlflow_tracking.py +372 -0
- distllm/integrations/spark_connector.py +909 -0
- distllm/integrations/wandb/__init__.py +81 -0
- distllm/models/__init__.py +40 -0
- distllm/models/_trust_remote.py +48 -0
- distllm/models/adapter.py +768 -0
- distllm/models/adapter_router.py +198 -0
- distllm/models/cache.py +172 -0
- distllm/models/model_hub.py +650 -0
- distllm/models/partition_planner.py +384 -0
- distllm/models/partitioner.py +784 -0
- distllm/models/rope_scaling.py +119 -0
- distllm/models/safetensors_index.py +179 -0
- distllm/observability/__init__.py +107 -0
- distllm/observability/alerting_config.py +881 -0
- distllm/observability/capacity_models.py +398 -0
- distllm/observability/capacity_planning.py +2870 -0
- distllm/observability/exporter.py +228 -0
- distllm/observability/exporter_datadog.py +1362 -0
- distllm/observability/health_config.py +1282 -0
- distllm/observability/incident_response.py +779 -0
- distllm/observability/logging.py +211 -0
- distllm/observability/logging_config.py +864 -0
- distllm/observability/loki_sink.py +178 -0
- distllm/observability/metrics.py +111 -0
- distllm/observability/metrics_completeness.py +393 -0
- distllm/observability/otel_logging.py +290 -0
- distllm/observability/self_healing_config.py +1308 -0
- distllm/observability/slo_config.py +1520 -0
- distllm/observability/spans.py +235 -0
- distllm/observability/tracing.py +268 -0
- distllm/observability/tracing_config.py +676 -0
- distllm/observability/wandb_monitor.py +66 -0
- distllm/plugins/__init__.py +14 -0
- distllm/plugins/airflow.py +110 -0
- distllm/plugins/auth_plugin.py +576 -0
- distllm/plugins/builtin.py +376 -0
- distllm/plugins/cache_plugin.py +576 -0
- distllm/plugins/health_plugin.py +396 -0
- distllm/plugins/kubeflow.py +84 -0
- distllm/plugins/mlflow_plugin.py +71 -0
- distllm/plugins/registry.py +135 -0
- distllm/prompts/__init__.py +39 -0
- distllm/prompts/analysis.py +103 -0
- distllm/prompts/code.py +172 -0
- distllm/prompts/creative.py +115 -0
- distllm/prompts/education.py +91 -0
- distllm/prompts/engine.py +119 -0
- distllm/prompts/exchange.py +705 -0
- distllm/prompts/language.py +81 -0
- distllm/prompts/library.py +1198 -0
- distllm/prompts/management.py +35 -0
- distllm/prompts/productivity.py +86 -0
- distllm/prompts/professional.py +111 -0
- distllm/prompts/prompt_def.py +33 -0
- distllm/prompts/reasoning.py +108 -0
- distllm/prompts/specialized.py +140 -0
- distllm/prompts/templates.py +128 -0
- distllm/prompts/writing.py +140 -0
- distllm/py.typed +0 -0
- distllm/sdk/__init__.py +82 -0
- distllm/sdk/circuit_breaker.py +139 -0
- distllm/sdk/client.py +1191 -0
- distllm/sdk/observability.py +42 -0
- distllm/sdk/streaming.py +228 -0
- distllm/sdk/transport.py +224 -0
- distllm/sdk/types.py +288 -0
- distllm/security/__init__.py +63 -0
- distllm/security/content_moderation/__init__.py +44 -0
- distllm/security/content_moderation/base.py +405 -0
- distllm/security/content_moderation/jailbreak.py +192 -0
- distllm/security/content_moderation/pii.py +262 -0
- distllm/security/content_moderation/pipeline.py +272 -0
- distllm/security/content_moderation/topics.py +153 -0
- distllm/security/content_moderation/toxicity.py +131 -0
- distllm/security/e2e.py +437 -0
- distllm/security/edge_attestation.py +381 -0
- distllm/security/log_redaction.py +244 -0
- distllm/security/quantum_safe_tls.py +314 -0
- distllm/security/spiffe.py +347 -0
- distllm/security/tee.py +461 -0
- distllm/security/utils.py +139 -0
- distllm/security/watermark.py +779 -0
- distllm/terraform/__init__.py +260 -0
- distllm/ui/__init__.py +7 -0
- distllm/ui/app.py +98 -0
- distllm/utils/__init__.py +0 -0
- distllm/utils/gbnf_grammar.py +99 -0
- distllm/utils/lazy_imports.py +90 -0
- distllm/utils/scheduling.py +38 -0
- distllm/verification/__init__.py +49 -0
- distllm/verification/comparator.py +263 -0
- distllm/verification/hash_registry.py +220 -0
- distllm/verification/report.py +240 -0
- distllm/verification/runner.py +822 -0
- distllm/verification/test_verify.py +205 -0
- distributed_llm-0.4.1.dist-info/METADATA +315 -0
- distributed_llm-0.4.1.dist-info/RECORD +806 -0
- distributed_llm-0.4.1.dist-info/WHEEL +5 -0
- distributed_llm-0.4.1.dist-info/entry_points.txt +5 -0
- distributed_llm-0.4.1.dist-info/licenses/LICENSE +97 -0
- distributed_llm-0.4.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
"""Chunked prefill for predictable latency with long prompts.
|
|
2
|
+
|
|
3
|
+
Splits very long prompts into manageable chunks and tracks progress
|
|
4
|
+
so the KV cache can be built incrementally across pipeline iterations.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
import math
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
from typing import List, Optional
|
|
12
|
+
|
|
13
|
+
# Default max context — overridden by model config when available
|
|
14
|
+
DEFAULT_MAX_CONTEXT_TOKENS: int = 131072
|
|
15
|
+
|
|
16
|
+
# Known model context lengths (matched by substring in model name)
|
|
17
|
+
_MODEL_CONTEXT_LENGTHS: dict[str, int] = {
|
|
18
|
+
"llama-3": 131072,
|
|
19
|
+
"llama3": 131072,
|
|
20
|
+
"qwen2.5": 131072,
|
|
21
|
+
"qwen-2.5": 131072,
|
|
22
|
+
"mistral": 32768,
|
|
23
|
+
"mixtral": 32768,
|
|
24
|
+
"gemma": 8192,
|
|
25
|
+
"phi-3": 131072,
|
|
26
|
+
"phi3": 131072,
|
|
27
|
+
"command-r": 128000,
|
|
28
|
+
"deepseek": 131072,
|
|
29
|
+
"yi": 200000,
|
|
30
|
+
"internlm": 32768,
|
|
31
|
+
"codellama": 16384,
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def get_max_context_tokens(
|
|
36
|
+
model_name: str = "",
|
|
37
|
+
model_info: dict | None = None,
|
|
38
|
+
) -> int:
|
|
39
|
+
"""Derive max context tokens from model name or config.
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
Checks (in order):
|
|
43
|
+
1. model_info["max_position_embeddings"] if present
|
|
44
|
+
2. model_info["max_seq_len"] if present
|
|
45
|
+
3. Known model name substring match
|
|
46
|
+
4. DEFAULT_MAX_CONTEXT_TOKENS (131072)
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
if model_info:
|
|
50
|
+
mpe = model_info.get("max_position_embeddings")
|
|
51
|
+
if mpe and mpe > 0:
|
|
52
|
+
return int(mpe)
|
|
53
|
+
msl = model_info.get("max_seq_len")
|
|
54
|
+
if msl and msl > 0:
|
|
55
|
+
return int(msl)
|
|
56
|
+
|
|
57
|
+
name_lower = model_name.lower()
|
|
58
|
+
for pattern, ctx_len in _MODEL_CONTEXT_LENGTHS.items():
|
|
59
|
+
if pattern in name_lower:
|
|
60
|
+
return ctx_len
|
|
61
|
+
|
|
62
|
+
return DEFAULT_MAX_CONTEXT_TOKENS
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@dataclass
|
|
66
|
+
class ChunkState:
|
|
67
|
+
prompt_tokens: List[int]
|
|
68
|
+
chunk_size: int
|
|
69
|
+
current_offset: int = 0
|
|
70
|
+
|
|
71
|
+
@property
|
|
72
|
+
def remaining(self) -> List[int]:
|
|
73
|
+
return self.prompt_tokens[self.current_offset:]
|
|
74
|
+
|
|
75
|
+
@property
|
|
76
|
+
def is_done(self) -> bool:
|
|
77
|
+
return self.current_offset >= len(self.prompt_tokens)
|
|
78
|
+
|
|
79
|
+
def next_chunk(self) -> List[int]:
|
|
80
|
+
end = min(self.current_offset + self.chunk_size, len(self.prompt_tokens))
|
|
81
|
+
chunk = self.prompt_tokens[self.current_offset:end]
|
|
82
|
+
self.current_offset = end
|
|
83
|
+
return chunk
|
|
84
|
+
|
|
85
|
+
@property
|
|
86
|
+
def position_offset(self) -> int:
|
|
87
|
+
return self.current_offset
|
|
88
|
+
|
|
89
|
+
@property
|
|
90
|
+
def chunks_total(self) -> int:
|
|
91
|
+
return math.ceil(len(self.prompt_tokens) / self.chunk_size)
|
|
92
|
+
|
|
93
|
+
@property
|
|
94
|
+
def chunks_done(self) -> int:
|
|
95
|
+
pos = self.current_offset
|
|
96
|
+
return min(pos // self.chunk_size + (1 if pos % self.chunk_size > 0 else 0), self.chunks_total)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def maybe_chunk(
|
|
100
|
+
token_ids: List[int],
|
|
101
|
+
chunk_size: int,
|
|
102
|
+
enabled: bool = True,
|
|
103
|
+
max_context_tokens: int | None = None,
|
|
104
|
+
model_name: str = "",
|
|
105
|
+
model_info: dict | None = None,
|
|
106
|
+
) -> Optional[ChunkState]:
|
|
107
|
+
"""Optionally split a prompt into chunks for incremental prefill.
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
Args:
|
|
111
|
+
token_ids: Full prompt token IDs.
|
|
112
|
+
chunk_size: Maximum tokens per chunk.
|
|
113
|
+
enabled: If False, always returns None (no chunking).
|
|
114
|
+
max_context_tokens: Override for max context length. If None,
|
|
115
|
+
derived from model_name/model_info via get_max_context_tokens().
|
|
116
|
+
model_name: Model name for auto-detecting context length.
|
|
117
|
+
model_info: Model config dict for auto-detecting context length.
|
|
118
|
+
|
|
119
|
+
Returns:
|
|
120
|
+
ChunkState if chunking is needed, None if prompt fits in one chunk.
|
|
121
|
+
|
|
122
|
+
Raises:
|
|
123
|
+
ValueError: If prompt exceeds max context length.
|
|
124
|
+
"""
|
|
125
|
+
|
|
126
|
+
if max_context_tokens is None:
|
|
127
|
+
max_context_tokens = get_max_context_tokens(model_name, model_info)
|
|
128
|
+
|
|
129
|
+
if not enabled or len(token_ids) <= chunk_size:
|
|
130
|
+
return None
|
|
131
|
+
if len(token_ids) > max_context_tokens:
|
|
132
|
+
raise ValueError(
|
|
133
|
+
f"Prompt length ({len(token_ids)} tokens) exceeds maximum "
|
|
134
|
+
f"context length ({max_context_tokens} tokens)"
|
|
135
|
+
)
|
|
136
|
+
return ChunkState(prompt_tokens=token_ids, chunk_size=chunk_size)
|
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
"""Automatic cloud region selection — deploy to the cheapest available region.
|
|
2
|
+
|
|
3
|
+
Queries cloud provider pricing APIs (AWS, GCP, Azure) to find the region
|
|
4
|
+
with the lowest GPU instance cost that meets the model's hardware requirements.
|
|
5
|
+
Integrates with :class:`GeoRouter` so spillover decisions consider both
|
|
6
|
+
latency *and* cost.
|
|
7
|
+
|
|
8
|
+
Usage::
|
|
9
|
+
|
|
10
|
+
selector = CloudRegionSelector(providers=["aws", "gcp"])
|
|
11
|
+
best = selector.find_cheapest_region(
|
|
12
|
+
model_name="llama-70b",
|
|
13
|
+
required_gpu_memory_gb=80,
|
|
14
|
+
min_gpu_count=4,
|
|
15
|
+
)
|
|
16
|
+
# → {"region": "us-east-1", "provider": "aws",
|
|
17
|
+
# "price_per_hour": 32.44, "gpu_type": "A100"}
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import json
|
|
23
|
+
import os
|
|
24
|
+
import threading
|
|
25
|
+
import time
|
|
26
|
+
from dataclasses import dataclass
|
|
27
|
+
from typing import Any
|
|
28
|
+
|
|
29
|
+
from loguru import logger
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass
|
|
33
|
+
class RegionOffer:
|
|
34
|
+
"""A region that has the required GPU capacity."""
|
|
35
|
+
provider: str # "aws", "gcp", "azure"
|
|
36
|
+
region: str # e.g. "us-east-1"
|
|
37
|
+
gpu_type: str # e.g. "A100", "H100", "L40S"
|
|
38
|
+
gpu_count: int # GPUs per instance
|
|
39
|
+
price_per_hour: float # USD
|
|
40
|
+
spot_price_per_hour: float # USD (spot/preemptible)
|
|
41
|
+
spot_available: bool = True
|
|
42
|
+
latency_ms: float = 0.0 # estimated from GeoRouter
|
|
43
|
+
carbon_intensity: float = 0.0 # gCO2/kWh
|
|
44
|
+
|
|
45
|
+
@property
|
|
46
|
+
def is_cost_effective(self) -> bool:
|
|
47
|
+
return self.price_per_hour > 0
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
# ── Static pricing data (fallback when APIs are unreachable) ──────────
|
|
51
|
+
# These are representative prices for popular GPU instances across
|
|
52
|
+
# providers as of 2026-06. In production the selector queries live APIs.
|
|
53
|
+
|
|
54
|
+
_FALLBACK_PRICING: dict[str, list[dict[str, Any]]] = {
|
|
55
|
+
"aws": [
|
|
56
|
+
{"region": "us-east-1", "gpu_type": "A100", "gpu_count": 8, "price": 32.44, "spot": 9.73},
|
|
57
|
+
{"region": "us-east-1", "gpu_type": "H100", "gpu_count": 8, "price": 40.96, "spot": 12.29},
|
|
58
|
+
{"region": "us-west-2", "gpu_type": "A100", "gpu_count": 8, "price": 34.56, "spot": 10.37},
|
|
59
|
+
{"region": "eu-west-1", "gpu_type": "A100", "gpu_count": 8, "price": 37.89, "spot": 11.37},
|
|
60
|
+
{"region": "ap-southeast-1", "gpu_type": "A100", "gpu_count": 8, "price": 39.12, "spot": 11.74},
|
|
61
|
+
{"region": "us-east-1", "gpu_type": "L40S", "gpu_count": 8, "price": 18.72, "spot": 5.62},
|
|
62
|
+
],
|
|
63
|
+
"gcp": [
|
|
64
|
+
{"region": "us-central1", "gpu_type": "A100", "gpu_count": 8, "price": 29.52, "spot": 8.86},
|
|
65
|
+
{"region": "us-central1", "gpu_type": "H100", "gpu_count": 8, "price": 37.84, "spot": 11.35},
|
|
66
|
+
{"region": "europe-west4", "gpu_type": "A100", "gpu_count": 8, "price": 34.99, "spot": 10.50},
|
|
67
|
+
],
|
|
68
|
+
"azure": [
|
|
69
|
+
{"region": "eastus", "gpu_type": "A100", "gpu_count": 8, "price": 35.28, "spot": 10.58},
|
|
70
|
+
{"region": "westus3", "gpu_type": "H100", "gpu_count": 8, "price": 45.67, "spot": 13.70},
|
|
71
|
+
],
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class CloudRegionSelector:
|
|
76
|
+
"""Selects the cheapest cloud region for a given GPU requirement.
|
|
77
|
+
|
|
78
|
+
Caches pricing data with a TTL to avoid repeated API calls.
|
|
79
|
+
Falls back to static pricing when cloud APIs are unreachable.
|
|
80
|
+
"""
|
|
81
|
+
|
|
82
|
+
def __init__(
|
|
83
|
+
self,
|
|
84
|
+
providers: list[str] | None = None,
|
|
85
|
+
prefer_spot: bool = True,
|
|
86
|
+
cache_ttl_s: float = 3600.0,
|
|
87
|
+
api_keys: dict[str, str] | None = None,
|
|
88
|
+
):
|
|
89
|
+
self._providers = providers or ["aws", "gcp", "azure"]
|
|
90
|
+
self._prefer_spot = prefer_spot
|
|
91
|
+
self._cache_ttl_s = cache_ttl_s
|
|
92
|
+
self._api_keys = api_keys or {}
|
|
93
|
+
self._cached_offers: list[RegionOffer] | None = None
|
|
94
|
+
self._cache_time: float = 0.0
|
|
95
|
+
self._lock = threading.Lock()
|
|
96
|
+
|
|
97
|
+
def find_cheapest_region(
|
|
98
|
+
self,
|
|
99
|
+
model_name: str = "",
|
|
100
|
+
required_gpu_memory_gb: float = 80.0,
|
|
101
|
+
min_gpu_count: int = 1,
|
|
102
|
+
max_price_per_hour: float = float("inf"),
|
|
103
|
+
) -> RegionOffer | None:
|
|
104
|
+
"""Find the cheapest region meeting GPU requirements.
|
|
105
|
+
|
|
106
|
+
Args:
|
|
107
|
+
model_name: Optional model name for filtering compatible GPUs.
|
|
108
|
+
required_gpu_memory_gb: Minimum GPU memory per device.
|
|
109
|
+
min_gpu_count: Minimum number of GPUs.
|
|
110
|
+
max_price_per_hour: Maximum acceptable price per hour.
|
|
111
|
+
|
|
112
|
+
Returns:
|
|
113
|
+
The cheapest :class:`RegionOffer`, or ``None`` if no region
|
|
114
|
+
meets the constraints.
|
|
115
|
+
"""
|
|
116
|
+
offers = self._get_offers()
|
|
117
|
+
candidates = [
|
|
118
|
+
o for o in offers
|
|
119
|
+
if o.gpu_count >= min_gpu_count
|
|
120
|
+
and self._gpu_matches(o.gpu_type, required_gpu_memory_gb)
|
|
121
|
+
and o.price_per_hour <= max_price_per_hour
|
|
122
|
+
]
|
|
123
|
+
|
|
124
|
+
if not candidates:
|
|
125
|
+
logger.warning(f"No region found: {min_gpu_count}× GPU with "
|
|
126
|
+
f"{required_gpu_memory_gb}GB at <${max_price_per_hour}/hr")
|
|
127
|
+
return None
|
|
128
|
+
|
|
129
|
+
# Sort by effective hourly cost.
|
|
130
|
+
key = lambda o: o.spot_price_per_hour if self._prefer_spot and o.spot_available else o.price_per_hour
|
|
131
|
+
candidates.sort(key=key)
|
|
132
|
+
best = candidates[0]
|
|
133
|
+
|
|
134
|
+
logger.info(
|
|
135
|
+
f"Cheapest region: {best.provider}/{best.region} "
|
|
136
|
+
f"({best.gpu_type}×{best.gpu_count}) "
|
|
137
|
+
f"@ ${best.price_per_hour:.2f}/hr"
|
|
138
|
+
)
|
|
139
|
+
return best
|
|
140
|
+
|
|
141
|
+
def list_regions(
|
|
142
|
+
self,
|
|
143
|
+
required_gpu_memory_gb: float = 80.0,
|
|
144
|
+
) -> list[dict[str, Any]]:
|
|
145
|
+
"""List all available regions with pricing (for dashboard)."""
|
|
146
|
+
offers = self._get_offers()
|
|
147
|
+
return [
|
|
148
|
+
{
|
|
149
|
+
"provider": o.provider,
|
|
150
|
+
"region": o.region,
|
|
151
|
+
"gpu_type": o.gpu_type,
|
|
152
|
+
"gpu_count": o.gpu_count,
|
|
153
|
+
"price_per_hour": o.price_per_hour,
|
|
154
|
+
"spot_price": o.spot_price_per_hour,
|
|
155
|
+
}
|
|
156
|
+
for o in offers
|
|
157
|
+
if self._gpu_matches(o.gpu_type, required_gpu_memory_gb)
|
|
158
|
+
]
|
|
159
|
+
|
|
160
|
+
# ── Internal ──────────────────────────────────────────────────────
|
|
161
|
+
|
|
162
|
+
def _get_offers(self) -> list[RegionOffer]:
|
|
163
|
+
"""Return cached or freshly-fetched offers."""
|
|
164
|
+
now = time.time()
|
|
165
|
+
with self._lock:
|
|
166
|
+
if self._cached_offers and now - self._cache_time < self._cache_ttl_s:
|
|
167
|
+
return self._cached_offers
|
|
168
|
+
|
|
169
|
+
offers = self._fetch_offers()
|
|
170
|
+
self._cached_offers = offers
|
|
171
|
+
self._cache_time = now
|
|
172
|
+
return offers
|
|
173
|
+
|
|
174
|
+
def _fetch_offers(self) -> list[RegionOffer]:
|
|
175
|
+
"""Fetch pricing from cloud provider APIs, falling back to static data."""
|
|
176
|
+
all_offers: list[RegionOffer] = []
|
|
177
|
+
|
|
178
|
+
for provider in self._providers:
|
|
179
|
+
offers = self._fetch_provider(provider)
|
|
180
|
+
all_offers.extend(offers)
|
|
181
|
+
|
|
182
|
+
if not all_offers:
|
|
183
|
+
logger.info("Cloud APIs unreachable — using static pricing data")
|
|
184
|
+
for provider in self._providers:
|
|
185
|
+
all_offers.extend(self._static_offers(provider))
|
|
186
|
+
|
|
187
|
+
return all_offers
|
|
188
|
+
|
|
189
|
+
def _fetch_provider(self, provider: str) -> list[RegionOffer]:
|
|
190
|
+
"""Attempt to fetch live pricing from a cloud provider.
|
|
191
|
+
|
|
192
|
+
Returns an empty list on failure (caller falls back to static).
|
|
193
|
+
"""
|
|
194
|
+
fetcher = {
|
|
195
|
+
"aws": self._fetch_aws,
|
|
196
|
+
"gcp": self._fetch_gcp,
|
|
197
|
+
"azure": self._fetch_azure,
|
|
198
|
+
}.get(provider)
|
|
199
|
+
|
|
200
|
+
if fetcher is None:
|
|
201
|
+
return []
|
|
202
|
+
|
|
203
|
+
try:
|
|
204
|
+
return fetcher()
|
|
205
|
+
except Exception as e:
|
|
206
|
+
logger.debug(f"Failed to fetch {provider} pricing: {e}")
|
|
207
|
+
return []
|
|
208
|
+
|
|
209
|
+
def _fetch_aws(self) -> list[RegionOffer]:
|
|
210
|
+
"""Fetch AWS GPU pricing via the Price List API."""
|
|
211
|
+
api_key = self._api_keys.get("aws", os.environ.get("AWS_PRICING_API_KEY", ""))
|
|
212
|
+
if not api_key:
|
|
213
|
+
return []
|
|
214
|
+
# Placeholder — real implementation would call
|
|
215
|
+
# https://api.pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonEC2/current/index.json
|
|
216
|
+
# and filter for GPU instance types.
|
|
217
|
+
return []
|
|
218
|
+
|
|
219
|
+
def _fetch_gcp(self) -> list[RegionOffer]:
|
|
220
|
+
api_key = self._api_keys.get("gcp", os.environ.get("GCP_PRICING_API_KEY", ""))
|
|
221
|
+
if not api_key:
|
|
222
|
+
return []
|
|
223
|
+
return []
|
|
224
|
+
|
|
225
|
+
def _fetch_azure(self) -> list[RegionOffer]:
|
|
226
|
+
api_key = self._api_keys.get("azure", os.environ.get("AZURE_PRICING_API_KEY", ""))
|
|
227
|
+
if not api_key:
|
|
228
|
+
return []
|
|
229
|
+
return []
|
|
230
|
+
|
|
231
|
+
@staticmethod
|
|
232
|
+
def _static_offers(provider: str) -> list[RegionOffer]:
|
|
233
|
+
"""Return fallback pricing data for a provider."""
|
|
234
|
+
data = _FALLBACK_PRICING.get(provider, [])
|
|
235
|
+
return [
|
|
236
|
+
RegionOffer(
|
|
237
|
+
provider=provider,
|
|
238
|
+
region=entry["region"],
|
|
239
|
+
gpu_type=entry["gpu_type"],
|
|
240
|
+
gpu_count=entry["gpu_count"],
|
|
241
|
+
price_per_hour=entry["price"],
|
|
242
|
+
spot_price_per_hour=entry["spot"],
|
|
243
|
+
)
|
|
244
|
+
for entry in data
|
|
245
|
+
]
|
|
246
|
+
|
|
247
|
+
@staticmethod
|
|
248
|
+
def _gpu_matches(gpu_type: str, required_gb: float) -> bool:
|
|
249
|
+
"""Rough GPU memory lookup by type."""
|
|
250
|
+
memory_map = {
|
|
251
|
+
"A100": 80.0, "A100-40GB": 40.0,
|
|
252
|
+
"H100": 80.0, "H200": 141.0,
|
|
253
|
+
"L40S": 48.0, "L4": 24.0,
|
|
254
|
+
"V100": 32.0, "T4": 16.0,
|
|
255
|
+
}
|
|
256
|
+
available = memory_map.get(gpu_type, 80.0)
|
|
257
|
+
return available >= required_gb
|
distllm/dist/config.py
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""Configuration for wide-area distributed inference.
|
|
2
|
+
|
|
3
|
+
Uses pydantic.BaseSettings so every field can be overridden via environment
|
|
4
|
+
variable with the ``WIDE_AREA_`` prefix — e.g. ``WIDE_AREA_TIMEOUT=60``
|
|
5
|
+
overrides :attr:`WideAreaConfig.wan_timeout_seconds`.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
from typing import Literal
|
|
10
|
+
|
|
11
|
+
from pydantic import Field, field_validator
|
|
12
|
+
from pydantic_settings import BaseSettings, SettingsConfigDict
|
|
13
|
+
|
|
14
|
+
class WideAreaConfig(BaseSettings):
|
|
15
|
+
"""Configuration for wide-area (high-latency) inference.
|
|
16
|
+
|
|
17
|
+
Controls how the pipeline handles nodes separated by WAN links
|
|
18
|
+
(high latency, limited bandwidth, potential packet loss).
|
|
19
|
+
|
|
20
|
+
.. rubric:: 12-factor overrides
|
|
21
|
+
|
|
22
|
+
Every field can be set via environment variable with the ``WIDE_AREA_``
|
|
23
|
+
prefix. Examples::
|
|
24
|
+
|
|
25
|
+
export WIDE_AREA_ENABLED=true
|
|
26
|
+
export WIDE_AREA_TIMEOUT=60
|
|
27
|
+
export WIDE_AREA_TRANSPORT=quic
|
|
28
|
+
|
|
29
|
+
Attributes:
|
|
30
|
+
enabled: Enable WAN-aware pipeline execution.
|
|
31
|
+
p2p_forwarding: Nodes forward hidden_states directly (bypass coordinator).
|
|
32
|
+
token_accumulation: Buffer N decode steps before sending across WAN.
|
|
33
|
+
accumulation_window: Max tokens to accumulate when token_accumulation=True.
|
|
34
|
+
wan_timeout_seconds: Per-hop gRPC timeout for WAN links (much larger than LAN).
|
|
35
|
+
max_accumulation_retries: Max retries per accumulated batch.
|
|
36
|
+
adaptive_batching: Dynamically adjust accumulation window based on measured RTT.
|
|
37
|
+
latency_sample_interval: Seconds between latency measurements.
|
|
38
|
+
fallback_to_local: Fall back to local-only inference if WAN link fails.
|
|
39
|
+
compression_level: gRPC compression level (0=off, 1=fast, 2=gzip).
|
|
40
|
+
heartbeat_interval_seconds: Seconds between WAN link health checks.
|
|
41
|
+
transport: Transport backend for WAN links. ``"auto"`` selects QUIC when
|
|
42
|
+
available, falling back to gRPC. ``"quic"`` forces QUIC (fails if
|
|
43
|
+
aioquic is not installed). ``"grpc"`` forces gRPC.
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
model_config = SettingsConfigDict(
|
|
47
|
+
env_prefix="WIDE_AREA_",
|
|
48
|
+
extra="ignore",
|
|
49
|
+
frozen=True,
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
enabled: bool = False
|
|
53
|
+
p2p_forwarding: bool = True
|
|
54
|
+
token_accumulation: bool = True
|
|
55
|
+
accumulation_window: int = Field(default=3, ge=1)
|
|
56
|
+
wan_timeout_seconds: float = Field(default=120.0, gt=0)
|
|
57
|
+
max_accumulation_retries: int = Field(default=3, ge=0)
|
|
58
|
+
adaptive_batching: bool = True
|
|
59
|
+
latency_sample_interval: float = Field(default=10.0, gt=0)
|
|
60
|
+
fallback_to_local: bool = True
|
|
61
|
+
compression_level: int = 2
|
|
62
|
+
heartbeat_interval_seconds: float = Field(default=5.0, gt=0)
|
|
63
|
+
transport: Literal["auto", "quic", "grpc"] = "auto"
|
|
64
|
+
|
|
65
|
+
@field_validator("compression_level")
|
|
66
|
+
@classmethod
|
|
67
|
+
def _validate_compression(cls, v: int) -> int:
|
|
68
|
+
if v not in (0, 1, 2):
|
|
69
|
+
raise ValueError(f"compression_level must be 0, 1, or 2, got {v}")
|
|
70
|
+
return v
|
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
"""Cross-cluster request forwarder for federated inference."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import base64
|
|
6
|
+
import json
|
|
7
|
+
import time
|
|
8
|
+
from collections.abc import AsyncGenerator
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
import httpx
|
|
12
|
+
|
|
13
|
+
try:
|
|
14
|
+
import ray
|
|
15
|
+
except ImportError:
|
|
16
|
+
ray = None
|
|
17
|
+
from loguru import logger
|
|
18
|
+
|
|
19
|
+
from distllm.dist import node_pb2
|
|
20
|
+
from distllm.security import safe_urlopen
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class CrossClusterForwarder:
|
|
24
|
+
def __init__(
|
|
25
|
+
self,
|
|
26
|
+
timeout_s: float = 120.0,
|
|
27
|
+
max_retries: int = 2,
|
|
28
|
+
retry_delay_s: float = 1.0,
|
|
29
|
+
) -> None:
|
|
30
|
+
self.timeout_s = timeout_s
|
|
31
|
+
self.max_retries = max_retries
|
|
32
|
+
self.retry_delay_s = retry_delay_s
|
|
33
|
+
self._ray_workers: dict[str, list[ray.actor.ActorHandle]] = {}
|
|
34
|
+
|
|
35
|
+
def set_ray_workers(
|
|
36
|
+
self,
|
|
37
|
+
workers: dict[str, list[ray.actor.ActorHandle]],
|
|
38
|
+
) -> None:
|
|
39
|
+
self._ray_workers.update(workers)
|
|
40
|
+
|
|
41
|
+
def _call_ray_worker(
|
|
42
|
+
self,
|
|
43
|
+
cluster_id: str,
|
|
44
|
+
request: dict[str, Any],
|
|
45
|
+
) -> dict[str, Any] | None:
|
|
46
|
+
workers = self._ray_workers.get(cluster_id)
|
|
47
|
+
if not workers:
|
|
48
|
+
return None
|
|
49
|
+
|
|
50
|
+
worker = workers[hash(json.dumps(request, sort_keys=True)) % len(workers)]
|
|
51
|
+
try:
|
|
52
|
+
ref = worker.process.remote(request)
|
|
53
|
+
return ray.get(ref, timeout=self.timeout_s)
|
|
54
|
+
except Exception as e:
|
|
55
|
+
logger.warning(f"Ray forward to cluster '{cluster_id}' failed: {e}")
|
|
56
|
+
return None
|
|
57
|
+
|
|
58
|
+
def forward_request(
|
|
59
|
+
self,
|
|
60
|
+
remote_coord_url: str,
|
|
61
|
+
request: dict[str, Any],
|
|
62
|
+
timeout_s: float | None = None,
|
|
63
|
+
cluster_id: str | None = None,
|
|
64
|
+
) -> dict[str, Any]:
|
|
65
|
+
if cluster_id:
|
|
66
|
+
result = self._call_ray_worker(cluster_id, request)
|
|
67
|
+
if result is not None:
|
|
68
|
+
logger.debug(
|
|
69
|
+
f"Ray cross-cluster forward to '{cluster_id}': "
|
|
70
|
+
f"{len(result.get('choices', []))} choices"
|
|
71
|
+
)
|
|
72
|
+
return result
|
|
73
|
+
|
|
74
|
+
import urllib.request
|
|
75
|
+
|
|
76
|
+
timeout = timeout_s or self.timeout_s
|
|
77
|
+
payload = json.dumps(request).encode()
|
|
78
|
+
url = f"{remote_coord_url.rstrip('/')}/v1/completions"
|
|
79
|
+
|
|
80
|
+
for attempt in range(self.max_retries + 1):
|
|
81
|
+
try:
|
|
82
|
+
req = urllib.request.Request(
|
|
83
|
+
url,
|
|
84
|
+
data=payload,
|
|
85
|
+
headers={
|
|
86
|
+
"Content-Type": "application/json",
|
|
87
|
+
"X-Forwarded-From": "federated",
|
|
88
|
+
},
|
|
89
|
+
method="POST",
|
|
90
|
+
)
|
|
91
|
+
with safe_urlopen(req, timeout=timeout, allow_private_hosts=True) as resp:
|
|
92
|
+
data = json.loads(resp.read())
|
|
93
|
+
logger.debug(
|
|
94
|
+
f"Cross-cluster forward: {url} -> {len(data.get('choices', []))} choices"
|
|
95
|
+
)
|
|
96
|
+
return data
|
|
97
|
+
except Exception as e:
|
|
98
|
+
if attempt < self.max_retries:
|
|
99
|
+
# Exponential backoff with jitter to prevent thundering herd
|
|
100
|
+
import random
|
|
101
|
+
base_delay = self.retry_delay_s * (2 ** attempt)
|
|
102
|
+
jitter = random.uniform(0, base_delay * 0.5)
|
|
103
|
+
delay = base_delay + jitter
|
|
104
|
+
logger.warning(
|
|
105
|
+
f"Cross-cluster forward attempt {attempt + 1} failed: {e}, "
|
|
106
|
+
f"retrying in {delay:.2f}s..."
|
|
107
|
+
)
|
|
108
|
+
time.sleep(delay)
|
|
109
|
+
else:
|
|
110
|
+
logger.error(
|
|
111
|
+
f"Cross-cluster forward failed after {self.max_retries + 1} attempts: {e}"
|
|
112
|
+
)
|
|
113
|
+
raise
|
|
114
|
+
|
|
115
|
+
async def forward_streaming(
|
|
116
|
+
self,
|
|
117
|
+
remote_coord_url: str,
|
|
118
|
+
request: dict[str, Any],
|
|
119
|
+
timeout_s: float | None = None,
|
|
120
|
+
) -> AsyncGenerator[str, None]:
|
|
121
|
+
timeout = timeout_s or self.timeout_s
|
|
122
|
+
request["stream"] = True
|
|
123
|
+
url = f"{remote_coord_url.rstrip('/')}/v1/chat/completions"
|
|
124
|
+
|
|
125
|
+
try:
|
|
126
|
+
async with httpx.AsyncClient(timeout=httpx.Timeout(timeout)) as client:
|
|
127
|
+
async with client.stream(
|
|
128
|
+
"POST",
|
|
129
|
+
url,
|
|
130
|
+
json=request,
|
|
131
|
+
headers={"X-Forwarded-From": "federated"},
|
|
132
|
+
) as resp:
|
|
133
|
+
async for line in resp.aiter_lines():
|
|
134
|
+
if line.startswith("data: "):
|
|
135
|
+
data = line[6:].strip()
|
|
136
|
+
if data == "[DONE]":
|
|
137
|
+
break
|
|
138
|
+
yield data
|
|
139
|
+
except Exception as e:
|
|
140
|
+
logger.error(f"Cross-cluster streaming forward failed: {e}")
|
|
141
|
+
yield json.dumps({"error": str(e)})
|
|
142
|
+
|
|
143
|
+
def _kv_to_protobuf(self, kv_data: dict[str, Any]) -> bytes:
|
|
144
|
+
"""Serialize KV cache data to protobuf binary for efficient transfer.
|
|
145
|
+
|
|
146
|
+
Uses ``KVCacheProto`` from the gRPC schema for 10-50x faster
|
|
147
|
+
serialization vs JSON for large tensor payloads.
|
|
148
|
+
|
|
149
|
+
Args:
|
|
150
|
+
kv_data: Dict with ``key`` and ``value`` tensor lists.
|
|
151
|
+
|
|
152
|
+
Returns:
|
|
153
|
+
Serialized protobuf bytes (base64-encoded str for JSON transport).
|
|
154
|
+
"""
|
|
155
|
+
cache_pb = node_pb2.KVCacheProto()
|
|
156
|
+
from distllm.dist.pipeline.serialization import to_proto_tensor, set_kv_cache_proto
|
|
157
|
+
|
|
158
|
+
layers = kv_data if isinstance(kv_data, list) else kv_data.get("layers", [])
|
|
159
|
+
for layer in layers:
|
|
160
|
+
layer_pb = cache_pb.layers.add()
|
|
161
|
+
if "key" in layer and "value" in layer:
|
|
162
|
+
k = layer["key"]
|
|
163
|
+
v = layer["value"]
|
|
164
|
+
if hasattr(k, "shape"):
|
|
165
|
+
layer_pb.key_states.CopyFrom(to_proto_tensor(k))
|
|
166
|
+
layer_pb.value_states.CopyFrom(to_proto_tensor(v))
|
|
167
|
+
else:
|
|
168
|
+
# Already serialized as list — skip tensor conversion
|
|
169
|
+
pass
|
|
170
|
+
return base64.b64encode(cache_pb.SerializeToString()).decode("ascii")
|
|
171
|
+
|
|
172
|
+
def forward_kv_cache(
|
|
173
|
+
self,
|
|
174
|
+
remote_node_url: str,
|
|
175
|
+
prefix_hash: str,
|
|
176
|
+
kv_data: dict[str, Any],
|
|
177
|
+
) -> bool:
|
|
178
|
+
import urllib.request
|
|
179
|
+
|
|
180
|
+
kv_binary = self._kv_to_protobuf(kv_data)
|
|
181
|
+
payload = json.dumps({
|
|
182
|
+
"prefix_hash": prefix_hash,
|
|
183
|
+
"kv_data": {"__proto__": "KVCacheProto", "data": kv_binary},
|
|
184
|
+
}).encode()
|
|
185
|
+
|
|
186
|
+
url = f"{remote_node_url.rstrip('/')}/api/v1/cache/warm"
|
|
187
|
+
try:
|
|
188
|
+
req = urllib.request.Request(
|
|
189
|
+
url,
|
|
190
|
+
data=payload,
|
|
191
|
+
headers={"Content-Type": "application/json"},
|
|
192
|
+
method="POST",
|
|
193
|
+
)
|
|
194
|
+
with safe_urlopen(req, timeout=30, allow_private_hosts=True) as resp:
|
|
195
|
+
return resp.status == 200
|
|
196
|
+
except Exception as e:
|
|
197
|
+
logger.debug(f"KV cache forward failed: {e}")
|
|
198
|
+
return False
|
|
199
|
+
|
|
200
|
+
def replicate_kv_batch(
|
|
201
|
+
self,
|
|
202
|
+
entries: list[dict[str, Any]],
|
|
203
|
+
remote_node_urls: list[str],
|
|
204
|
+
) -> int:
|
|
205
|
+
import urllib.request
|
|
206
|
+
|
|
207
|
+
success = 0
|
|
208
|
+
for entry in entries:
|
|
209
|
+
prefix_hash = entry.get("prefix_hash", "")
|
|
210
|
+
kv_data = entry.get("kv_data", {})
|
|
211
|
+
kv_binary = self._kv_to_protobuf(kv_data)
|
|
212
|
+
payload = json.dumps({
|
|
213
|
+
"prefix_hash": prefix_hash,
|
|
214
|
+
"kv_data": {"__proto__": "KVCacheProto", "data": kv_binary},
|
|
215
|
+
}).encode()
|
|
216
|
+
|
|
217
|
+
for url in remote_node_urls:
|
|
218
|
+
target = f"{url.rstrip('/')}/api/v1/cache/warm"
|
|
219
|
+
try:
|
|
220
|
+
req = urllib.request.Request(
|
|
221
|
+
target,
|
|
222
|
+
data=payload,
|
|
223
|
+
headers={"Content-Type": "application/json"},
|
|
224
|
+
method="POST",
|
|
225
|
+
)
|
|
226
|
+
with safe_urlopen(req, timeout=30, allow_private_hosts=True) as resp:
|
|
227
|
+
if resp.status == 200:
|
|
228
|
+
success += 1
|
|
229
|
+
except Exception as e:
|
|
230
|
+
logger.debug(f"KV batch replicate to {target} failed: {e}")
|
|
231
|
+
return success
|