distributed-llm 0.4.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- distllm/__init__.py +47 -0
- distllm/api/__init__.py +11 -0
- distllm/api/api_settings.py +126 -0
- distllm/api/api_state.py +85 -0
- distllm/api/api_versioning.py +984 -0
- distllm/api/app_state.py +29 -0
- distllm/api/auth/__init__.py +181 -0
- distllm/api/auth/models.py +37 -0
- distllm/api/auth/oauth2.py +123 -0
- distllm/api/auth/oidc.py +317 -0
- distllm/api/auth/saml.py +107 -0
- distllm/api/auth/store.py +39 -0
- distllm/api/auth_deps.py +68 -0
- distllm/api/authz/__init__.py +30 -0
- distllm/api/authz/opa.py +297 -0
- distllm/api/background_tasks.py +295 -0
- distllm/api/body_cache_middleware.py +117 -0
- distllm/api/circuit_breaker_middleware.py +175 -0
- distllm/api/cost_middleware.py +186 -0
- distllm/api/csrf_middleware.py +196 -0
- distllm/api/dedup.py +180 -0
- distllm/api/errors.py +256 -0
- distllm/api/grpc_bridge.py +296 -0
- distllm/api/ip_utils.py +58 -0
- distllm/api/middleware.py +706 -0
- distllm/api/observability_middleware.py +169 -0
- distllm/api/persistent_store.py +295 -0
- distllm/api/prompt_injection.py +418 -0
- distllm/api/quota_middleware.py +186 -0
- distllm/api/rate_limit_middleware.py +13 -0
- distllm/api/rate_limiter.py +250 -0
- distllm/api/rate_limiter_unified.py +975 -0
- distllm/api/redis_rate_limiter.py +164 -0
- distllm/api/routes/__init__.py +42 -0
- distllm/api/routes/admin.py +599 -0
- distllm/api/routes/api_keys.py +115 -0
- distllm/api/routes/assistants.py +443 -0
- distllm/api/routes/batch.py +1094 -0
- distllm/api/routes/chat.py +875 -0
- distllm/api/routes/completion.py +141 -0
- distllm/api/routes/debug.py +185 -0
- distllm/api/routes/defrag.py +66 -0
- distllm/api/routes/embeddings.py +399 -0
- distllm/api/routes/eval.py +254 -0
- distllm/api/routes/exchange.py +367 -0
- distllm/api/routes/experiments.py +293 -0
- distllm/api/routes/federated.py +155 -0
- distllm/api/routes/files.py +243 -0
- distllm/api/routes/fine_tuning.py +381 -0
- distllm/api/routes/gossip.py +72 -0
- distllm/api/routes/health.py +395 -0
- distllm/api/routes/leaderboard.py +377 -0
- distllm/api/routes/marketplace.py +320 -0
- distllm/api/routes/metrics_history.py +415 -0
- distllm/api/routes/model_registry.py +255 -0
- distllm/api/routes/moderation.py +420 -0
- distllm/api/routes/plugins.py +184 -0
- distllm/api/routes/prompts.py +347 -0
- distllm/api/routes/router_admin.py +187 -0
- distllm/api/routes/scheduler.py +168 -0
- distllm/api/routes/tokenize.py +77 -0
- distllm/api/routes/tools.py +89 -0
- distllm/api/routes/webhooks.py +336 -0
- distllm/api/routes/webrtc.py +194 -0
- distllm/api/semantic_cache.py +337 -0
- distllm/api/server.py +2038 -0
- distllm/api/server_config.py +140 -0
- distllm/api/server_coordinator.py +125 -0
- distllm/api/server_errors.py +79 -0
- distllm/api/server_lifespan.py +184 -0
- distllm/api/server_routes_api.py +456 -0
- distllm/api/server_routes_dashboard.py +246 -0
- distllm/api/server_state.py +29 -0
- distllm/api/services/__init__.py +2 -0
- distllm/api/services/chat_service.py +677 -0
- distllm/api/services/completion_service.py +227 -0
- distllm/api/services/embedding_service.py +451 -0
- distllm/api/services/eval_service.py +198 -0
- distllm/api/sso_auth.py +644 -0
- distllm/api/sso_middleware.py +808 -0
- distllm/api/streaming.py +479 -0
- distllm/api/token_estimator.py +86 -0
- distllm/api/tool_calling.py +240 -0
- distllm/api/tracing_middleware.py +579 -0
- distllm/api/validation.py +58 -0
- distllm/api/waf.py +344 -0
- distllm/api/webhooks/__init__.py +17 -0
- distllm/api/webhooks/delivery.py +452 -0
- distllm/api/ws_chat.py +495 -0
- distllm/backends/__init__.py +121 -0
- distllm/backends/config.py +85 -0
- distllm/backends/exllama_backend.py +229 -0
- distllm/backends/llamacpp_backend.py +266 -0
- distllm/backends/mlx_backend.py +199 -0
- distllm/backends/nim_backend.py +838 -0
- distllm/backends/ollama_backend.py +48 -0
- distllm/backends/onnx_backend.py +239 -0
- distllm/backends/paged_attention.py +547 -0
- distllm/backends/paged_attention_quantized.py +319 -0
- distllm/backends/protocol.py +203 -0
- distllm/backends/pytorch_backend.py +157 -0
- distllm/backends/registry.py +428 -0
- distllm/backends/tensorrt_backend.py +333 -0
- distllm/backends/tgi_backend.py +120 -0
- distllm/backends/triton_backend.py +875 -0
- distllm/backends/vllm_backend.py +489 -0
- distllm/backends/webgpu_backend.py +255 -0
- distllm/benchmarks/__init__.py +10 -0
- distllm/benchmarks/evaluation_harness.py +69 -0
- distllm/benchmarks/scaling.py +298 -0
- distllm/cache/__init__.py +19 -0
- distllm/cache/crdt.py +507 -0
- distllm/cache/cross_cluster_prefix_index.py +709 -0
- distllm/cli/__init__.py +17 -0
- distllm/cli/adapters.py +93 -0
- distllm/cli/autopsy.py +287 -0
- distllm/cli/backup.py +118 -0
- distllm/cli/benchmark.py +241 -0
- distllm/cli/cert.py +86 -0
- distllm/cli/chat.py +79 -0
- distllm/cli/chat_stream.py +494 -0
- distllm/cli/client.py +233 -0
- distllm/cli/cluster.py +405 -0
- distllm/cli/completion.py +79 -0
- distllm/cli/compress.py +98 -0
- distllm/cli/config_commands.py +71 -0
- distllm/cli/cost_avoid.py +168 -0
- distllm/cli/daas_commands.py +123 -0
- distllm/cli/defrag_commands.py +128 -0
- distllm/cli/deploy.py +252 -0
- distllm/cli/doctor.py +688 -0
- distllm/cli/draft_commands.py +82 -0
- distllm/cli/error_handler.py +192 -0
- distllm/cli/eval.py +168 -0
- distllm/cli/federate_commands.py +125 -0
- distllm/cli/install.py +198 -0
- distllm/cli/logs.py +85 -0
- distllm/cli/main.py +1795 -0
- distllm/cli/models.py +74 -0
- distllm/cli/notify.py +91 -0
- distllm/cli/observe.py +314 -0
- distllm/cli/onboard.py +671 -0
- distllm/cli/output.py +437 -0
- distllm/cli/profile.py +152 -0
- distllm/cli/profile_presets.py +129 -0
- distllm/cli/prompts.py +139 -0
- distllm/cli/quota.py +227 -0
- distllm/cli/router.py +131 -0
- distllm/cli/run.py +59 -0
- distllm/cli/setup.py +104 -0
- distllm/cli/setup_wizard.py +191 -0
- distllm/cli/status.py +65 -0
- distllm/cli/system_commands.py +255 -0
- distllm/cli/tune.py +488 -0
- distllm/cli/tutorial.py +128 -0
- distllm/cli/verify.py +37 -0
- distllm/cli/webhook.py +85 -0
- distllm/client/__init__.py +38 -0
- distllm/client/client.py +503 -0
- distllm/cloud/__init__.py +53 -0
- distllm/cloud/aws.py +199 -0
- distllm/cloud/azure.py +179 -0
- distllm/cloud/common.py +217 -0
- distllm/cloud/gcp.py +238 -0
- distllm/cloud/spot_orchestrator.py +1642 -0
- distllm/cloud/worker_agent.py +344 -0
- distllm/compliance/__init__.py +10 -0
- distllm/compliance/evidence_pack.py +699 -0
- distllm/config/__init__.py +81 -0
- distllm/config/_application.py +41 -0
- distllm/config/_backends.py +80 -0
- distllm/config/_cache.py +121 -0
- distllm/config/_deployment.py +95 -0
- distllm/config/_generation.py +36 -0
- distllm/config/_hardware.py +42 -0
- distllm/config/_model.py +270 -0
- distllm/config/_network.py +235 -0
- distllm/config/_observability.py +42 -0
- distllm/config/_parallelism.py +172 -0
- distllm/config/_performance.py +47 -0
- distllm/config/loader.py +168 -0
- distllm/config/model_heuristics.py +169 -0
- distllm/config/profiles.py +168 -0
- distllm/config/reference.py +104 -0
- distllm/config/resolver.py +240 -0
- distllm/config/settings.py +463 -0
- distllm/constants.py +117 -0
- distllm/core/__init__.py +111 -0
- distllm/core/_advanced_scheduling_original.py +1892 -0
- distllm/core/ab_test_coordinator.py +178 -0
- distllm/core/activation_profiler.py +172 -0
- distllm/core/adaptive_batching.py +234 -0
- distllm/core/adaptive_cache_compressor.py +168 -0
- distllm/core/adaptive_compression.py +427 -0
- distllm/core/adaptive_compression_hierarchy.py +402 -0
- distllm/core/advanced_scheduling/__init__.py +39 -0
- distllm/core/advanced_scheduling/cost_aware.py +55 -0
- distllm/core/advanced_scheduling/disaggregated.py +352 -0
- distllm/core/advanced_scheduling/energy.py +58 -0
- distllm/core/advanced_scheduling/federated.py +59 -0
- distllm/core/advanced_scheduling/heterogeneous.py +84 -0
- distllm/core/advanced_scheduling/multi_objective.py +372 -0
- distllm/core/advanced_scheduling/policy.py +95 -0
- distllm/core/advanced_scheduling/predictive.py +47 -0
- distllm/core/advanced_scheduling/preemption.py +43 -0
- distllm/core/advanced_scheduling/tiered_store.py +604 -0
- distllm/core/advanced_scheduling/token_bank.py +62 -0
- distllm/core/advanced_scheduling/wan.py +85 -0
- distllm/core/aegis_compliance.py +890 -0
- distllm/core/aether_federated.py +1326 -0
- distllm/core/agentic_router.py +749 -0
- distllm/core/api_key_store.py +478 -0
- distllm/core/arbitrage_engine.py +613 -0
- distllm/core/aria_autoscaler.py +1339 -0
- distllm/core/async_connection_pool.py +187 -0
- distllm/core/async_pipelined_speculative.py +543 -0
- distllm/core/atlas_mesh.py +1437 -0
- distllm/core/auto_discovery.py +351 -0
- distllm/core/auto_partitioner.py +396 -0
- distllm/core/autonomous_healer.py +466 -0
- distllm/core/autoq.py +1545 -0
- distllm/core/backup_manager.py +344 -0
- distllm/core/bargaining_engine.py +450 -0
- distllm/core/batch_builder.py +480 -0
- distllm/core/batch_scheduler.py +1342 -0
- distllm/core/block_affinity_tracker.py +162 -0
- distllm/core/block_eviction_policy.py +363 -0
- distllm/core/cache_aware_router.py +116 -0
- distllm/core/cache_bench.py +239 -0
- distllm/core/cache_coherence.py +139 -0
- distllm/core/cache_doctor.py +220 -0
- distllm/core/cache_eviction.py +181 -0
- distllm/core/cache_index.py +91 -0
- distllm/core/cache_manager.py +703 -0
- distllm/core/cache_migration.py +134 -0
- distllm/core/cache_persistence.py +200 -0
- distllm/core/cache_query_log.py +195 -0
- distllm/core/cache_snapshot.py +178 -0
- distllm/core/cache_template_warmer.py +112 -0
- distllm/core/cache_warming.py +188 -0
- distllm/core/calibration.py +223 -0
- distllm/core/carbon_migration.py +338 -0
- distllm/core/cert_rotation.py +402 -0
- distllm/core/certificate_manager.py +414 -0
- distllm/core/cluster_manager.py +212 -0
- distllm/core/cluster_predictive_prefetcher.py +239 -0
- distllm/core/cluster_registry.py +227 -0
- distllm/core/cluster_state_store.py +442 -0
- distllm/core/compliance_evidence.py +503 -0
- distllm/core/compressed_speculative.py +399 -0
- distllm/core/connection_pool.py +387 -0
- distllm/core/constrained_decoder.py +678 -0
- distllm/core/coordinator.py +1594 -0
- distllm/core/coordinator_cli.py +130 -0
- distllm/core/coordinator_config.py +203 -0
- distllm/core/coordinator_config_wiring.py +316 -0
- distllm/core/coordinator_election.py +246 -0
- distllm/core/coordinator_failover.py +197 -0
- distllm/core/coordinator_health.py +135 -0
- distllm/core/coordinator_lifecycle.py +272 -0
- distllm/core/coordinator_metrics.py +155 -0
- distllm/core/coordinator_multi_model.py +41 -0
- distllm/core/coordinator_request.py +246 -0
- distllm/core/coordinator_state.py +215 -0
- distllm/core/coordinator_subsystem.py +614 -0
- distllm/core/correctness_cert.py +133 -0
- distllm/core/correctness_harness.py +421 -0
- distllm/core/cortex_multimodel.py +939 -0
- distllm/core/cost_comparison.py +250 -0
- distllm/core/cost_dashboard.py +363 -0
- distllm/core/cost_optimizer.py +317 -0
- distllm/core/cost_tracker.py +569 -0
- distllm/core/cross_cloud_router.py +924 -0
- distllm/core/cross_model_prefix_sharing.py +237 -0
- distllm/core/cuda_graph.py +165 -0
- distllm/core/debug.py +84 -0
- distllm/core/device_registry.py +289 -0
- distllm/core/di.py +153 -0
- distllm/core/differential_privacy.py +183 -0
- distllm/core/distributed_distillation.py +402 -0
- distllm/core/distributed_speculative.py +1563 -0
- distllm/core/dp_inference/__init__.py +657 -0
- distllm/core/dp_inference/accounting.py +144 -0
- distllm/core/dp_inference/config.py +139 -0
- distllm/core/dp_inference/mechanisms.py +137 -0
- distllm/core/dp_inference.py +1016 -0
- distllm/core/draft_model_router.py +435 -0
- distllm/core/draft_quality_scorer.py +157 -0
- distllm/core/draft_tree.py +232 -0
- distllm/core/dynamic_memory_budget.py +179 -0
- distllm/core/dynamic_sharder.py +364 -0
- distllm/core/dynamic_speculation.py +131 -0
- distllm/core/evaluation/__init__.py +57 -0
- distllm/core/evaluation/constants.py +86 -0
- distllm/core/evaluation/db.py +185 -0
- distllm/core/evaluation/formatters.py +82 -0
- distllm/core/evaluation/loaders.py +320 -0
- distllm/core/evaluation/models.py +66 -0
- distllm/core/evaluation/report.py +79 -0
- distllm/core/evaluation/runner.py +518 -0
- distllm/core/evaluation/scorers.py +278 -0
- distllm/core/evaluation/worker.py +123 -0
- distllm/core/evaluation_harness.py +1530 -0
- distllm/core/event_bus.py +516 -0
- distllm/core/faas_7b.py +201 -0
- distllm/core/feature_flags.py +245 -0
- distllm/core/federated_finetuner.py +377 -0
- distllm/core/federated_incentives.py +267 -0
- distllm/core/framework_adapters.py +390 -0
- distllm/core/gaia_cache.py +1330 -0
- distllm/core/gossip_cache_bridge.py +202 -0
- distllm/core/gpu_power_manager.py +173 -0
- distllm/core/gpu_profiler.py +59 -0
- distllm/core/gpu_resource_manager.py +338 -0
- distllm/core/graceful_degradation.py +264 -0
- distllm/core/grammar_constrained.py +549 -0
- distllm/core/grammar_constrained_draft.py +91 -0
- distllm/core/grammar_decoder.py +285 -0
- distllm/core/ha_coordinator.py +452 -0
- distllm/core/hardware_plugin_registry.py +126 -0
- distllm/core/hashing.py +29 -0
- distllm/core/health_manager.py +320 -0
- distllm/core/heterogeneous_scheduler.py +663 -0
- distllm/core/hibernation_manager.py +250 -0
- distllm/core/hierarchical_digest.py +168 -0
- distllm/core/hybrid_cache.py +253 -0
- distllm/core/hydra_diffusion.py +177 -0
- distllm/core/inference_engine.py +1072 -0
- distllm/core/intelligent_autoscaler.py +219 -0
- distllm/core/kraken_chaos.py +1407 -0
- distllm/core/kv_backup.py +217 -0
- distllm/core/kv_cache.py +1271 -0
- distllm/core/kv_cache_adaptive_quantizer.py +168 -0
- distllm/core/kv_cache_compressor.py +290 -0
- distllm/core/kv_cache_manager.py +142 -0
- distllm/core/kv_cache_marketplace.py +319 -0
- distllm/core/kv_cache_metrics.py +205 -0
- distllm/core/kv_cache_migration.py +137 -0
- distllm/core/kv_cache_paged.py +78 -0
- distllm/core/kv_cache_replication.py +107 -0
- distllm/core/kv_cache_serialization.py +182 -0
- distllm/core/latency_tracker.py +17 -0
- distllm/core/leaky_bucket_limiter.py +194 -0
- distllm/core/learning_router.py +652 -0
- distllm/core/llamacpp_backend.py +23 -0
- distllm/core/load_balancer.py +286 -0
- distllm/core/media_pipeline.py +387 -0
- distllm/core/memory_defragmenter.py +666 -0
- distllm/core/metering.py +449 -0
- distllm/core/metrics_collector.py +39 -0
- distllm/core/micro_batch_scheduler.py +170 -0
- distllm/core/model_hub_enhanced.py +382 -0
- distllm/core/model_registry.py +9 -0
- distllm/core/model_router.py +930 -0
- distllm/core/model_selector.py +245 -0
- distllm/core/model_sizing.py +334 -0
- distllm/core/model_version_manager.py +414 -0
- distllm/core/moe_orchestrator.py +205 -0
- distllm/core/money.py +53 -0
- distllm/core/monitor.py +141 -0
- distllm/core/mtp_head.py +444 -0
- distllm/core/multi_draft_verifier.py +346 -0
- distllm/core/multi_model_serving.py +624 -0
- distllm/core/multimodal_engine.py +194 -0
- distllm/core/neural_partition_optimizer.py +1398 -0
- distllm/core/node_recovery.py +27 -0
- distllm/core/notification_manager.py +317 -0
- distllm/core/param_update_channel.py +104 -0
- distllm/core/pareto_optimizer.py +227 -0
- distllm/core/performance_alerts.py +198 -0
- distllm/core/performance_baseline.py +252 -0
- distllm/core/persistence.py +947 -0
- distllm/core/persistent_store.py +485 -0
- distllm/core/pipeline_composer.py +211 -0
- distllm/core/pipeline_executor.py +500 -0
- distllm/core/pipeline_orchestrator.py +17 -0
- distllm/core/pipeline_overlap.py +177 -0
- distllm/core/placement.py +442 -0
- distllm/core/plugin_marketplace.py +483 -0
- distllm/core/plugin_sandbox.py +751 -0
- distllm/core/plugin_system.py +601 -0
- distllm/core/plugins/__init__.py +26 -0
- distllm/core/plugins/sandbox.py +38 -0
- distllm/core/predictive_cache.py +29 -0
- distllm/core/predictive_cache_warming.py +323 -0
- distllm/core/predictive_failure.py +163 -0
- distllm/core/preemptible_scheduler.py +343 -0
- distllm/core/preference_learning.py +606 -0
- distllm/core/prefix_cache.py +17 -0
- distllm/core/pricing_providers.py +576 -0
- distllm/core/priority_heap.py +62 -0
- distllm/core/privacy_budget.py +177 -0
- distllm/core/prompt_caching_service.py +228 -0
- distllm/core/prompt_library.py +838 -0
- distllm/core/protocols.py +213 -0
- distllm/core/provider_health.py +234 -0
- distllm/core/provisioning.py +329 -0
- distllm/core/pulse_performance_model.py +1558 -0
- distllm/core/quantization_selector.py +210 -0
- distllm/core/radix_tree_cache.py +306 -0
- distllm/core/rebalancer.py +25 -0
- distllm/core/redis_prompt_cache.py +311 -0
- distllm/core/replication_controller.py +128 -0
- distllm/core/request_auditor.py +215 -0
- distllm/core/request_fingerprinting.py +231 -0
- distllm/core/request_latency.py +211 -0
- distllm/core/request_pipeline.py +810 -0
- distllm/core/request_replay.py +253 -0
- distllm/core/request_tracker.py +184 -0
- distllm/core/resource_manager.py +657 -0
- distllm/core/route_audit.py +175 -0
- distllm/core/routing_extensions.py +604 -0
- distllm/core/schedule_simulator.py +289 -0
- distllm/core/schedule_viz.py +233 -0
- distllm/core/scheduler/__init__.py +34 -0
- distllm/core/scheduler/budget.py +25 -0
- distllm/core/scheduler/budget_computer.py +322 -0
- distllm/core/scheduler/chunked_prefill.py +52 -0
- distllm/core/scheduler/kv_cache_manager.py +313 -0
- distllm/core/scheduler/preemption_manager.py +251 -0
- distllm/core/scheduler/pressure.py +48 -0
- distllm/core/scheduler/sequence.py +179 -0
- distllm/core/secret_manager.py +315 -0
- distllm/core/semantic_cache.py +369 -0
- distllm/core/sentinel_qos.py +833 -0
- distllm/core/shadow_eval_runner.py +194 -0
- distllm/core/shared_layer_pool.py +279 -0
- distllm/core/smart_model_router.py +309 -0
- distllm/core/spec_equivalence.py +140 -0
- distllm/core/spec_verify.py +225 -0
- distllm/core/speculative_adaptor.py +108 -0
- distllm/core/speculative_dashboard.py +134 -0
- distllm/core/speculative_decoder.py +1286 -0
- distllm/core/speculative_profiler.py +132 -0
- distllm/core/split_brain.py +236 -0
- distllm/core/spot_failover.py +292 -0
- distllm/core/spot_forecasting.py +463 -0
- distllm/core/starvation_monitor.py +70 -0
- distllm/core/state_replication.py +261 -0
- distllm/core/stats_collector.py +64 -0
- distllm/core/step_processor.py +115 -0
- distllm/core/straggler_alerts.py +174 -0
- distllm/core/straggler_aware_scheduler.py +255 -0
- distllm/core/straggler_detector.py +29 -0
- distllm/core/streaming_cost.py +286 -0
- distllm/core/streaming_generator.py +228 -0
- distllm/core/structured_output/__init__.py +348 -0
- distllm/core/structured_output/config.py +38 -0
- distllm/core/structured_output/engine.py +367 -0
- distllm/core/structured_output/streaming.py +183 -0
- distllm/core/structured_output/validator.py +199 -0
- distllm/core/subsystem_registry.py +171 -0
- distllm/core/synapse_debugger.py +226 -0
- distllm/core/synth_data_generator.py +122 -0
- distllm/core/telemetry.py +240 -0
- distllm/core/tenant_billing.py +446 -0
- distllm/core/tenant_cost_attribution.py +314 -0
- distllm/core/token_generator.py +368 -0
- distllm/core/token_streaming_buffer.py +147 -0
- distllm/core/topology_aware_lb.py +324 -0
- distllm/core/topology_aware_tiering.py +344 -0
- distllm/core/tree_speculative_decoder.py +405 -0
- distllm/core/types.py +17 -0
- distllm/core/unified_router.py +439 -0
- distllm/core/unified_sla_router.py +242 -0
- distllm/core/usage_meter.py +915 -0
- distllm/core/vectorstore/__init__.py +30 -0
- distllm/core/vectorstore/base.py +106 -0
- distllm/core/vectorstore/chroma_store.py +99 -0
- distllm/core/vectorstore/factory.py +97 -0
- distllm/core/vectorstore/pgvector_store.py +123 -0
- distllm/core/vectorstore/providers/__init__.py +7 -0
- distllm/core/vectorstore/providers/milvus.py +219 -0
- distllm/core/vectorstore/providers/pinecone.py +142 -0
- distllm/core/vectorstore/providers/qdrant.py +176 -0
- distllm/core/vectorstore/providers/weaviate.py +185 -0
- distllm/core/vectorstore/qdrant_store.py +148 -0
- distllm/core/vectorstore/rag.py +151 -0
- distllm/core/vllm_backend.py +21 -0
- distllm/core/vllm_node.py +174 -0
- distllm/core/voyager_multimodal.py +1354 -0
- distllm/core/webgpu_manager.py +382 -0
- distllm/core/webhook_formatters.py +243 -0
- distllm/core/webhook_manager.py +433 -0
- distllm/core/wisp_wasm.py +934 -0
- distllm/core/workload_classifier.py +137 -0
- distllm/dashboard/__init__.py +17 -0
- distllm/dashboard/verification_history.py +192 -0
- distllm/dashboard/ws_handler.py +586 -0
- distllm/dist/__init__.py +85 -0
- distllm/dist/__init__.pyi +132 -0
- distllm/dist/_pipeline_original.py +2916 -0
- distllm/dist/adaptive_speculator.py +384 -0
- distllm/dist/admin_cli.py +293 -0
- distllm/dist/api_docs.py +594 -0
- distllm/dist/async_pipeline.py +578 -0
- distllm/dist/attention.py +1699 -0
- distllm/dist/attention_block_pool.py +634 -0
- distllm/dist/attention_distributed.py +138 -0
- distllm/dist/autoscaler.py +196 -0
- distllm/dist/backends/__init__.py +29 -0
- distllm/dist/backends/backend_profiles.py +357 -0
- distllm/dist/backends/graceful_degradation.py +409 -0
- distllm/dist/backends/health_monitor.py +518 -0
- distllm/dist/backends/llamacpp.py +125 -0
- distllm/dist/backends/ray.py +89 -0
- distllm/dist/backends/vllm.py +177 -0
- distllm/dist/block_pool.py +662 -0
- distllm/dist/block_transfer_service.py +267 -0
- distllm/dist/byzantine.py +2193 -0
- distllm/dist/cache.py +305 -0
- distllm/dist/cache_digest.py +405 -0
- distllm/dist/chaos/__init__.py +52 -0
- distllm/dist/chaos/chaos_orchestrator.py +229 -0
- distllm/dist/chaos/fault_injector.py +1273 -0
- distllm/dist/chunked_prefill.py +136 -0
- distllm/dist/cloud_selector.py +257 -0
- distllm/dist/config.py +70 -0
- distllm/dist/cross_cluster.py +231 -0
- distllm/dist/daas/__init__.py +46 -0
- distllm/dist/daas/load_balancer.py +275 -0
- distllm/dist/daas/marketplace_integration.py +374 -0
- distllm/dist/daas/resource_isolation.py +280 -0
- distllm/dist/daas/tenant_dispatcher.py +252 -0
- distllm/dist/daas/usage_meter.py +360 -0
- distllm/dist/daas_server.py +514 -0
- distllm/dist/disagg/__init__.py +162 -0
- distllm/dist/disagg/pool.py +214 -0
- distllm/dist/disagg/transfer.py +77 -0
- distllm/dist/disagg/transfer_compression.py +793 -0
- distllm/dist/disagg/transfer_engine.py +998 -0
- distllm/dist/discovery.py +166 -0
- distllm/dist/draft_bank.py +344 -0
- distllm/dist/draft_migration.py +382 -0
- distllm/dist/ebpf_transport.py +364 -0
- distllm/dist/edge_cloud.py +544 -0
- distllm/dist/edge_federation.py +426 -0
- distllm/dist/execution_planner.py +148 -0
- distllm/dist/federated_merge.py +560 -0
- distllm/dist/federation.py +1114 -0
- distllm/dist/flash_attention.py +237 -0
- distllm/dist/fsdp.py +295 -0
- distllm/dist/geo.py +361 -0
- distllm/dist/ice_transport.py +2968 -0
- distllm/dist/kv_migration.py +237 -0
- distllm/dist/latency.py +60 -0
- distllm/dist/lifecycle.py +467 -0
- distllm/dist/marketplace.py +891 -0
- distllm/dist/merkle.py +178 -0
- distllm/dist/metrics.py +456 -0
- distllm/dist/model_store.py +107 -0
- distllm/dist/multi_tenant.py +339 -0
- distllm/dist/multimodal.py +392 -0
- distllm/dist/nat.py +934 -0
- distllm/dist/nccl.py +733 -0
- distllm/dist/network.py +126 -0
- distllm/dist/node_client.py +568 -0
- distllm/dist/node_pb2.py +66 -0
- distllm/dist/node_pb2.pyi +236 -0
- distllm/dist/node_pb2_grpc.py +312 -0
- distllm/dist/node_pb2_grpc.pyi +41 -0
- distllm/dist/node_registrar.py +247 -0
- distllm/dist/node_service.py +572 -0
- distllm/dist/observability.py +258 -0
- distllm/dist/otel.py +158 -0
- distllm/dist/p2p/__init__.py +22 -0
- distllm/dist/p2p/discovery.py +132 -0
- distllm/dist/p2p/gossip.py +1339 -0
- distllm/dist/p2p/kademlia_dht.py +1058 -0
- distllm/dist/p2p/load_balancer.py +131 -0
- distllm/dist/p2p/quic_transport.py +700 -0
- distllm/dist/p2p/router.py +607 -0
- distllm/dist/p2p/transport.py +399 -0
- distllm/dist/p2p_model_distributor.py +160 -0
- distllm/dist/paged_attention_kernel.py +281 -0
- distllm/dist/parallel.py +1061 -0
- distllm/dist/parallel_planner.py +586 -0
- distllm/dist/parallel_topology.py +231 -0
- distllm/dist/partition/__init__.py +146 -0
- distllm/dist/partition/adaptive.py +532 -0
- distllm/dist/partition/benchmark_suite.py +479 -0
- distllm/dist/partition/cli.py +189 -0
- distllm/dist/partition/cloud_arbitrage.py +523 -0
- distllm/dist/partition/config.py +97 -0
- distllm/dist/partition/cost_model.py +553 -0
- distllm/dist/partition/learned_cost.py +653 -0
- distllm/dist/partition/mixed_precision_tuner.py +310 -0
- distllm/dist/partition/network_cost_model.py +664 -0
- distllm/dist/partition/optimizer.py +650 -0
- distllm/dist/partition/pareto_optimizer.py +466 -0
- distllm/dist/partition/partitioner.py +363 -0
- distllm/dist/partition/persistence.py +392 -0
- distllm/dist/partition/plugins.py +270 -0
- distllm/dist/partition/profiles.py +659 -0
- distllm/dist/partition/quant_bench.py +373 -0
- distllm/dist/partition/quant_calibrate.py +229 -0
- distllm/dist/partition/quant_coordinator.py +260 -0
- distllm/dist/partition/quant_cost.py +293 -0
- distllm/dist/partition/quant_partition.py +432 -0
- distllm/dist/partition/quant_report.py +301 -0
- distllm/dist/partition/quantization_metrics.py +355 -0
- distllm/dist/partition/quantization_search.py +290 -0
- distllm/dist/partition/quantization_tuner.py +1383 -0
- distllm/dist/partition/topology.py +218 -0
- distllm/dist/partition/validator.py +405 -0
- distllm/dist/partition/visualizer.py +275 -0
- distllm/dist/pipeline/__init__.py +71 -0
- distllm/dist/pipeline/bandwidth_controller.py +780 -0
- distllm/dist/pipeline/compression_negotiation.py +771 -0
- distllm/dist/pipeline/context.py +80 -0
- distllm/dist/pipeline/continuous_batching.py +435 -0
- distllm/dist/pipeline/disagg_orchestrator.py +284 -0
- distllm/dist/pipeline/orchestrator.py +727 -0
- distllm/dist/pipeline/pipeline_reconfig.py +1395 -0
- distllm/dist/pipeline/profiler.py +84 -0
- distllm/dist/pipeline/serialization.py +176 -0
- distllm/dist/pipeline/simulator.py +172 -0
- distllm/dist/pipeline/strategy.py +193 -0
- distllm/dist/pipeline/token_accumulator.py +211 -0
- distllm/dist/pipeline/transport.py +230 -0
- distllm/dist/pipeline/wan_disagg_orchestrator.py +564 -0
- distllm/dist/power_cap.py +358 -0
- distllm/dist/predictive_cache.py +399 -0
- distllm/dist/preemption.py +198 -0
- distllm/dist/prefix_cache.py +461 -0
- distllm/dist/prefix_clustering.py +305 -0
- distllm/dist/privacy/__init__.py +411 -0
- distllm/dist/privacy/privacy_accountant.py +344 -0
- distllm/dist/privacy.py +240 -0
- distllm/dist/provisioning.py +338 -0
- distllm/dist/quality.py +118 -0
- distllm/dist/quic_transport.py +335 -0
- distllm/dist/quota_enforcer.py +265 -0
- distllm/dist/rebalancer.py +234 -0
- distllm/dist/recovery.py +827 -0
- distllm/dist/recovery_drill.py +281 -0
- distllm/dist/redis_cache.py +265 -0
- distllm/dist/redundant.py +308 -0
- distllm/dist/redundant_executor.py +1536 -0
- distllm/dist/reputation.py +231 -0
- distllm/dist/router.py +382 -0
- distllm/dist/routing/__init__.py +19 -0
- distllm/dist/routing/arbitrage_engine.py +575 -0
- distllm/dist/routing/composite.py +179 -0
- distllm/dist/routing/consistent_hash.py +144 -0
- distllm/dist/routing/latency_aware.py +116 -0
- distllm/dist/routing/load_aware.py +114 -0
- distllm/dist/scheduling/__init__.py +38 -0
- distllm/dist/scheduling/batcher.py +95 -0
- distllm/dist/scheduling/classifier.py +153 -0
- distllm/dist/scheduling/deadline_scheduler.py +958 -0
- distllm/dist/scheduling/iteration.py +404 -0
- distllm/dist/scheduling/profiler.py +57 -0
- distllm/dist/simulation/__init__.py +54 -0
- distllm/dist/simulation/chaos_simulator.py +901 -0
- distllm/dist/simulation/cluster_simulator.py +667 -0
- distllm/dist/simulation/cost_aware_provisioning.py +370 -0
- distllm/dist/simulation/digital_twin.py +617 -0
- distllm/dist/simulation/topology_optimizer.py +747 -0
- distllm/dist/simulation/what_if.py +492 -0
- distllm/dist/speculative/__init__.py +47 -0
- distllm/dist/speculative/adaptive_spec.py +291 -0
- distllm/dist/speculative/draft_cache.py +246 -0
- distllm/dist/speculative/draft_orchestrator.py +634 -0
- distllm/dist/speculative/draft_registry.py +439 -0
- distllm/dist/speculative/multi_draft.py +304 -0
- distllm/dist/speculative/online_sisd.py +523 -0
- distllm/dist/straggler.py +819 -0
- distllm/dist/streaming_kv_transfer.py +174 -0
- distllm/dist/structured_output/__init__.py +7 -0
- distllm/dist/structured_output/engine.py +144 -0
- distllm/dist/tensor_pool.py +120 -0
- distllm/dist/tgi_compat.py +254 -0
- distllm/dist/topology.py +141 -0
- distllm/dist/topology_consensus.py +1961 -0
- distllm/dist/topology_dynamic.py +270 -0
- distllm/dist/tp_inprocess.py +214 -0
- distllm/dist/tp_launcher.py +310 -0
- distllm/dist/tracing.py +325 -0
- distllm/dist/transport.py +493 -0
- distllm/dist/wan_speculative.py +295 -0
- distllm/dist/wandb_integration.py +252 -0
- distllm/dist/webrtc.py +846 -0
- distllm/dist/wide_area.py +515 -0
- distllm/dist/worker.py +635 -0
- distllm/dist/zero_copy.py +307 -0
- distllm/errors/__init__.py +105 -0
- distllm/errors/policies.py +106 -0
- distllm/errors/retry.py +168 -0
- distllm/errors/types.py +721 -0
- distllm/health/__init__.py +6 -0
- distllm/health/failover.py +81 -0
- distllm/health/prober.py +31 -0
- distllm/health/service.py +173 -0
- distllm/health/state.py +76 -0
- distllm/integrations/__init__.py +72 -0
- distllm/integrations/_common/__init__.py +7 -0
- distllm/integrations/_common/base_tool_provider.py +189 -0
- distllm/integrations/_common/cost_tracker.py +144 -0
- distllm/integrations/_common/model_router.py +148 -0
- distllm/integrations/ci/__init__.py +24 -0
- distllm/integrations/ci/_common.py +92 -0
- distllm/integrations/ci/gitlab.py +345 -0
- distllm/integrations/ci/jenkins.py +273 -0
- distllm/integrations/mlflow_tracking.py +372 -0
- distllm/integrations/spark_connector.py +909 -0
- distllm/integrations/wandb/__init__.py +81 -0
- distllm/models/__init__.py +40 -0
- distllm/models/_trust_remote.py +48 -0
- distllm/models/adapter.py +768 -0
- distllm/models/adapter_router.py +198 -0
- distllm/models/cache.py +172 -0
- distllm/models/model_hub.py +650 -0
- distllm/models/partition_planner.py +384 -0
- distllm/models/partitioner.py +784 -0
- distllm/models/rope_scaling.py +119 -0
- distllm/models/safetensors_index.py +179 -0
- distllm/observability/__init__.py +107 -0
- distllm/observability/alerting_config.py +881 -0
- distllm/observability/capacity_models.py +398 -0
- distllm/observability/capacity_planning.py +2870 -0
- distllm/observability/exporter.py +228 -0
- distllm/observability/exporter_datadog.py +1362 -0
- distllm/observability/health_config.py +1282 -0
- distllm/observability/incident_response.py +779 -0
- distllm/observability/logging.py +211 -0
- distllm/observability/logging_config.py +864 -0
- distllm/observability/loki_sink.py +178 -0
- distllm/observability/metrics.py +111 -0
- distllm/observability/metrics_completeness.py +393 -0
- distllm/observability/otel_logging.py +290 -0
- distllm/observability/self_healing_config.py +1308 -0
- distllm/observability/slo_config.py +1520 -0
- distllm/observability/spans.py +235 -0
- distllm/observability/tracing.py +268 -0
- distllm/observability/tracing_config.py +676 -0
- distllm/observability/wandb_monitor.py +66 -0
- distllm/plugins/__init__.py +14 -0
- distllm/plugins/airflow.py +110 -0
- distllm/plugins/auth_plugin.py +576 -0
- distllm/plugins/builtin.py +376 -0
- distllm/plugins/cache_plugin.py +576 -0
- distllm/plugins/health_plugin.py +396 -0
- distllm/plugins/kubeflow.py +84 -0
- distllm/plugins/mlflow_plugin.py +71 -0
- distllm/plugins/registry.py +135 -0
- distllm/prompts/__init__.py +39 -0
- distllm/prompts/analysis.py +103 -0
- distllm/prompts/code.py +172 -0
- distllm/prompts/creative.py +115 -0
- distllm/prompts/education.py +91 -0
- distllm/prompts/engine.py +119 -0
- distllm/prompts/exchange.py +705 -0
- distllm/prompts/language.py +81 -0
- distllm/prompts/library.py +1198 -0
- distllm/prompts/management.py +35 -0
- distllm/prompts/productivity.py +86 -0
- distllm/prompts/professional.py +111 -0
- distllm/prompts/prompt_def.py +33 -0
- distllm/prompts/reasoning.py +108 -0
- distllm/prompts/specialized.py +140 -0
- distllm/prompts/templates.py +128 -0
- distllm/prompts/writing.py +140 -0
- distllm/py.typed +0 -0
- distllm/sdk/__init__.py +82 -0
- distllm/sdk/circuit_breaker.py +139 -0
- distllm/sdk/client.py +1191 -0
- distllm/sdk/observability.py +42 -0
- distllm/sdk/streaming.py +228 -0
- distllm/sdk/transport.py +224 -0
- distllm/sdk/types.py +288 -0
- distllm/security/__init__.py +63 -0
- distllm/security/content_moderation/__init__.py +44 -0
- distllm/security/content_moderation/base.py +405 -0
- distllm/security/content_moderation/jailbreak.py +192 -0
- distllm/security/content_moderation/pii.py +262 -0
- distllm/security/content_moderation/pipeline.py +272 -0
- distllm/security/content_moderation/topics.py +153 -0
- distllm/security/content_moderation/toxicity.py +131 -0
- distllm/security/e2e.py +437 -0
- distllm/security/edge_attestation.py +381 -0
- distllm/security/log_redaction.py +244 -0
- distllm/security/quantum_safe_tls.py +314 -0
- distllm/security/spiffe.py +347 -0
- distllm/security/tee.py +461 -0
- distllm/security/utils.py +139 -0
- distllm/security/watermark.py +779 -0
- distllm/terraform/__init__.py +260 -0
- distllm/ui/__init__.py +7 -0
- distllm/ui/app.py +98 -0
- distllm/utils/__init__.py +0 -0
- distllm/utils/gbnf_grammar.py +99 -0
- distllm/utils/lazy_imports.py +90 -0
- distllm/utils/scheduling.py +38 -0
- distllm/verification/__init__.py +49 -0
- distllm/verification/comparator.py +263 -0
- distllm/verification/hash_registry.py +220 -0
- distllm/verification/report.py +240 -0
- distllm/verification/runner.py +822 -0
- distllm/verification/test_verify.py +205 -0
- distributed_llm-0.4.1.dist-info/METADATA +315 -0
- distributed_llm-0.4.1.dist-info/RECORD +806 -0
- distributed_llm-0.4.1.dist-info/WHEEL +5 -0
- distributed_llm-0.4.1.dist-info/entry_points.txt +5 -0
- distributed_llm-0.4.1.dist-info/licenses/LICENSE +97 -0
- distributed_llm-0.4.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,338 @@
|
|
|
1
|
+
"""Carbon-Aware Load Migration.
|
|
2
|
+
|
|
3
|
+
Monitors carbon intensity in active regions and triggers migration
|
|
4
|
+
to cleaner regions when carbon spikes above a threshold. Uses KV cache
|
|
5
|
+
shipping from cross_cluster.py for live migration.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
import threading
|
|
12
|
+
import time
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
from typing import Any, Callable
|
|
15
|
+
|
|
16
|
+
from loguru import logger
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass
|
|
20
|
+
class MigrationEvent:
|
|
21
|
+
"""A carbon-triggered migration event."""
|
|
22
|
+
from_region: str
|
|
23
|
+
to_region: str
|
|
24
|
+
from_carbon: float
|
|
25
|
+
to_carbon: float
|
|
26
|
+
carbon_saved: float
|
|
27
|
+
request_ids: list[str] = field(default_factory=list)
|
|
28
|
+
timestamp: float = field(default_factory=time.time)
|
|
29
|
+
success: bool = False
|
|
30
|
+
duration_ms: float = 0.0
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class CarbonIntensityClient:
|
|
34
|
+
"""Client for real-time carbon intensity data via electricityMap API.
|
|
35
|
+
|
|
36
|
+
Uses the https://api.electricitymap.org endpoint. Set
|
|
37
|
+
``DISTLLM_CARBON_API_KEY`` to your electricityMap API token.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
def __init__(self, api_key: str | None = None):
|
|
41
|
+
self._api_key = api_key or os.environ.get("DISTLLM_CARBON_API_KEY", "")
|
|
42
|
+
self._base_url = "https://api.electricitymap.org/v3"
|
|
43
|
+
self._cache: dict[str, tuple[float, float]] = {} # zone -> (gco2/kWh, timestamp)
|
|
44
|
+
self._cache_ttl: float = 600.0 # 10 min
|
|
45
|
+
self._lock = threading.Lock()
|
|
46
|
+
|
|
47
|
+
def get_intensity(self, zone: str) -> float:
|
|
48
|
+
"""Get current carbon intensity for a zone (gCO2/kWh).
|
|
49
|
+
|
|
50
|
+
Uses cached data when available (< 10 min old).
|
|
51
|
+
Returns 0.0 on failure.
|
|
52
|
+
"""
|
|
53
|
+
now = time.time()
|
|
54
|
+
with self._lock:
|
|
55
|
+
cached = self._cache.get(zone)
|
|
56
|
+
if cached and (now - cached[1]) < self._cache_ttl:
|
|
57
|
+
return cached[0]
|
|
58
|
+
|
|
59
|
+
if not self._api_key:
|
|
60
|
+
# Fallback to static regional data
|
|
61
|
+
from distllm.core.cross_cloud_router import _REGIONAL_CARBON_INTENSITY
|
|
62
|
+
val = _REGIONAL_CARBON_INTENSITY.get(zone, 0.0)
|
|
63
|
+
with self._lock:
|
|
64
|
+
self._cache[zone] = (val, now)
|
|
65
|
+
return val
|
|
66
|
+
|
|
67
|
+
try:
|
|
68
|
+
import httpx
|
|
69
|
+
resp = httpx.get(
|
|
70
|
+
f"{self._base_url}/carbon-intensity/latest",
|
|
71
|
+
params={"zone": zone},
|
|
72
|
+
headers={"auth-token": self._api_key},
|
|
73
|
+
timeout=10.0,
|
|
74
|
+
)
|
|
75
|
+
if resp.status_code == 200:
|
|
76
|
+
data = resp.json()
|
|
77
|
+
val = float(data.get("carbonIntensity", 0.0))
|
|
78
|
+
with self._lock:
|
|
79
|
+
self._cache[zone] = (val, now)
|
|
80
|
+
return val
|
|
81
|
+
except Exception as e:
|
|
82
|
+
logger.debug(f"Carbon API request failed for {zone}: {e}")
|
|
83
|
+
|
|
84
|
+
return 0.0
|
|
85
|
+
|
|
86
|
+
def get_all_intensities(self, zones: list[str]) -> dict[str, float]:
|
|
87
|
+
"""Fetch intensities for multiple zones."""
|
|
88
|
+
return {z: self.get_intensity(z) for z in zones}
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class CarbonMigrationEngine:
|
|
92
|
+
"""Monitors carbon intensity and triggers live migration to cleaner regions.
|
|
93
|
+
|
|
94
|
+
Uses the electricityMap API when configured, with static fallback.
|
|
95
|
+
|
|
96
|
+
Usage::
|
|
97
|
+
|
|
98
|
+
engine = CarbonMigrationEngine(threshold=400)
|
|
99
|
+
engine.set_active_region("us-east-1", ["req-1", "req-2"])
|
|
100
|
+
engine.start(migration_callback=my_migrate_fn)
|
|
101
|
+
"""
|
|
102
|
+
|
|
103
|
+
def __init__(
|
|
104
|
+
self,
|
|
105
|
+
carbon_provider: Any = None,
|
|
106
|
+
threshold: float = 400.0,
|
|
107
|
+
check_interval_s: float = 300.0,
|
|
108
|
+
migration_cooldown_s: float = 900.0,
|
|
109
|
+
min_savings_pct: float = 20.0,
|
|
110
|
+
):
|
|
111
|
+
if carbon_provider is None:
|
|
112
|
+
carbon_provider = CarbonIntensityClient()
|
|
113
|
+
self._carbon_provider = carbon_provider
|
|
114
|
+
self._threshold = threshold
|
|
115
|
+
self._check_interval = check_interval_s
|
|
116
|
+
self._cooldown = migration_cooldown_s
|
|
117
|
+
self._min_savings_pct = min_savings_pct / 100.0
|
|
118
|
+
self._active_region: str = ""
|
|
119
|
+
self._active_requests: list[str] = []
|
|
120
|
+
self._migration_callback: Callable[[str, str, list[str]], bool] | None = None
|
|
121
|
+
self._running = False
|
|
122
|
+
self._thread: threading.Thread | None = None
|
|
123
|
+
self._events: list[MigrationEvent] = []
|
|
124
|
+
self._last_migration: float = 0.0
|
|
125
|
+
self._lock = threading.Lock()
|
|
126
|
+
|
|
127
|
+
def set_active_region(self, region: str, request_ids: list[str] | None = None) -> None:
|
|
128
|
+
"""Set the currently active region and tracked requests."""
|
|
129
|
+
with self._lock:
|
|
130
|
+
self._active_region = region
|
|
131
|
+
if request_ids is not None:
|
|
132
|
+
self._active_requests = request_ids
|
|
133
|
+
|
|
134
|
+
def set_migration_callback(self, callback: Callable[[str, str, list[str]], bool]) -> None:
|
|
135
|
+
"""Set the migration function.
|
|
136
|
+
|
|
137
|
+
The callback receives (from_region, to_region, request_ids) and
|
|
138
|
+
returns True if migration succeeded.
|
|
139
|
+
"""
|
|
140
|
+
self._migration_callback = callback
|
|
141
|
+
|
|
142
|
+
def start(self) -> None:
|
|
143
|
+
"""Start the carbon monitoring loop."""
|
|
144
|
+
if self._running:
|
|
145
|
+
return
|
|
146
|
+
self._running = True
|
|
147
|
+
self._thread = threading.Thread(
|
|
148
|
+
target=self._monitor_loop,
|
|
149
|
+
daemon=True,
|
|
150
|
+
name="carbon-migration",
|
|
151
|
+
)
|
|
152
|
+
self._thread.start()
|
|
153
|
+
logger.info(f"Carbon migration engine started (threshold={self._threshold} gCO2/kWh)")
|
|
154
|
+
|
|
155
|
+
def stop(self) -> None:
|
|
156
|
+
"""Stop the monitoring loop."""
|
|
157
|
+
self._running = False
|
|
158
|
+
if self._thread:
|
|
159
|
+
self._thread.join(timeout=5)
|
|
160
|
+
self._thread = None
|
|
161
|
+
|
|
162
|
+
def _monitor_loop(self) -> None:
|
|
163
|
+
while self._running:
|
|
164
|
+
time.sleep(self._check_interval)
|
|
165
|
+
try:
|
|
166
|
+
self._check_and_migrate()
|
|
167
|
+
except Exception as e:
|
|
168
|
+
logger.debug(f"Carbon migration check failed: {e}")
|
|
169
|
+
|
|
170
|
+
def _check_and_migrate(self) -> None:
|
|
171
|
+
"""Check if current region exceeds carbon threshold and migrate if needed."""
|
|
172
|
+
if not self._carbon_provider or not self._active_region:
|
|
173
|
+
return
|
|
174
|
+
# Check cooldown
|
|
175
|
+
if time.time() - self._last_migration < self._cooldown:
|
|
176
|
+
return
|
|
177
|
+
# Get current carbon intensity via real-time API (with static fallback)
|
|
178
|
+
current_val = self._carbon_provider.get_intensity(self._active_region) if hasattr(self._carbon_provider, 'get_intensity') else 0.0
|
|
179
|
+
if isinstance(current_val, (int, float)):
|
|
180
|
+
current_gco2 = current_val
|
|
181
|
+
else:
|
|
182
|
+
current_gco2 = current_val.gco2_per_kwh if hasattr(current_val, 'gco2_per_kwh') else 0.0
|
|
183
|
+
|
|
184
|
+
if current_gco2 <= self._threshold:
|
|
185
|
+
return
|
|
186
|
+
|
|
187
|
+
# Find a cleaner region using carbon provider + fallback
|
|
188
|
+
regions_to_check = getattr(self._carbon_provider, 'get_all_intensities', None)
|
|
189
|
+
if regions_to_check:
|
|
190
|
+
from distllm.core.cross_cloud_router import _REGIONAL_CARBON_INTENSITY
|
|
191
|
+
intensities = regions_to_check(list(_REGIONAL_CARBON_INTENSITY.keys()))
|
|
192
|
+
else:
|
|
193
|
+
from distllm.core.cross_cloud_router import _REGIONAL_CARBON_INTENSITY
|
|
194
|
+
intensities = dict(_REGIONAL_CARBON_INTENSITY)
|
|
195
|
+
|
|
196
|
+
best_region = None
|
|
197
|
+
best_carbon = current_gco2
|
|
198
|
+
for region, carbon in intensities.items():
|
|
199
|
+
if carbon > 0 and carbon < best_carbon * (1.0 - self._min_savings_pct):
|
|
200
|
+
best_region = region
|
|
201
|
+
best_carbon = carbon
|
|
202
|
+
if not best_region:
|
|
203
|
+
return
|
|
204
|
+
# Trigger migration
|
|
205
|
+
logger.warning(
|
|
206
|
+
f"Carbon spike in {self._active_region}: {current_gco2:.0f} gCO2/kWh "
|
|
207
|
+
f"(threshold: {self._threshold}). Migrating to {best_region} ({best_carbon:.0f})"
|
|
208
|
+
)
|
|
209
|
+
event = MigrationEvent(
|
|
210
|
+
from_region=self._active_region,
|
|
211
|
+
to_region=best_region,
|
|
212
|
+
from_carbon=current_gco2,
|
|
213
|
+
to_carbon=best_carbon,
|
|
214
|
+
carbon_saved=current_gco2 - best_carbon,
|
|
215
|
+
request_ids=list(self._active_requests),
|
|
216
|
+
)
|
|
217
|
+
if self._migration_callback:
|
|
218
|
+
try:
|
|
219
|
+
success = self._migration_callback(
|
|
220
|
+
self._active_region, best_region, self._active_requests
|
|
221
|
+
)
|
|
222
|
+
event.success = success
|
|
223
|
+
if success:
|
|
224
|
+
self._active_region = best_region
|
|
225
|
+
self._last_migration = time.time()
|
|
226
|
+
except Exception as e:
|
|
227
|
+
logger.error(f"Migration callback failed: {e}")
|
|
228
|
+
event.success = False
|
|
229
|
+
with self._lock:
|
|
230
|
+
self._events.append(event)
|
|
231
|
+
|
|
232
|
+
def get_events(self, limit: int = 50) -> list[dict[str, Any]]:
|
|
233
|
+
"""Get recent migration events."""
|
|
234
|
+
with self._lock:
|
|
235
|
+
return [
|
|
236
|
+
{
|
|
237
|
+
"from": e.from_region,
|
|
238
|
+
"to": e.to_region,
|
|
239
|
+
"from_carbon": e.from_carbon,
|
|
240
|
+
"to_carbon": e.to_carbon,
|
|
241
|
+
"carbon_saved": e.carbon_saved,
|
|
242
|
+
"success": e.success,
|
|
243
|
+
"timestamp": e.timestamp,
|
|
244
|
+
}
|
|
245
|
+
for e in self._events[-limit:]
|
|
246
|
+
]
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
class SLATier:
|
|
250
|
+
"""An SLA tier with routing constraints."""
|
|
251
|
+
|
|
252
|
+
def __init__(
|
|
253
|
+
self,
|
|
254
|
+
name: str,
|
|
255
|
+
prefer_spot: bool = True,
|
|
256
|
+
allow_on_demand: bool = True,
|
|
257
|
+
max_latency_ms: float = 200.0,
|
|
258
|
+
max_carbon_intensity: float = float("inf"),
|
|
259
|
+
max_price_per_hour: float = float("inf"),
|
|
260
|
+
carbon_weight: float = 0.3,
|
|
261
|
+
):
|
|
262
|
+
self.name = name
|
|
263
|
+
self.prefer_spot = prefer_spot
|
|
264
|
+
self.allow_on_demand = allow_on_demand
|
|
265
|
+
self.max_latency_ms = max_latency_ms
|
|
266
|
+
self.max_carbon_intensity = max_carbon_intensity
|
|
267
|
+
self.max_price_per_hour = max_price_per_hour
|
|
268
|
+
self.carbon_weight = carbon_weight
|
|
269
|
+
|
|
270
|
+
def to_routing_kwargs(self) -> dict[str, Any]:
|
|
271
|
+
"""Convert to kwargs for CrossCloudRouter.select_provider_carbon_aware()."""
|
|
272
|
+
return {
|
|
273
|
+
"prefer_spot": self.prefer_spot,
|
|
274
|
+
"max_latency_ms": self.max_latency_ms,
|
|
275
|
+
"max_price": self.max_price_per_hour,
|
|
276
|
+
"max_carbon_intensity": self.max_carbon_intensity,
|
|
277
|
+
"carbon_weight": self.carbon_weight,
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
# Predefined SLA tiers
|
|
282
|
+
SLA_TIERS: dict[str, SLATier] = {
|
|
283
|
+
"critical": SLATier(
|
|
284
|
+
name="Critical",
|
|
285
|
+
prefer_spot=False,
|
|
286
|
+
allow_on_demand=True,
|
|
287
|
+
max_latency_ms=50.0,
|
|
288
|
+
max_carbon_intensity=float("inf"),
|
|
289
|
+
max_price_per_hour=float("inf"),
|
|
290
|
+
carbon_weight=0.0,
|
|
291
|
+
),
|
|
292
|
+
"standard": SLATier(
|
|
293
|
+
name="Standard",
|
|
294
|
+
prefer_spot=True,
|
|
295
|
+
allow_on_demand=True,
|
|
296
|
+
max_latency_ms=200.0,
|
|
297
|
+
max_carbon_intensity=500.0,
|
|
298
|
+
max_price_per_hour=50.0,
|
|
299
|
+
carbon_weight=0.15,
|
|
300
|
+
),
|
|
301
|
+
"batch": SLATier(
|
|
302
|
+
name="Batch",
|
|
303
|
+
prefer_spot=True,
|
|
304
|
+
allow_on_demand=False,
|
|
305
|
+
max_latency_ms=5000.0,
|
|
306
|
+
max_carbon_intensity=300.0,
|
|
307
|
+
max_price_per_hour=10.0,
|
|
308
|
+
carbon_weight=0.5,
|
|
309
|
+
),
|
|
310
|
+
"green": SLATier(
|
|
311
|
+
name="Green",
|
|
312
|
+
prefer_spot=True,
|
|
313
|
+
allow_on_demand=False,
|
|
314
|
+
max_latency_ms=5000.0,
|
|
315
|
+
max_carbon_intensity=200.0,
|
|
316
|
+
max_price_per_hour=float("inf"),
|
|
317
|
+
carbon_weight=1.0,
|
|
318
|
+
),
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def get_sla_tier(name: str) -> SLATier:
|
|
323
|
+
"""Get a predefined SLA tier by name."""
|
|
324
|
+
return SLA_TIERS.get(name.lower(), SLA_TIERS["standard"])
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def list_sla_tiers() -> list[dict[str, Any]]:
|
|
328
|
+
"""List all predefined SLA tiers."""
|
|
329
|
+
return [
|
|
330
|
+
{
|
|
331
|
+
"name": tier.name,
|
|
332
|
+
"prefer_spot": tier.prefer_spot,
|
|
333
|
+
"max_latency_ms": tier.max_latency_ms,
|
|
334
|
+
"max_carbon_intensity": tier.max_carbon_intensity,
|
|
335
|
+
"carbon_weight": tier.carbon_weight,
|
|
336
|
+
}
|
|
337
|
+
for tier in SLA_TIERS.values()
|
|
338
|
+
]
|