terradev-cli 3.7.6__tar.gz → 3.7.7__tar.gz

{terradev_cli-3.7.6 → terradev_cli-3.7.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: terradev-cli
-Version: 3.7.6
+Version: 3.7.7
 Summary: NUMA-aware GPU provisioning and orchestration for stateless MoE workloads of all sizes
 Author-email: Terradev Team <team@terradev.com>
 Maintainer-email: Terradev Team <team@terradev.com>
@@ -91,7 +91,7 @@ Requires-Dist: redis>=5.0.0; extra == "enterprise"
 Requires-Dist: PyJWT>=2.8.0; extra == "enterprise"
 Dynamic: license-file
 
-# Terradev CLI v3.7.5
+# Terradev CLI v3.7.7
 
 **NUMA-aware GPU provisioning and orchestration for stateless MoE workloads of all sizes**
 
@@ -99,6 +99,46 @@ Dynamic: license-file
 
 Terradev is a cross-cloud compute-provisioning CLI that compresses + stages datasets, provisions optimal instances + nodes, and deploys **3-5x faster** than sequential provisioning.
 
+## What's New in v3.7.7
+
+**Complete SGLang Optimization Stack**
+
+Revolutionary workload-specific auto-optimization for SGLang serving with 7 workload types:
+
+### 🚀 SGLang Workload Optimizations
+- **Agentic/Multi-turn Chat**: LPM + RadixAttention + cache-aware routing (75-90% cache hit rate)
+- **High-Throughput Batch**: FCFS + CUDA graphs + FP8 quantization (maximum tokens/sec)
+- **Low-Latency/Real-Time**: EAGLE3 + Spec V2 + capped concurrency (30-50% TTFT improvement)
+- **MoE Models**: DeepEP auto + TBO/SBO + EPLB + redundant experts (up to 2x throughput)
+- **PD Disaggregated**: Separate prefill/decode configurations with production optimizations
+- **Structured Output/RAG**: xGrammar + FSM optimization (10x faster structured output)
+- **Hardware-Specific**: H100/H200, H20, GB200, AMD MI300X optimizations
+
+### 🎯 Auto-Apply Decision Tree
+```bash
+# Auto-optimize any model for workload type
+terradev sglang optimize deepseek-ai/DeepSeek-V3
+
+# Detect workload from description
+terradev sglang detect meta-llama/Llama-2-7b-hf --user-description "Real-time API"
+
+# Multi-replica cache-aware routing
+terradev sglang router meta-llama/Llama-2-7b-hf --dp-size 8
+```
+
+### 📊 Performance Gains
+- **Agentic Chat**: 1.9x throughput with multi-replica, 95-98% GPU utilization
+- **Batch Inference**: Maximum tokens/second with pre-compiled CUDA graphs
+- **Low Latency**: 30-50% TTFT improvement, 20-40% TPOT improvement
+- **MoE Models**: Up to 2x throughput with Two-Batch Overlap
+- **Cache-Aware Routing**: 3.8x higher cache hit rate
+
+### 🔧 Hardware Optimization
+- **H100/H200**: FlashInfer + FP8 KV cache optimization
+- **H20**: FA3 + MoE→QKV→FP8 stacking + swapAB runner
+- **GB200 NVL72**: Rack-scale TP + NUMA-aware placement
+- **AMD MI300X**: Triton backend + ROCm EPLB tuning
+
 ## What's New in v3.7.3
 
 Performance and scalability improvements for enterprise deployments.

{terradev_cli-3.7.6/terradev_cli → terradev_cli-3.7.7}/README.md

@@ -1,4 +1,4 @@
-# Terradev CLI v3.7.5
+# Terradev CLI v3.7.7
 
 **NUMA-aware GPU provisioning and orchestration for stateless MoE workloads of all sizes**
 
@@ -6,6 +6,46 @@
 
 Terradev is a cross-cloud compute-provisioning CLI that compresses + stages datasets, provisions optimal instances + nodes, and deploys **3-5x faster** than sequential provisioning.
 
+## What's New in v3.7.7
+
+**Complete SGLang Optimization Stack**
+
+Revolutionary workload-specific auto-optimization for SGLang serving with 7 workload types:
+
+### 🚀 SGLang Workload Optimizations
+- **Agentic/Multi-turn Chat**: LPM + RadixAttention + cache-aware routing (75-90% cache hit rate)
+- **High-Throughput Batch**: FCFS + CUDA graphs + FP8 quantization (maximum tokens/sec)
+- **Low-Latency/Real-Time**: EAGLE3 + Spec V2 + capped concurrency (30-50% TTFT improvement)
+- **MoE Models**: DeepEP auto + TBO/SBO + EPLB + redundant experts (up to 2x throughput)
+- **PD Disaggregated**: Separate prefill/decode configurations with production optimizations
+- **Structured Output/RAG**: xGrammar + FSM optimization (10x faster structured output)
+- **Hardware-Specific**: H100/H200, H20, GB200, AMD MI300X optimizations
+
+### 🎯 Auto-Apply Decision Tree
+```bash
+# Auto-optimize any model for workload type
+terradev sglang optimize deepseek-ai/DeepSeek-V3
+
+# Detect workload from description
+terradev sglang detect meta-llama/Llama-2-7b-hf --user-description "Real-time API"
+
+# Multi-replica cache-aware routing
+terradev sglang router meta-llama/Llama-2-7b-hf --dp-size 8
+```
+
+### 📊 Performance Gains
+- **Agentic Chat**: 1.9x throughput with multi-replica, 95-98% GPU utilization
+- **Batch Inference**: Maximum tokens/second with pre-compiled CUDA graphs
+- **Low Latency**: 30-50% TTFT improvement, 20-40% TPOT improvement
+- **MoE Models**: Up to 2x throughput with Two-Batch Overlap
+- **Cache-Aware Routing**: 3.8x higher cache hit rate
+
+### 🔧 Hardware Optimization
+- **H100/H200**: FlashInfer + FP8 KV cache optimization
+- **H20**: FA3 + MoE→QKV→FP8 stacking + swapAB runner
+- **GB200 NVL72**: Rack-scale TP + NUMA-aware placement
+- **AMD MI300X**: Triton backend + ROCm EPLB tuning
+
 ## What's New in v3.7.3
 
 Performance and scalability improvements for enterprise deployments.

{terradev_cli-3.7.6 → terradev_cli-3.7.7}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "terradev-cli"
-version = "3.7.6"
+version = "3.7.7"
 description = "NUMA-aware GPU provisioning and orchestration for stateless MoE workloads of all sizes"
 readme = "README.md"
 license = {text = "BUSL-1.1"}

{terradev_cli-3.7.6 → terradev_cli-3.7.7/terradev_cli}/README.md

@@ -1,4 +1,4 @@
-# Terradev CLI v3.7.5
+# Terradev CLI v3.7.7
 
 **NUMA-aware GPU provisioning and orchestration for stateless MoE workloads of all sizes**
 
@@ -6,6 +6,46 @@
 
 Terradev is a cross-cloud compute-provisioning CLI that compresses + stages datasets, provisions optimal instances + nodes, and deploys **3-5x faster** than sequential provisioning.
 
+## What's New in v3.7.7
+
+**Complete SGLang Optimization Stack**
+
+Revolutionary workload-specific auto-optimization for SGLang serving with 7 workload types:
+
+### 🚀 SGLang Workload Optimizations
+- **Agentic/Multi-turn Chat**: LPM + RadixAttention + cache-aware routing (75-90% cache hit rate)
+- **High-Throughput Batch**: FCFS + CUDA graphs + FP8 quantization (maximum tokens/sec)
+- **Low-Latency/Real-Time**: EAGLE3 + Spec V2 + capped concurrency (30-50% TTFT improvement)
+- **MoE Models**: DeepEP auto + TBO/SBO + EPLB + redundant experts (up to 2x throughput)
+- **PD Disaggregated**: Separate prefill/decode configurations with production optimizations
+- **Structured Output/RAG**: xGrammar + FSM optimization (10x faster structured output)
+- **Hardware-Specific**: H100/H200, H20, GB200, AMD MI300X optimizations
+
+### 🎯 Auto-Apply Decision Tree
+```bash
+# Auto-optimize any model for workload type
+terradev sglang optimize deepseek-ai/DeepSeek-V3
+
+# Detect workload from description
+terradev sglang detect meta-llama/Llama-2-7b-hf --user-description "Real-time API"
+
+# Multi-replica cache-aware routing
+terradev sglang router meta-llama/Llama-2-7b-hf --dp-size 8
+```
+
+### 📊 Performance Gains
+- **Agentic Chat**: 1.9x throughput with multi-replica, 95-98% GPU utilization
+- **Batch Inference**: Maximum tokens/second with pre-compiled CUDA graphs
+- **Low Latency**: 30-50% TTFT improvement, 20-40% TPOT improvement
+- **MoE Models**: Up to 2x throughput with Two-Batch Overlap
+- **Cache-Aware Routing**: 3.8x higher cache hit rate
+
+### 🔧 Hardware Optimization
+- **H100/H200**: FlashInfer + FP8 KV cache optimization
+- **H20**: FA3 + MoE→QKV→FP8 stacking + swapAB runner
+- **GB200 NVL72**: Rack-scale TP + NUMA-aware placement
+- **AMD MI300X**: Triton backend + ROCm EPLB tuning
+
 ## What's New in v3.7.3
 
 Performance and scalability improvements for enterprise deployments.

{terradev_cli-3.7.6 → terradev_cli-3.7.7}/terradev_cli/__init__.py

@@ -3,7 +3,7 @@ Terradev CLI - Cross-Cloud Compute Optimization Platform
 Parallel provisioning and orchestration for optimized compute costs
 """
 
-__version__ = "3.7.2"
+__version__ = "3.7.7"
 __author__ = "Terradev Team"
 __description__ = (
     "NUMA-aware GPU provisioning and orchestration for stateless MoE workloads of all sizes"

{terradev_cli-3.7.6 → terradev_cli-3.7.7}/terradev_cli/cli.py

@@ -8708,8 +8708,293 @@ def sso_test(provider):
             print(f"✅ {p} configuration appears valid")
 
 
-# Add SSO command group to CLI
+# ── SGLANG COMMAND GROUP ──
+
+@cli.group()
+def sglang():
+    """SGLang optimization and management with workload-specific auto-tuning"""
+    pass
+
+
+@sglang.command()
+@click.argument('model_path')
+@click.option('--workload-type', type=click.Choice(['agentic_chat', 'batch_inference', 'low_latency', 'moe_model', 'pd_disaggregated', 'structured_output', 'rag_workload']), help='Workload type for optimization')
+@click.option('--user-description', help='Natural language description of workload')
+@click.option('--host', default='0.0.0.0', help='Server host')
+@click.option('--port', default=8000, help='Server port')
+@click.option('--dry-run', is_flag=True, help='Show optimization plan without launching')
+def optimize(model_path, workload_type, user_description, host, port, dry_run):
+    """Auto-optimize SGLang configuration for workload type and hardware"""
+    from ml_services.sglang_service import SGLangService, WorkloadType
+    
+    service = SGLangService()
+    
+    # Convert string to enum if provided
+    workload_enum = None
+    if workload_type:
+        workload_enum = WorkloadType(workload_type)
+    
+    # Create optimized configuration
+    config = service.create_optimized_config(
+        model_path=model_path,
+        workload_type=workload_enum,
+        user_description=user_description,
+        host=host,
+        port=port
+    )
+    
+    # Get optimization summary
+    summary = service.get_optimization_summary(config)
+    
+    print(f"🚀 SGLang Optimization Configuration")
+    print(f"Model: {model_path}")
+    print(f"Workload Type: {summary['workload_type']}")
+    print(f"Hardware Detected: {summary['hardware_detected']}")
+    print(f"Schedule Policy: {summary['schedule_policy']}")
+    print(f"Attention Backend: {summary['attention_backend']}")
+    print()
+    
+    print("Applied Optimizations:")
+    for opt in summary['optimizations_applied']:
+        print(f"  ✅ {opt}")
+    print()
+    
+    if summary['performance_expectations']:
+        print("Performance Expectations:")
+        for key, value in summary['performance_expectations'].items():
+            print(f"  📊 {key.replace('_', ' ').title()}: {value}")
+        print()
+    
+    if summary['hardware_tuned']:
+        print("🔧 Hardware-specific optimizations applied")
+        print()
+    
+    # Validate configuration
+    warnings = service.validate_config(config)
+    if warnings:
+        print("⚠️  Configuration Warnings:")
+        for warning in warnings:
+            print(f"  ⚠️  {warning}")
+        print()
+    
+    if dry_run:
+        print("🔍 Dry run - configuration generated but not launched")
+        return
+    
+    # Generate and display launch command
+    launch_cmd = service.generate_launch_command(config)
+    print("🚀 Launch Command:")
+    print(launch_cmd)
+    print()
+    
+    print("💡 To start the server, run:")
+    print(f"   {launch_cmd}")
+
+
+@sglang.command()
+@click.argument('model_path')
+@click.option('--dp-size', default=8, help='Data parallel size for multi-replica')
+@click.option('--workload-type', type=click.Choice(['agentic_chat', 'batch_inference', 'low_latency', 'moe_model', 'pd_disaggregated', 'structured_output', 'rag_workload']), help='Workload type for optimization')
+def router(model_path, dp_size, workload_type):
+    """Generate cache-aware router command for multi-replica deployments"""
+    from ml_services.sglang_service import SGLangService, WorkloadType
+    
+    service = SGLangService()
+    
+    # Convert string to enum if provided
+    workload_enum = None
+    if workload_type:
+        workload_enum = WorkloadType(workload_type)
+    
+    # Create optimized configuration
+    config = service.create_optimized_config(
+        model_path=model_path,
+        workload_type=workload_enum
+    )
+    
+    # Generate router command
+    router_cmd = service.generate_multi_replica_command(config, dp_size)
+    
+    print(f"🔄 Cache-Aware Router Configuration")
+    print(f"Model: {model_path}")
+    print(f"DP Size: {dp_size}")
+    print(f"Workload Type: {config.workload_type.value}")
+    print()
+    print("🚀 Router Launch Command:")
+    print(router_cmd)
+    print()
+    
+    print("💡 This router provides:")
+    print("  📈 Up to 1.9x throughput increase")
+    print("  🎯 3.8x higher cache hit rate")
+    print("  🧠 Intelligent request routing based on cache predictions")
+
+
+@sglang.command()
+@click.argument('model_path')
+@click.option('--workload-type', type=click.Choice(['agentic_chat', 'batch_inference', 'low_latency', 'moe_model', 'pd_disaggregated', 'structured_output', 'rag_workload']), help='Workload type to test')
+@click.option('--user-description', help='Natural language description of workload')
+def detect(model_path, workload_type, user_description):
+    """Auto-detect workload type and show optimization recommendations"""
+    from ml_services.sglang_service import SGLangService, WorkloadType
+    
+    service = SGLangService()
+    
+    # Detect workload type
+    detected_type = service.detect_workload_type(model_path, user_description)
+    
+    print(f"🔍 Workload Detection Results")
+    print(f"Model: {model_path}")
+    print(f"Detected Workload Type: {detected_type.value}")
+    
+    if workload_type:
+        manual_type = WorkloadType(workload_type)
+        print(f"Manual Workload Type: {manual_type.value}")
+        if detected_type != manual_type:
+            print("⚠️  Manual and detected types differ - using manual specification")
+            final_type = manual_type
+        else:
+            print("✅ Manual and detected types match")
+            final_type = detected_type
+    else:
+        final_type = detected_type
+    
+    print()
+    
+    # Show optimization recommendations
+    config = service.create_optimized_config(
+        model_path=model_path,
+        workload_type=final_type,
+        user_description=user_description
+    )
+    
+    summary = service.get_optimization_summary(config)
+    
+    print("🎯 Optimization Recommendations:")
+    for opt in summary['optimizations_applied']:
+        print(f"  ✅ {opt}")
+    print()
+    
+    if summary['performance_expectations']:
+        print("📊 Expected Performance:")
+        for key, value in summary['performance_expectations'].items():
+            print(f"  📈 {key.replace('_', ' ').title()}: {value}")
+    
+    print()
+    print("💡 Run 'terradev sglang optimize' to generate the full launch command")
+
+
+@sglang.command()
+@click.option('--instance-ip', help='Remote instance IP for installation')
+@click.option('--ssh-user', default='root', help='SSH user for remote installation')
+@click.option('--ssh-key', help='SSH private key path')
+def install(instance_ip, ssh_user, ssh_key):
+    """Install SGLang with optimization stack"""
+    from ml_services.sglang_service import SGLangService
+    
+    service = SGLangService()
+    
+    if instance_ip:
+        # Remote installation
+        print(f"📦 Installing SGLang on {instance_ip}...")
+        result = asyncio.run(service.install_on_instance(
+            instance_ip=instance_ip,
+            ssh_user=ssh_user,
+            ssh_key=ssh_key
+        ))
+        
+        if result["status"] == "installed":
+            print("✅ SGLang installed successfully")
+            print(f"📋 Output: {result['output']}")
+        else:
+            print(f"❌ Installation failed: {result['error']}")
+    else:
+        # Local installation
+        print("📦 Installing SGLang locally...")
+        import subprocess
+        import sys
+        
+        try:
+            subprocess.run([
+                sys.executable, "-m", "pip", "install", 
+                "sglang[all]", "--find-links", "https://flashinfer.ai/whl/cu124/torch2.5/flashinfer-python"
+            ], check=True)
+            print("✅ SGLang installed successfully")
+        except subprocess.CalledProcessError as e:
+            print(f"❌ Installation failed: {e}")
+
+
+@sglang.command()
+@click.argument('model_path')
+@click.option('--instance-ip', help='Remote instance IP')
+@click.option('--ssh-user', default='root', help='SSH user for remote deployment')
+@click.option('--ssh-key', help='SSH private key path')
+@click.option('--workload-type', type=click.Choice(['agentic_chat', 'batch_inference', 'low_latency', 'moe_model', 'pd_disaggregated', 'structured_output', 'rag_workload']), help='Workload type for optimization')
+@click.option('--port', default=8000, help='Server port')
+def start(model_path, instance_ip, ssh_user, ssh_key, workload_type, port):
+    """Start optimized SGLang server"""
+    from ml_services.sglang_service import SGLangService, WorkloadType
+    
+    service = SGLangService()
+    
+    # Create optimized configuration
+    workload_enum = None
+    if workload_type:
+        workload_enum = WorkloadType(workload_type)
+    
+    config = service.create_optimized_config(
+        model_path=model_path,
+        workload_type=workload_enum,
+        port=port
+    )
+    
+    if instance_ip:
+        # Remote deployment
+        print(f"🚀 Starting SGLang server on {instance_ip}...")
+        result = asyncio.run(service.start_server(
+            instance_ip=instance_ip,
+            ssh_user=ssh_user,
+            ssh_key=ssh_key
+        ))
+        
+        if result["status"] == "started":
+            print("✅ SGLang server started successfully")
+            print(f"🌐 Endpoint: http://{instance_ip}:{port}")
+        else:
+            print(f"❌ Failed to start server: {result['error']}")
+    else:
+        # Local launch
+        launch_cmd = service.generate_launch_command(config)
+        print(f"🚀 Starting SGLang server locally...")
+        print(f"🌐 Endpoint: http://localhost:{port}")
+        print()
+        print("💡 Launch command:")
+        print(launch_cmd)
+        print()
+        print("⚠️  Run the command above to start the server")
+
+
+@sglang.command()
+def test():
+    """Test SGLang installation and configuration"""
+    from ml_services.sglang_service import SGLangService
+    
+    service = SGLangService()
+    
+    print("🔍 Testing SGLang installation...")
+    result = asyncio.run(service.test_connection())
+    
+    if result["status"] == "connected":
+        print("✅ SGLang is installed and available")
+        print(f"📦 Version: {result['sglang_version']}")
+    else:
+        print(f"❌ SGLang test failed: {result['error']}")
+        print("💡 Run 'terradev sglang install' to install SGLang")
+
+
+# Add command groups to CLI
 cli.add_command(sso)
+cli.add_command(sglang)
 
 
 if __name__ == '__main__':