vllm-sr 0.1.0b2.dev20260204070724__tar.gz → 0.1.0b2.dev20260204090051__tar.gz

{vllm_sr-0.1.0b2.dev20260204070724/vllm_sr.egg-info → vllm_sr-0.1.0b2.dev20260204090051}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vllm-sr
-Version: 0.1.0b2.dev20260204070724
+Version: 0.1.0b2.dev20260204090051
 Summary: vLLM Semantic Router - Intelligent routing for Mixture-of-Models
 Author: vLLM-SR Team
 License: Apache-2.0

{vllm_sr-0.1.0b2.dev20260204070724 → vllm_sr-0.1.0b2.dev20260204090051}/cli/merger.py

@@ -154,8 +154,20 @@ def translate_latency_signals(latencies: list) -> list:
     for signal in latencies:
         rule = {
             "name": signal.name,
-            "max_tpot": signal.max_tpot,
         }
+        # At least one of tpot_percentile or ttft_percentile should be set
+        if signal.tpot_percentile is not None and signal.tpot_percentile > 0:
+            rule["tpot_percentile"] = signal.tpot_percentile
+        if signal.ttft_percentile is not None and signal.ttft_percentile > 0:
+            rule["ttft_percentile"] = signal.ttft_percentile
+
+        # Validate that at least one is set
+        if "tpot_percentile" not in rule and "ttft_percentile" not in rule:
+            log.warn(
+                f"Latency signal '{signal.name}' has neither tpot_percentile nor ttft_percentile set, skipping"
+            )
+            continue
+
         if signal.description:
             rule["description"] = signal.description
         rules.append(rule)

{vllm_sr-0.1.0b2.dev20260204070724 → vllm_sr-0.1.0b2.dev20260204090051}/cli/models.py

@@ -72,7 +72,8 @@ class Latency(BaseModel):
     """Latency signal configuration."""
 
     name: str
-    max_tpot: float
+    tpot_percentile: Optional[int] = None
+    ttft_percentile: Optional[int] = None
     description: str
 
 

{vllm_sr-0.1.0b2.dev20260204070724 → vllm_sr-0.1.0b2.dev20260204090051}/cli/templates/config.template.yaml

@@ -177,14 +177,48 @@ signals:
     - name: "ja"
       description: "Japanese language queries"
 
-  # latency - Latency-based routing signals (TPOT-based)
+  # latency - Latency-based routing signals (TPOT and TTFT percentile-based)
+  # Percentile-based rules adapt to each model's actual performance distribution
+  # Works with any number of observations (1+): uses average for 1-2, percentile for 3+
+  # 
+  # ⚠️  RECOMMENDATION: Use BOTH tpot_percentile AND ttft_percentile for comprehensive latency evaluation
+  #    - TPOT: Measures token generation speed (throughput)
+  #    - TTFT: Measures first token latency (user-perceived latency)
+  #    - Together: Complete latency picture for optimal routing decisions
+  # 
+  # You CAN use only one if needed for specific use cases:
+  #   - TPOT only: Batch processing, cost optimization (throughput matters)
+  #   - TTFT only: Real-time chat (user perception matters)
+  #   - Both: RECOMMENDED for most applications (comprehensive evaluation)
+  # 
+  # At least one of tpot_percentile or ttft_percentile must be set (validation requirement)
+  # When both are set, model must meet BOTH thresholds (AND logic)
   latency:
-    - name: "low_latency"
-      max_tpot: 0.05  # 50ms per token
-      description: "For real-time chat applications requiring fast responses"
-    - name: "medium_latency"
-      max_tpot: 0.15  # 150ms per token
-      description: "For standard applications with moderate latency tolerance"
+    # Example 1: RECOMMENDED - Both TPOT and TTFT percentiles (comprehensive latency evaluation)
+    - name: "low_latency_comprehensive"
+      tpot_percentile: 10  # 10th percentile for TPOT (top 10% fastest token generation)
+      ttft_percentile: 10  # 10th percentile for TTFT (top 10% fastest first token)
+      description: "RECOMMENDED: For real-time applications - fast start and fast generation"
+    
+    # Example 2: Different percentiles for different priorities
+    - name: "balanced_latency"
+      tpot_percentile: 50  # Median TPOT (top 50%)
+      ttft_percentile: 10  # Top 10% TTFT (prioritize fast start)
+      description: "Prioritize fast start, accept moderate generation speed"
+    
+    # Example 3: TPOT percentile only (use case: batch processing, cost optimization)
+    # ⚠️  Note: Only using one metric is allowed but not recommended for most use cases
+    - name: "batch_processing_optimized"
+      tpot_percentile: 10  # 10th percentile for TPOT (top 10% fastest token generation)
+      # ttft_percentile: not set - acceptable for batch processing where throughput matters
+      description: "For batch processing where throughput (TPOT) is critical, TTFT less important"
+    
+    # Example 4: TTFT percentile only (use case: real-time chat where first token matters)
+    # ⚠️  Note: Only using one metric is allowed but not recommended for most use cases
+    - name: "chat_fast_start"
+      ttft_percentile: 10  # 10th percentile for TTFT (top 10% fastest first token)
+      # tpot_percentile: not set - acceptable for chat apps where user perception matters
+      description: "For chat applications where fast first token (TTFT) is critical for UX"
 
   # context - Context length signals (Token Count)
   context:
@@ -431,6 +465,39 @@ decisions:
           enabled: true
           similarity_threshold: 0.85
 
+  # Latency-based routing example: Route to models that meet latency requirements
+  - name: "low_latency_route"
+    description: "Route to models with low latency (fast TPOT and TTFT)"
+    priority: 90
+    rules:
+      operator: "AND"
+      conditions:
+        - type: "latency"
+          name: "low_latency_comprehensive"  # Requires both TPOT and TTFT percentiles
+    modelRefs:
+      - model: "openai/gpt-oss-120b"
+        use_reasoning: false
+    plugins:
+      - type: "system_prompt"
+        configuration:
+          system_prompt: "Provide fast, concise responses suitable for real-time applications."
+
+  - name: "fast_start_route"
+    description: "Route to models with fast first token (prioritize TTFT for chat apps)"
+    priority: 85
+    rules:
+      operator: "AND"
+      conditions:
+        - type: "latency"
+          name: "chat_fast_start"  # Only requires TTFT percentile
+    modelRefs:
+      - model: "openai/gpt-oss-120b"
+        use_reasoning: false
+    plugins:
+      - type: "system_prompt"
+        configuration:
+          system_prompt: "Start responding quickly. User is waiting for immediate feedback."
+
   # Size-aware routing example: Try smaller models first, escalate if confidence is low
   - name: "confidence_route"
     description: "Cost-efficient routing: try small model first, escalate if needed"

{vllm_sr-0.1.0b2.dev20260204070724 → vllm_sr-0.1.0b2.dev20260204090051}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "vllm-sr"
-version = "0.1.0.beta.2.dev20260204070724"
+version = "0.1.0.beta.2.dev20260204090051"
 description = "vLLM Semantic Router - Intelligent routing for Mixture-of-Models"
 authors = [{name = "vLLM-SR Team"}]
 readme = "README.md"

{vllm_sr-0.1.0b2.dev20260204070724 → vllm_sr-0.1.0b2.dev20260204090051/vllm_sr.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vllm-sr
-Version: 0.1.0b2.dev20260204070724
+Version: 0.1.0b2.dev20260204090051
 Summary: vLLM Semantic Router - Intelligent routing for Mixture-of-Models
 Author: vLLM-SR Team
 License: Apache-2.0