vllm-sr 0.1.0b2.dev20260203201608__py3-none-any.whl → 0.1.0b2.dev20260204071119__py3-none-any.whl

cli/merger.py

@@ -154,8 +154,20 @@ def translate_latency_signals(latencies: list) -> list:
     for signal in latencies:
         rule = {
             "name": signal.name,
-            "max_tpot": signal.max_tpot,
         }
+        # At least one of tpot_percentile or ttft_percentile should be set
+        if signal.tpot_percentile is not None and signal.tpot_percentile > 0:
+            rule["tpot_percentile"] = signal.tpot_percentile
+        if signal.ttft_percentile is not None and signal.ttft_percentile > 0:
+            rule["ttft_percentile"] = signal.ttft_percentile
+
+        # Validate that at least one is set
+        if "tpot_percentile" not in rule and "ttft_percentile" not in rule:
+            log.warn(
+                f"Latency signal '{signal.name}' has neither tpot_percentile nor ttft_percentile set, skipping"
+            )
+            continue
+
         if signal.description:
             rule["description"] = signal.description
         rules.append(rule)

cli/models.py

@@ -72,7 +72,8 @@ class Latency(BaseModel):
     """Latency signal configuration."""
 
     name: str
-    max_tpot: float
+    tpot_percentile: Optional[int] = None
+    ttft_percentile: Optional[int] = None
     description: str
 
 
@@ -192,18 +193,63 @@ class ConcurrentAlgorithmConfig(BaseModel):
     on_error: Optional[str] = "skip"
 
 
+class ReMoMAlgorithmConfig(BaseModel):
+    """Configuration for ReMoM (Reasoning for Mixture of Models) algorithm.
+
+    This algorithm performs multi-round parallel reasoning with intelligent synthesis.
+    Inspired by PaCoRe (arXiv:2601.05593) but extended to support mixture of models.
+    """
+
+    # Breadth schedule: array of parallel calls per round (e.g., [32, 4] means 32 calls in round 1, 4 in round 2, then 1 final)
+    breadth_schedule: list[int]
+
+    # Model distribution strategy: "weighted", "equal", or "first_only"
+    model_distribution: Optional[str] = "weighted"
+
+    # Temperature for model calls (default: 1.0 for diverse exploration)
+    temperature: Optional[float] = 1.0
+
+    # Whether to include reasoning content in synthesis prompts
+    include_reasoning: Optional[bool] = False
+
+    # Compaction strategy: "full" or "last_n_tokens"
+    compaction_strategy: Optional[str] = "full"
+
+    # Number of tokens to keep when using last_n_tokens compaction
+    compaction_tokens: Optional[int] = 1000
+
+    # Custom synthesis template (uses default if not provided)
+    synthesis_template: Optional[str] = None
+
+    # Maximum concurrent model calls per round
+    max_concurrent: Optional[int] = None
+
+    # Behavior on model call failure: "skip" or "fail"
+    on_error: Optional[str] = "skip"
+
+    # Random seed for shuffling responses (for reproducibility)
+    shuffle_seed: Optional[int] = 42
+
+    # Whether to include intermediate responses in the response body for visualization
+    include_intermediate_responses: Optional[bool] = True
+
+    # Maximum number of responses to keep per round (for memory efficiency)
+    max_responses_per_round: Optional[int] = None
+
+
 class AlgorithmConfig(BaseModel):
     """Algorithm configuration for multi-model decisions.
 
     Specifies how multiple models in a decision should be orchestrated.
     """
 
-    # Algorithm type: "sequential", "confidence", "concurrent"
+    # Algorithm type: "sequential", "confidence", "concurrent", "remom"
     type: str
 
     # Algorithm-specific configurations (only one should be set based on type)
     confidence: Optional[ConfidenceAlgorithmConfig] = None
     concurrent: Optional[ConcurrentAlgorithmConfig] = None
+    remom: Optional[ReMoMAlgorithmConfig] = None
 
 
 class PluginType(str, Enum):

cli/templates/config.template.yaml

@@ -177,14 +177,48 @@ signals:
     - name: "ja"
       description: "Japanese language queries"
 
-  # latency - Latency-based routing signals (TPOT-based)
+  # latency - Latency-based routing signals (TPOT and TTFT percentile-based)
+  # Percentile-based rules adapt to each model's actual performance distribution
+  # Works with any number of observations (1+): uses average for 1-2, percentile for 3+
+  # 
+  # ⚠️  RECOMMENDATION: Use BOTH tpot_percentile AND ttft_percentile for comprehensive latency evaluation
+  #    - TPOT: Measures token generation speed (throughput)
+  #    - TTFT: Measures first token latency (user-perceived latency)
+  #    - Together: Complete latency picture for optimal routing decisions
+  # 
+  # You CAN use only one if needed for specific use cases:
+  #   - TPOT only: Batch processing, cost optimization (throughput matters)
+  #   - TTFT only: Real-time chat (user perception matters)
+  #   - Both: RECOMMENDED for most applications (comprehensive evaluation)
+  # 
+  # At least one of tpot_percentile or ttft_percentile must be set (validation requirement)
+  # When both are set, model must meet BOTH thresholds (AND logic)
   latency:
-    - name: "low_latency"
-      max_tpot: 0.05  # 50ms per token
-      description: "For real-time chat applications requiring fast responses"
-    - name: "medium_latency"
-      max_tpot: 0.15  # 150ms per token
-      description: "For standard applications with moderate latency tolerance"
+    # Example 1: RECOMMENDED - Both TPOT and TTFT percentiles (comprehensive latency evaluation)
+    - name: "low_latency_comprehensive"
+      tpot_percentile: 10  # 10th percentile for TPOT (top 10% fastest token generation)
+      ttft_percentile: 10  # 10th percentile for TTFT (top 10% fastest first token)
+      description: "RECOMMENDED: For real-time applications - fast start and fast generation"
+    
+    # Example 2: Different percentiles for different priorities
+    - name: "balanced_latency"
+      tpot_percentile: 50  # Median TPOT (top 50%)
+      ttft_percentile: 10  # Top 10% TTFT (prioritize fast start)
+      description: "Prioritize fast start, accept moderate generation speed"
+    
+    # Example 3: TPOT percentile only (use case: batch processing, cost optimization)
+    # ⚠️  Note: Only using one metric is allowed but not recommended for most use cases
+    - name: "batch_processing_optimized"
+      tpot_percentile: 10  # 10th percentile for TPOT (top 10% fastest token generation)
+      # ttft_percentile: not set - acceptable for batch processing where throughput matters
+      description: "For batch processing where throughput (TPOT) is critical, TTFT less important"
+    
+    # Example 4: TTFT percentile only (use case: real-time chat where first token matters)
+    # ⚠️  Note: Only using one metric is allowed but not recommended for most use cases
+    - name: "chat_fast_start"
+      ttft_percentile: 10  # 10th percentile for TTFT (top 10% fastest first token)
+      # tpot_percentile: not set - acceptable for chat apps where user perception matters
+      description: "For chat applications where fast first token (TTFT) is critical for UX"
 
   # context - Context length signals (Token Count)
   context:
@@ -431,6 +465,39 @@ decisions:
           enabled: true
           similarity_threshold: 0.85
 
+  # Latency-based routing example: Route to models that meet latency requirements
+  - name: "low_latency_route"
+    description: "Route to models with low latency (fast TPOT and TTFT)"
+    priority: 90
+    rules:
+      operator: "AND"
+      conditions:
+        - type: "latency"
+          name: "low_latency_comprehensive"  # Requires both TPOT and TTFT percentiles
+    modelRefs:
+      - model: "openai/gpt-oss-120b"
+        use_reasoning: false
+    plugins:
+      - type: "system_prompt"
+        configuration:
+          system_prompt: "Provide fast, concise responses suitable for real-time applications."
+
+  - name: "fast_start_route"
+    description: "Route to models with fast first token (prioritize TTFT for chat apps)"
+    priority: 85
+    rules:
+      operator: "AND"
+      conditions:
+        - type: "latency"
+          name: "chat_fast_start"  # Only requires TTFT percentile
+    modelRefs:
+      - model: "openai/gpt-oss-120b"
+        use_reasoning: false
+    plugins:
+      - type: "system_prompt"
+        configuration:
+          system_prompt: "Start responding quickly. User is waiting for immediate feedback."
+
   # Size-aware routing example: Try smaller models first, escalate if confidence is low
   - name: "confidence_route"
     description: "Cost-efficient routing: try small model first, escalate if needed"
@@ -483,6 +550,59 @@ decisions:
         configuration:
           enabled: false
 
+  # ReMoM algorithm example: Multi-round parallel reasoning with intelligent synthesis
+  - name: "remom_route"
+    description: "Complex reasoning using ReMoM (Reasoning for Mixture of Models)"
+    priority: 70
+    rules:
+      operator: "AND"
+      conditions:
+        - type: "keyword"
+          name: "looper_keywords"
+    modelRefs:
+      - model: "openai/gpt-oss-120b"
+      - model: "gpt-5.2"
+    algorithm:
+      type: "remom"
+      remom:
+        # Breadth schedule: [32, 4] means 32 parallel calls in round 1, 4 in round 2, then 1 final
+        # Low intensity: [4], Medium: [16], High: [32, 4]
+        breadth_schedule: [4]  # Low intensity for demonstration
+
+        # Model distribution strategy:
+        # - "weighted": Distribute calls based on model weights (default)
+        # - "equal": Distribute evenly across all models
+        # - "first_only": Use only the first model (PaCoRe-compatible)
+        model_distribution: "equal"
+
+        # Temperature for diverse exploration (default: 1.0)
+        temperature: 1.0
+
+        # Include reasoning content from vLLM in synthesis prompts
+        include_reasoning: true
+
+        # Compaction strategy: "full" or "last_n_tokens"
+        compaction_strategy: "last_n_tokens"
+        compaction_tokens: 1000  # Keep last 1000 tokens when compacting
+
+        # Custom synthesis template (optional, uses default if not provided)
+        # synthesis_template: "Your custom template here"
+
+        # Maximum concurrent model calls per round (optional, defaults to all)
+        max_concurrent: 4
+
+        # Behavior on model call failure: "skip" or "fail"
+        on_error: "skip"
+
+        # Random seed for reproducibility
+        shuffle_seed: 42
+
+        # Include intermediate responses for visualization in dashboard
+        include_intermediate_responses: true
+
+        # Maximum responses to keep per round (optional, for memory efficiency)
+        # max_responses_per_round: 10
+
 # LLM - Backend model configuration
 providers:
   # Model configuration

cli/templates/envoy.template.yaml

@@ -12,7 +12,7 @@ static_resources:
       socket_address:
         address: {{ listener.address }}
         port_value: {{ listener.port }}
-    perConnectionBufferLimitBytes: 52428800
+    perConnectionBufferLimitBytes: 524288000
     filter_chains:
     - filters:
       - name: envoy.filters.network.http_connection_manager
@@ -36,8 +36,8 @@ static_resources:
                       exact: "{{ model.name }}"
                 route:
                   cluster: {{ model.cluster_name }}_cluster
-                  timeout: {{ listener.timeout | default('300s') }}
-                  idleTimeout: 300s
+                  timeout: {{ listener.timeout | default('600s') }}
+                  idleTimeout: 600s
                   # Rewrite Host header to match upstream server
                   host_rewrite_literal: "{{ model.endpoints[0].address }}"
                   {% if model.path_prefix %}
@@ -59,8 +59,8 @@ static_resources:
                       exact: "{{ model.name }}"
                 route:
                   cluster: anthropic_api_cluster
-                  timeout: {{ listener.timeout | default('300s') }}
-                  idleTimeout: 300s
+                  timeout: {{ listener.timeout | default('600s') }}
+                  idleTimeout: 600s
                   host_rewrite_literal: "api.anthropic.com"
               {% endfor %}
               # Default route (no x-selected-model header)
@@ -73,7 +73,7 @@ static_resources:
                   {% else %}
                   cluster: vllm_static_cluster
                   {% endif %}
-                  timeout: {{ listener.timeout | default('300s') }}
+                  timeout: {{ listener.timeout | default('600s') }}
                   {% if models %}
                   # Rewrite Host header to match upstream server
                   host_rewrite_literal: "{{ models[0].endpoints[0].address }}"
@@ -94,13 +94,13 @@ static_resources:
               grpc_service:
                 envoy_grpc:
                   cluster_name: extproc_service
-                timeout: 300s
+                timeout: 600s
               processing_mode:
                 request_header_mode: "SEND"
                 response_header_mode: "SEND"
                 request_body_mode: "BUFFERED"
                 response_body_mode: "BUFFERED"
-              message_timeout: {{ listener.timeout | default('300s') }}
+              message_timeout: {{ listener.timeout | default('600s') }}
           - name: envoy.filters.http.router
             typed_config:
               "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
@@ -115,7 +115,7 @@ static_resources:
   clusters:
   # ExtProc service (semantic router)
   - name: extproc_service
-    connect_timeout: 300s
+    connect_timeout: 600s
     type: STATIC
     lb_policy: ROUND_ROBIN
     http2_protocol_options: {}
@@ -150,7 +150,7 @@ static_resources:
   {% for model in models %}
   # Cluster for model: {{ model.name }}
   - name: {{ model.cluster_name }}_cluster
-    connect_timeout: 300s
+    connect_timeout: 600s
     type: {{ model.cluster_type }}
     {% if model.cluster_type == 'LOGICAL_DNS' %}
     dns_lookup_family: V4_ONLY
@@ -189,7 +189,7 @@ static_resources:
   - name: anthropic_api_cluster
     type: LOGICAL_DNS
     dns_lookup_family: V4_ONLY
-    connect_timeout: 60s
+    connect_timeout: 600s
     lb_policy: ROUND_ROBIN
     load_assignment:
       cluster_name: anthropic_api_cluster

cli/templates/router-defaults.yaml

@@ -172,7 +172,7 @@ looper:
   enabled: true  # Enable looper for multi-model decisions
   # Endpoint points to Envoy (same container), which handles load balancing and auth
   # Port should match listener port (default: 8888)
-  endpoint: "http://localhost:8888/v1/chat/completions"
+  endpoint: "http://localhost:8899/v1/chat/completions"
   timeout_seconds: 120  # Timeout in seconds for each model call
   headers: {}  # Optional headers (e.g., {"Authorization": "Bearer xxx"})
 

{vllm_sr-0.1.0b2.dev20260203201608.dist-info → vllm_sr-0.1.0b2.dev20260204071119.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vllm-sr
-Version: 0.1.0b2.dev20260203201608
+Version: 0.1.0b2.dev20260204071119
 Summary: vLLM Semantic Router - Intelligent routing for Mixture-of-Models
 Author: vLLM-SR Team
 License: Apache-2.0

{vllm_sr-0.1.0b2.dev20260203201608.dist-info → vllm_sr-0.1.0b2.dev20260204071119.dist-info}/RECORD

@@ -6,8 +6,8 @@ cli/defaults.py,sha256=x3mdZPBtI74bzgO5UDL-xoNV5jh9uPeSDcfgnkWHVyA,1969
 cli/docker_cli.py,sha256=kj3VvNfUJwcizZQj63mB2yLBBUbvZ9P3MtleIXhTp0M,25230
 cli/logo.py,sha256=I0qnCCGyOsHmN6MRgqa_c07MZSArv_6eHcHAL36V0Eg,1512
 cli/main.py,sha256=-PkXZw_z0VtzLbA5BCQlvLTYRuNbAT0CcBSFvjrsmBo,9137
-cli/merger.py,sha256=KMajL8tzdZeVRSdyq6MJt-DNhOQ5K_suMGx3jhl0odQ,16569
-cli/models.py,sha256=2oSGxDJzEA-fLGyamAfrKun4RqGlosbMQzq3nCzcDEo,11723
+cli/merger.py,sha256=Cd3DHt5_bMAw6oxSMtu5Debd6gCCRb0wjj-RTZSrZw4,17172
+cli/models.py,sha256=IOpG-XtLZtVDzQVIVd_PyLvMj_5BJDAt4R7kYBqr7Wo,13532
 cli/parser.py,sha256=lFPrROAYyc2cPoMMULV2sPid5IklIvc8zpDekj5Pocw,5653
 cli/utils.py,sha256=CHMwSZC-zxrxCnlZqiyzF2CToCVB5tHku2zyYig8fo0,3787
 cli/validator.py,sha256=G49I8_vyg0HvfI2LHwFlOagTwWAEsIkIDzX5riYoZhA,10651
@@ -19,8 +19,8 @@ cli/commands/serve.py,sha256=vsK3T4uWx49CPTgy4sc6oM7F5riq0sNbyiT7KbS37Wc,6104
 cli/commands/show_config.py,sha256=emkIdH9LKbzMzxna3Lxl-hEG85isfAXY5BlZjP_vaIM,4689
 cli/commands/show_defaults.py,sha256=r95KOHKQWeeNnoFO5EmCUZSPeOKb-5NzCoaZ6CaBcIw,701
 cli/commands/validate.py,sha256=Chm_MfyJMESiOo6IXiroYwMUkm8HCI26OHYYUBkGXGk,2773
-cli/templates/config.template.yaml,sha256=Pw-BX4afrzUKULRaxhc6VQ14H45CLPTELs0xEaiylwY,16598
-cli/templates/envoy.template.yaml,sha256=Vws_di-mQXpzqnrYmQUdfHvf0jtlt92ce4UE3loU3Vw,8291
+cli/templates/config.template.yaml,sha256=8qnnyWiSLT7EkCFWK_O3qyHqvzfY_Q8GjmcbF13-e3k,22012
+cli/templates/envoy.template.yaml,sha256=j43mxQ3NcFMlmbGyrDM0-1rvQkC1N4MpXUC2o7SLT9g,8293
 cli/templates/generate_dashboard.py,sha256=gCui1rPrxUFIeioebyQ_JhL7zbjoEAg8GjUJcWXHcys,24233
 cli/templates/grafana-dashboard.serve.yaml,sha256=o1H5xx60esBzk19qnMQtX4LmL5DswpMvOO7cO09Q3kI,350
 cli/templates/grafana-datasource-jaeger.serve.yaml,sha256=7eigRe8mdyHjbPA7B-ZoJGHrq1chhCPWWsE-K4Dvj30,319
@@ -28,10 +28,10 @@ cli/templates/grafana-datasource.serve.yaml,sha256=Cxjz1zVWoUdSzbSsS_iJhMHRrmRi6
 cli/templates/grafana.serve.ini,sha256=x9bCkzxqm5gC4fKToY2lhNPdWhwAaJGVe5ABMW6Dv-c,1674
 cli/templates/llm-router-dashboard.serve.json,sha256=pwnTjUh7z3_3LnIwtaLXjDWH4aHd2Mc57z0oekgt-Bk,60903
 cli/templates/prometheus.serve.yaml,sha256=MGYq8dlRq_i2m5sogQ--kwTvJpkf44QQoCNoI7oyVT8,270
-cli/templates/router-defaults.yaml,sha256=IIiQWQAQ5KsCJMYbTOdOjwb98KSNbzcMS_VHby4y5Hs,9373
+cli/templates/router-defaults.yaml,sha256=a0riw9juttoBr4G7rtUEJA2FY9TgAvK-8mngWGCCIoc,9373
 cli/templates/tools_db.json,sha256=CPqPBkd5nc966m1YEozz06frrmv3Pd5rrkxKkO3rTiA,4537
-vllm_sr-0.1.0b2.dev20260203201608.dist-info/METADATA,sha256=KIrcmvBZQsV1cC5mSSN5203_CZFfCwLVHqT0RZv1bZo,7298
-vllm_sr-0.1.0b2.dev20260203201608.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-vllm_sr-0.1.0b2.dev20260203201608.dist-info/entry_points.txt,sha256=WhlBPbLHUpWUsMuUQX9cnvsYMf0ih5i57vvJ1jJNi0k,42
-vllm_sr-0.1.0b2.dev20260203201608.dist-info/top_level.txt,sha256=2ImG917oaVHlm0nP9oJE-Qrgs-fq_fGWgba2H1f8fpE,4
-vllm_sr-0.1.0b2.dev20260203201608.dist-info/RECORD,,
+vllm_sr-0.1.0b2.dev20260204071119.dist-info/METADATA,sha256=fdSPNqwQef9TRpKOuSKW_ryTgOvA5dAhzkGCeO5Fihc,7298
+vllm_sr-0.1.0b2.dev20260204071119.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+vllm_sr-0.1.0b2.dev20260204071119.dist-info/entry_points.txt,sha256=WhlBPbLHUpWUsMuUQX9cnvsYMf0ih5i57vvJ1jJNi0k,42
+vllm_sr-0.1.0b2.dev20260204071119.dist-info/top_level.txt,sha256=2ImG917oaVHlm0nP9oJE-Qrgs-fq_fGWgba2H1f8fpE,4
+vllm_sr-0.1.0b2.dev20260204071119.dist-info/RECORD,,