sglang 0.1.16__py3-none-any.whl → 0.1.18__py3-none-any.whl

sglang/srt/managers/{router → controller}/radix_cache.py

@@ -1,3 +1,7 @@
+"""
+The radix tree data structure for managing the KV cache.
+"""
+
 import heapq
 import time
 from collections import defaultdict
@@ -58,7 +62,7 @@ class RadixCache:
 
     def insert(self, key, value=None):
         if self.disable:
-            return len(key)
+            return 0
 
         if value is None:
             value = [x for x in key]
@@ -76,6 +80,12 @@ class RadixCache:
         indices = self.req_to_token_pool.req_to_token[req_pool_idx, : len(token_ids)]
         new_prefix_len = self.insert(token_ids, indices.clone())
 
+        if self.disable:
+            if del_in_memory_pool:
+                self.token_to_kv_pool.dec_refs(indices)
+            else:
+                return torch.tensor([], dtype=torch.int64), self.root_node
+
         # Radix Cache takes one ref in memory pool
         self.token_to_kv_pool.dec_refs(indices[last_uncached_pos:new_prefix_len])
 

sglang/srt/managers/{router/scheduler.py → controller/schedule_heuristic.py}

@@ -1,20 +1,22 @@
+"""Request scheduler heuristic."""
+
 import random
 from collections import defaultdict
 
 
-class Scheduler:
+class ScheduleHeuristic:
     def __init__(
         self,
         schedule_heuristic,
-        max_running_seq,
-        max_prefill_num_token,
-        max_total_num_token,
+        max_running_seqs,
+        max_prefill_num_tokens,
+        max_total_num_tokens,
         tree_cache,
     ):
         self.schedule_heuristic = schedule_heuristic
-        self.max_running_seq = max_running_seq
-        self.max_prefill_num_token = max_prefill_num_token
-        self.max_total_num_token = max_total_num_token
+        self.max_running_seqs = max_running_seqs
+        self.max_prefill_num_tokens = max_prefill_num_tokens
+        self.max_total_num_tokens = max_total_num_tokens
         self.tree_cache = tree_cache
 
     def get_priority_queue(self, forward_queue):