sglang 0.1.17__py3-none-any.whl → 0.1.19__py3-none-any.whl

sglang/srt/managers/router/model_runner.py

@@ -1,445 +0,0 @@
-import importlib
-import importlib.resources
-import logging
-import pkgutil
-from dataclasses import dataclass
-from functools import lru_cache
-from typing import List, Optional, Type
-
-import numpy as np
-import torch
-import torch.nn as nn
-from vllm.config import DeviceConfig, LoadConfig
-from vllm.config import ModelConfig as VllmModelConfig
-from vllm.distributed import initialize_model_parallel
-from vllm.model_executor.model_loader import get_model
-from vllm.model_executor.models import ModelRegistry
-
-from sglang.srt.managers.router.infer_batch import Batch, ForwardMode
-from sglang.srt.memory_pool import ReqToTokenPool, TokenToKVPool
-from sglang.srt.server_args import ServerArgs
-from sglang.srt.utils import get_available_gpu_memory, is_multimodal_model
-
-
-logger = logging.getLogger("model_runner")
-
-# for server args in model endpoints
-global_server_args_dict = {}
-
-
-@dataclass
-class InputMetadata:
-    model_runner: "ModelRunner"
-    forward_mode: ForwardMode
-    batch_size: int
-    total_num_tokens: int
-    max_seq_len: int
-    req_pool_indices: torch.Tensor
-    start_loc: torch.Tensor
-    seq_lens: torch.Tensor
-    prefix_lens: torch.Tensor
-    positions: torch.Tensor
-    req_to_token_pool: ReqToTokenPool
-    token_to_kv_pool: TokenToKVPool
-
-    # for extend
-    extend_seq_lens: torch.Tensor = None
-    extend_start_loc: torch.Tensor = None
-    max_extend_len: int = 0
-
-    out_cache_loc: torch.Tensor = None
-    out_cache_cont_start: torch.Tensor = None
-    out_cache_cont_end: torch.Tensor = None
-
-    other_kv_index: torch.Tensor = None
-    return_logprob: bool = False
-    top_logprobs_nums: List[int] = None
-
-    # for flashinfer
-    qo_indptr: torch.Tensor = None
-    kv_indptr: torch.Tensor = None
-    kv_indices: torch.Tensor = None
-    kv_last_page_len: torch.Tensor = None
-    prefill_wrapper = None
-    decode_wrapper = None
-
-    def init_flashinfer_args(self, tp_size):
-        from flashinfer import (
-            BatchDecodeWithPagedKVCacheWrapper,
-            BatchPrefillWithPagedKVCacheWrapper,
-        )
-
-        self.kv_indptr = torch.zeros(
-            (self.batch_size + 1,), dtype=torch.int32, device="cuda"
-        )
-        self.kv_indptr[1:] = torch.cumsum(self.seq_lens, dim=0)
-        self.kv_last_page_len = torch.ones(
-            (self.batch_size,), dtype=torch.int32, device="cuda"
-        )
-        req_pool_indices_cpu = self.req_pool_indices.cpu().numpy()
-        seq_lens_cpu = self.seq_lens.cpu().numpy()
-        self.kv_indices = torch.cat(
-            [
-                self.req_to_token_pool.req_to_token[
-                    req_pool_indices_cpu[i], : seq_lens_cpu[i]
-                ]
-                for i in range(self.batch_size)
-            ],
-            dim=0,
-        ).contiguous()
-
-        workspace_buffer = torch.empty(
-            32 * 1024 * 1024, dtype=torch.int8, device="cuda"
-        )
-        if (
-            self.forward_mode == ForwardMode.PREFILL
-            or self.forward_mode == ForwardMode.EXTEND
-        ):
-            self.qo_indptr = torch.zeros(
-                (self.batch_size + 1,), dtype=torch.int32, device="cuda"
-            )
-            self.qo_indptr[1:] = torch.cumsum(self.extend_seq_lens, dim=0)
-            self.prefill_wrapper = BatchPrefillWithPagedKVCacheWrapper(
-                workspace_buffer, "NHD"
-            )
-            args = [
-                self.qo_indptr,
-                self.kv_indptr,
-                self.kv_indices,
-                self.kv_last_page_len,
-                self.model_runner.model_config.num_attention_heads // tp_size,
-                self.model_runner.model_config.num_key_value_heads // tp_size,
-                self.model_runner.model_config.head_dim,
-            ]
-
-            self.prefill_wrapper.begin_forward(*args)
-        else:
-            self.decode_wrapper = BatchDecodeWithPagedKVCacheWrapper(
-                workspace_buffer, "NHD"
-            )
-            self.decode_wrapper.begin_forward(
-                self.kv_indptr,
-                self.kv_indices,
-                self.kv_last_page_len,
-                self.model_runner.model_config.num_attention_heads // tp_size,
-                self.model_runner.model_config.num_key_value_heads // tp_size,
-                self.model_runner.model_config.head_dim,
-                1,
-                "NONE",
-                "float16",
-            )
-
-    def init_extend_args(self):
-        self.extend_seq_lens = self.seq_lens - self.prefix_lens
-        self.extend_start_loc = torch.zeros_like(self.seq_lens)
-        self.extend_start_loc[1:] = torch.cumsum(self.extend_seq_lens[:-1], dim=0)
-        self.max_extend_len = int(torch.max(self.extend_seq_lens))
-
-    @classmethod
-    def create(
-        cls,
-        model_runner,
-        tp_size,
-        forward_mode,
-        req_pool_indices,
-        seq_lens,
-        prefix_lens,
-        position_ids_offsets,
-        out_cache_loc,
-        out_cache_cont_start=None,
-        out_cache_cont_end=None,
-        top_logprobs_nums=None,
-        return_logprob=False,
-    ):
-        batch_size = len(req_pool_indices)
-        start_loc = torch.zeros((batch_size,), dtype=torch.int32, device="cuda")
-        start_loc[1:] = torch.cumsum(seq_lens[:-1], dim=0)
-        total_num_tokens = int(torch.sum(seq_lens))
-        max_seq_len = int(torch.max(seq_lens))
-
-        if forward_mode == ForwardMode.DECODE:
-            positions = ((seq_lens - 1) + position_ids_offsets).to(torch.int64)
-            other_kv_index = model_runner.req_to_token_pool.req_to_token[
-                req_pool_indices[0], seq_lens[0] - 1
-            ].item()
-        else:
-            seq_lens_cpu = seq_lens.cpu().numpy()
-            prefix_lens_cpu = prefix_lens.cpu().numpy()
-            position_ids_offsets_cpu = position_ids_offsets.cpu().numpy()
-            positions = torch.tensor(
-                np.concatenate(
-                    [
-                        np.arange(
-                            prefix_lens_cpu[i] + position_ids_offsets_cpu[i],
-                            seq_lens_cpu[i] + position_ids_offsets_cpu[i],
-                        )
-                        for i in range(batch_size)
-                    ],
-                    axis=0,
-                ),
-                device="cuda",
-            )
-            other_kv_index = None
-
-        ret = cls(
-            model_runner=model_runner,
-            forward_mode=forward_mode,
-            batch_size=batch_size,
-            total_num_tokens=total_num_tokens,
-            max_seq_len=max_seq_len,
-            req_pool_indices=req_pool_indices,
-            start_loc=start_loc,
-            seq_lens=seq_lens,
-            prefix_lens=prefix_lens,
-            positions=positions,
-            req_to_token_pool=model_runner.req_to_token_pool,
-            token_to_kv_pool=model_runner.token_to_kv_pool,
-            out_cache_loc=out_cache_loc,
-            out_cache_cont_start=out_cache_cont_start,
-            out_cache_cont_end=out_cache_cont_end,
-            other_kv_index=other_kv_index,
-            return_logprob=return_logprob,
-            top_logprobs_nums=top_logprobs_nums,
-        )
-
-        if forward_mode == ForwardMode.EXTEND:
-            ret.init_extend_args()
-
-        if global_server_args_dict.get("enable_flashinfer", False):
-            ret.init_flashinfer_args(tp_size)
-
-        return ret
-
-
-class ModelRunner:
-    def __init__(
-        self,
-        model_config,
-        mem_fraction_static,
-        tp_rank,
-        tp_size,
-        nccl_port,
-        server_args: ServerArgs,
-    ):
-        self.model_config = model_config
-        self.mem_fraction_static = mem_fraction_static
-        self.tp_rank = tp_rank
-        self.tp_size = tp_size
-        self.nccl_port = nccl_port
-        self.server_args = server_args
-
-        global global_server_args_dict
-        global_server_args_dict = {
-            "enable_flashinfer": server_args.enable_flashinfer,
-            "attention_reduce_in_fp32": server_args.attention_reduce_in_fp32,
-        }
-
-        # Init torch distributed
-        logger.info(f"[rank={self.tp_rank}] Set cuda device.")
-        torch.cuda.set_device(self.tp_rank)
-        logger.info(f"[rank={self.tp_rank}] Init torch begin. Avail mem={get_available_gpu_memory(self.tp_rank):.2f} GB")
-        torch.distributed.init_process_group(
-            backend="nccl",
-            world_size=self.tp_size,
-            rank=self.tp_rank,
-            init_method=f"tcp://127.0.0.1:{self.nccl_port}",
-        )
-        initialize_model_parallel(tensor_model_parallel_size=self.tp_size)
-        logger.info(f"[rank={self.tp_rank}] Init torch end.")
-
-        total_gpu_memory = get_available_gpu_memory(self.tp_rank, distributed=self.tp_size > 1)
-
-        if self.tp_size > 1:
-            total_local_gpu_memory = get_available_gpu_memory(self.tp_rank)
-            if total_local_gpu_memory < total_gpu_memory * 0.9:
-                raise ValueError("The memory capacity is unbalanced. Some GPUs may be occupied by other processes.")
-
-        self.load_model()
-        self.init_memory_pool(total_gpu_memory)
-
-        self.is_multimodal_model = is_multimodal_model(self.model_config)
-
-    def load_model(self):
-        logger.info(f"[rank={self.tp_rank}] Load weight begin.")
-
-        device_config = DeviceConfig()
-        load_config = LoadConfig(load_format=self.server_args.load_format)
-        vllm_model_config = VllmModelConfig(
-            model=self.server_args.model_path,
-            quantization=self.server_args.quantization,
-            tokenizer=None,
-            tokenizer_mode=None,
-            trust_remote_code=self.server_args.trust_remote_code,
-            dtype=torch.float16,
-            seed=42,
-            skip_tokenizer_init=True,
-        )
-        if self.model_config.model_overide_args is not None:
-            vllm_model_config.hf_config.update(self.model_config.model_overide_args)
-
-        self.model = get_model(
-            model_config=vllm_model_config,
-            device_config=device_config,
-            load_config=load_config,
-            lora_config=None,
-            vision_language_config=None,
-            parallel_config=None,
-            scheduler_config=None,
-        )
-        logger.info(f"[rank={self.tp_rank}] Load weight end. "
-                    f"Type={type(self.model).__name__}. "
-                    f"Avail mem={get_available_gpu_memory(self.tp_rank):.2f} GB")
-
-    def profile_max_num_token(self, total_gpu_memory):
-        available_gpu_memory = get_available_gpu_memory(self.tp_rank, distributed=self.tp_size > 1)
-        head_dim = self.model_config.head_dim
-        head_num = self.model_config.num_key_value_heads // self.tp_size
-        cell_size = head_num * head_dim * self.model_config.num_hidden_layers * 2 * 2
-        rest_memory = available_gpu_memory - total_gpu_memory * (
-            1 - self.mem_fraction_static
-        )
-        max_num_token = int(rest_memory * (1 << 30) // cell_size)
-        return max_num_token
-
-    def init_memory_pool(self, total_gpu_memory):
-        self.max_total_num_tokens = self.profile_max_num_token(total_gpu_memory)
-
-        if self.max_total_num_tokens <= 0:
-            raise RuntimeError(
-                "Not enought memory. " "Please try to increase --mem-fraction-static."
-            )
-
-        self.req_to_token_pool = ReqToTokenPool(
-            int(self.max_total_num_tokens / self.model_config.context_len * 256),
-            self.model_config.context_len + 8,
-        )
-        self.token_to_kv_pool = TokenToKVPool(
-            self.max_total_num_tokens,
-            dtype=torch.float16,
-            head_num=self.model_config.num_key_value_heads // self.tp_size,
-            head_dim=self.model_config.head_dim,
-            layer_num=self.model_config.num_hidden_layers,
-        )
-
-    @torch.inference_mode()
-    def forward_prefill(self, batch: Batch):
-        input_metadata = InputMetadata.create(
-            self,
-            forward_mode=ForwardMode.PREFILL,
-            tp_size=self.tp_size,
-            req_pool_indices=batch.req_pool_indices,
-            seq_lens=batch.seq_lens,
-            prefix_lens=batch.prefix_lens,
-            position_ids_offsets=batch.position_ids_offsets,
-            out_cache_loc=batch.out_cache_loc,
-            top_logprobs_nums=batch.top_logprobs_nums,
-            return_logprob=batch.return_logprob,
-        )
-        return self.model.forward(
-            batch.input_ids, input_metadata.positions, input_metadata
-        )
-
-    @torch.inference_mode()
-    def forward_extend(self, batch: Batch):
-        input_metadata = InputMetadata.create(
-            self,
-            forward_mode=ForwardMode.EXTEND,
-            tp_size=self.tp_size,
-            req_pool_indices=batch.req_pool_indices,
-            seq_lens=batch.seq_lens,
-            prefix_lens=batch.prefix_lens,
-            position_ids_offsets=batch.position_ids_offsets,
-            out_cache_loc=batch.out_cache_loc,
-            top_logprobs_nums=batch.top_logprobs_nums,
-            return_logprob=batch.return_logprob,
-        )
-        return self.model.forward(
-            batch.input_ids, input_metadata.positions, input_metadata
-        )
-
-    @torch.inference_mode()
-    def forward_decode(self, batch: Batch):
-        input_metadata = InputMetadata.create(
-            self,
-            forward_mode=ForwardMode.DECODE,
-            tp_size=self.tp_size,
-            req_pool_indices=batch.req_pool_indices,
-            seq_lens=batch.seq_lens,
-            prefix_lens=batch.prefix_lens,
-            position_ids_offsets=batch.position_ids_offsets,
-            out_cache_loc=batch.out_cache_loc,
-            out_cache_cont_start=batch.out_cache_cont_start,
-            out_cache_cont_end=batch.out_cache_cont_end,
-            top_logprobs_nums=batch.top_logprobs_nums,
-            return_logprob=batch.return_logprob,
-        )
-        return self.model.forward(
-            batch.input_ids, input_metadata.positions, input_metadata
-        )
-
-    @torch.inference_mode()
-    def forward_extend_multi_modal(self, batch: Batch):
-        input_metadata = InputMetadata.create(
-            self,
-            forward_mode=ForwardMode.EXTEND,
-            tp_size=self.tp_size,
-            req_pool_indices=batch.req_pool_indices,
-            seq_lens=batch.seq_lens,
-            prefix_lens=batch.prefix_lens,
-            position_ids_offsets=batch.position_ids_offsets,
-            out_cache_loc=batch.out_cache_loc,
-            top_logprobs_nums=batch.top_logprobs_nums,
-            return_logprob=batch.return_logprob,
-        )
-        return self.model.forward(
-            batch.input_ids,
-            input_metadata.positions,
-            input_metadata,
-            batch.pixel_values,
-            batch.image_sizes,
-            batch.image_offsets,
-        )
-
-    def forward(self, batch: Batch, forward_mode: ForwardMode):
-        if self.is_multimodal_model and forward_mode == ForwardMode.EXTEND:
-            return self.forward_extend_multi_modal(batch)
-        elif forward_mode == ForwardMode.DECODE:
-            return self.forward_decode(batch)
-        elif forward_mode == ForwardMode.EXTEND:
-            return self.forward_extend(batch)
-        elif forward_mode == ForwardMode.PREFILL:
-            return self.forward_prefill(batch)
-        else:
-            raise ValueError(f"Invaid forward mode: {forward_mode}")
-
-
-@lru_cache()
-def import_model_classes():
-    model_arch_name_to_cls = {}
-    package_name = "sglang.srt.models"
-    package = importlib.import_module(package_name)
-    for _, name, ispkg in pkgutil.iter_modules(package.__path__, package_name + "."):
-        if not ispkg:
-            module = importlib.import_module(name)
-            if hasattr(module, "EntryClass"):
-                entry = module.EntryClass
-                if isinstance(entry, list): # To support multiple model classes in one module
-                    for cls in entry:
-                        model_arch_name_to_cls[cls.__name__] = cls
-                else:
-                    model_arch_name_to_cls[entry.__name__] = entry
-    return model_arch_name_to_cls
-
-
-def load_model_cls_srt(model_arch: str) -> Optional[Type[nn.Module]]:
-    model_arch_name_to_cls = import_model_classes()
-    if model_arch not in model_arch_name_to_cls:
-        raise ValueError(
-            f"Unsupported architectures: {model_arch}. "
-            f"Supported list: {list(model_arch_name_to_cls.keys())}"
-        )
-    return model_arch_name_to_cls[model_arch]
-
-
-# Monkey patch model loader
-setattr(ModelRegistry, "load_model_cls", load_model_cls_srt)

sglang/srt/managers/router/radix_cache.py

@@ -1,267 +0,0 @@
-import heapq
-import time
-from collections import defaultdict
-
-import torch
-
-
-class TreeNode:
-    def __init__(self):
-        self.children = defaultdict(TreeNode)
-        self.parent = None
-        self.key = None
-        self.value = None
-        self.lock_ref = 0
-        self.last_access_time = time.time()
-
-    def __lt__(self, other: "TreeNode"):
-        return self.last_access_time < other.last_access_time
-
-
-def _key_match(key0, key1):
-    i = 0
-    for k0, k1 in zip(key0, key1):
-        if k0 != k1:
-            break
-        i += 1
-    return i
-
-
-class RadixCache:
-    def __init__(self, req_to_token_pool, token_to_kv_pool, disable: bool = False):
-        self.req_to_token_pool = req_to_token_pool
-        self.token_to_kv_pool = token_to_kv_pool
-        self.disable = disable
-        self.reset()
-
-    ##### Public API #####
-
-    def reset(self):
-        self.root_node = TreeNode()
-        self.root_node.key = []
-        self.root_node.value = []
-        self.root_node.lock_ref = 1
-        self.evictable_size_ = 0
-
-    def match_prefix(self, key):
-        if self.disable:
-            return [], self.root_node
-
-        value = []
-        last_node = [self.root_node]
-        self._match_prefix_helper(self.root_node, key, value, last_node)
-        if value:
-            value = torch.concat(value)
-        else:
-            value = torch.tensor([], dtype=torch.int64)
-        return value, last_node[0]
-
-    def insert(self, key, value=None):
-        if self.disable:
-            return 0
-
-        if value is None:
-            value = [x for x in key]
-        return self._insert_helper(self.root_node, key, value)
-
-    def cache_req(
-        self,
-        token_ids,
-        last_uncached_pos,
-        req_pool_idx,
-        del_in_memory_pool=True,
-        old_last_node=None,
-    ):
-        # Insert the request into radix cache
-        indices = self.req_to_token_pool.req_to_token[req_pool_idx, : len(token_ids)]
-        new_prefix_len = self.insert(token_ids, indices.clone())
-
-        if self.disable:
-            if del_in_memory_pool:
-                self.token_to_kv_pool.dec_refs(indices)
-            else:
-                return torch.tensor([], dtype=torch.int64), self.root_node
-
-        # Radix Cache takes one ref in memory pool
-        self.token_to_kv_pool.dec_refs(indices[last_uncached_pos:new_prefix_len])
-
-        if del_in_memory_pool:
-            self.req_to_token_pool.free(req_pool_idx)
-        else:
-            cached_indices, new_last_node = self.match_prefix(token_ids)
-            assert len(cached_indices) == len(token_ids)
-
-            self.req_to_token_pool.req_to_token[
-                req_pool_idx, last_uncached_pos : len(cached_indices)
-            ] = cached_indices[last_uncached_pos:]
-            self.dec_lock_ref(old_last_node)
-            self.inc_lock_ref(new_last_node)
-            return cached_indices, new_last_node
-
-    def pretty_print(self):
-        self._print_helper(self.root_node, 0)
-        print(f"#tokens: {self.total_size()}")
-
-    def total_size(self):
-        return self._total_size_helper(self.root_node)
-
-    def evict(self, num_tokens, evict_callback):
-        if self.disable:
-            return
-
-        leaves = self._collect_leaves()
-        heapq.heapify(leaves)
-
-        num_evicted = 0
-        while num_evicted < num_tokens and len(leaves):
-            x = heapq.heappop(leaves)
-
-            if x == self.root_node:
-                break
-            if x.lock_ref > 0:
-                continue
-
-            num_evicted += evict_callback(x.value)
-            self._delete_leaf(x)
-
-            if len(x.parent.children) == 0:
-                heapq.heappush(leaves, x.parent)
-
-    def inc_lock_ref(self, node: TreeNode):
-        delta = 0
-        while node != self.root_node:
-            if node.lock_ref == 0:
-                self.evictable_size_ -= len(node.value)
-                delta -= len(node.value)
-            node.lock_ref += 1
-            node = node.parent
-        return delta
-
-    def dec_lock_ref(self, node: TreeNode):
-        delta = 0
-        while node != self.root_node:
-            if node.lock_ref == 1:
-                self.evictable_size_ += len(node.value)
-                delta += len(node.value)
-            node.lock_ref -= 1
-            node = node.parent
-        return delta
-
-    def evictable_size(self):
-        return self.evictable_size_
-
-    ##### Internal Helper Functions #####
-
-    def _match_prefix_helper(self, node, key, value, last_node):
-        node.last_access_time = time.time()
-        if len(key) == 0:
-            return
-
-        if key[0] in node.children.keys():
-            child = node.children[key[0]]
-            prefix_len = _key_match(child.key, key)
-            if prefix_len < len(child.key):
-                new_node = self._split_node(child.key, child, prefix_len)
-                value.append(new_node.value)
-                last_node[0] = new_node
-            else:
-                value.append(child.value)
-                last_node[0] = child
-                self._match_prefix_helper(child, key[prefix_len:], value, last_node)
-
-    def _split_node(self, key, child: TreeNode, split_len):
-        # new_node -> child
-        new_node = TreeNode()
-        new_node.children = {key[split_len:][0]: child}
-        new_node.parent = child.parent
-        new_node.lock_ref = child.lock_ref
-        new_node.key = child.key[:split_len]
-        new_node.value = child.value[:split_len]
-        child.parent = new_node
-        child.key = child.key[split_len:]
-        child.value = child.value[split_len:]
-        new_node.parent.children[key[:split_len][0]] = new_node
-        return new_node
-
-    def _insert_helper(self, node, key, value):
-        node.last_access_time = time.time()
-        if len(key) == 0:
-            return 0
-
-        if key[0] in node.children.keys():
-            child = node.children[key[0]]
-            prefix_len = _key_match(child.key, key)
-
-            if prefix_len == len(child.key):
-                if prefix_len == len(key):
-                    return prefix_len
-                else:
-                    key = key[prefix_len:]
-                    value = value[prefix_len:]
-                    return prefix_len + self._insert_helper(child, key, value)
-
-            new_node = self._split_node(child.key, child, prefix_len)
-            return prefix_len + self._insert_helper(
-                new_node, key[prefix_len:], value[prefix_len:]
-            )
-
-        if len(key):
-            new_node = TreeNode()
-            new_node.parent = node
-            new_node.key = key
-            new_node.value = value
-            node.children[key[0]] = new_node
-            self.evictable_size_ += len(value)
-        return 0
-
-    def _print_helper(self, node: TreeNode, indent):
-        for _, child in node.children.items():
-            print(" " * indent, len(child.key), child.key[:10], f"r={child.lock_ref}")
-            self._print_helper(child, indent=indent + 2)
-
-    def _delete_leaf(self, node):
-        for k, v in node.parent.children.items():
-            if v == node:
-                break
-        del node.parent.children[k]
-        self.evictable_size_ -= len(node.key)
-
-    def _total_size_helper(self, node):
-        x = len(node.value)
-        for child in node.children.values():
-            x += self._total_size_helper(child)
-        return x
-
-    def _collect_leaves(self):
-        ret_list = []
-
-        def dfs_(cur_node):
-            if len(cur_node.children) == 0:
-                ret_list.append(cur_node)
-
-            for x in cur_node.children.values():
-                dfs_(x)
-
-        dfs_(self.root_node)
-        return ret_list
-
-
-if __name__ == "__main__":
-    tree = RadixCache(None, None, False)
-
-    tree.insert("Hello")
-    tree.insert("Hello")
-    tree.insert("Hello_L.A.!")
-    # tree.insert("Hello_world! Happy")
-    # tree.insert("I love you!")
-    tree.pretty_print()
-
-    # print(tree.match_prefix("I love you! aha"))
-
-    # def evict_callback(x):
-    #    print("evict", x)
-    #    return len(x)
-
-    # tree.evict(5, evict_callback)
-    # tree.evict(10, evict_callback)
-    # tree.pretty_print()