sglang 0.1.12__tar.gz → 0.1.14__tar.gz

{sglang-0.1.12/sglang.egg-info → sglang-0.1.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sglang
-Version: 0.1.12
+Version: 0.1.14
 Summary: A structured generation langauge for LLMs.
 License:                                  Apache License
                                    Version 2.0, January 2004
@@ -221,7 +221,7 @@ Requires-Dist: torch; extra == "srt"
 Requires-Dist: uvloop; extra == "srt"
 Requires-Dist: uvicorn; extra == "srt"
 Requires-Dist: zmq; extra == "srt"
-Requires-Dist: vllm>=0.2.5; extra == "srt"
+Requires-Dist: vllm>=0.3.3; extra == "srt"
 Requires-Dist: interegular; extra == "srt"
 Requires-Dist: lark; extra == "srt"
 Requires-Dist: numba; extra == "srt"
@@ -235,14 +235,19 @@ Provides-Extra: openai
 Requires-Dist: openai>=1.0; extra == "openai"
 Requires-Dist: numpy; extra == "openai"
 Provides-Extra: anthropic
-Requires-Dist: anthropic; extra == "anthropic"
+Requires-Dist: anthropic>=0.20.0; extra == "anthropic"
 Requires-Dist: numpy; extra == "anthropic"
 Provides-Extra: all
 Requires-Dist: sglang[srt]; extra == "all"
 Requires-Dist: sglang[openai]; extra == "all"
 Requires-Dist: sglang[anthropic]; extra == "all"
 
-# SGLang
+<div align="center">
+<img src="assets/logo.png" alt="logo" width="400"></img>
+</div>
+
+--------------------------------------------------------------------------------
+
 | [**Blog**](https://lmsys.org/blog/2024-01-17-sglang/) | [**Paper**](https://arxiv.org/abs/2312.07104) |
 
 SGLang is a structured generation language designed for large language models (LLMs).
@@ -254,7 +259,7 @@ The core features of SGLang include:
 
 ## News
 - [2024/02] 🔥 SGLang enables **3x faster JSON decoding** with compressed finite state machine ([blog](https://lmsys.org/blog/2024-02-05-compressed-fsm/)).
-- [2024/01] 🔥 SGLang powers the serving of the offical **LLaVA v1.6** release demo ([usage](https://github.com/haotian-liu/LLaVA?tab=readme-ov-file#demo)).
+- [2024/01] 🔥 SGLang powers the serving of the official **LLaVA v1.6** release demo ([usage](https://github.com/haotian-liu/LLaVA?tab=readme-ov-file#demo)).
 - [2024/01] SGLang provides up to **5x faster inference** with RadixAttention ([blog](https://lmsys.org/blog/2024-01-17-sglang/)).
 
 ## Contents
@@ -496,7 +501,7 @@ def text_qa(s, question):
     s += "Q: " + question + "\n"
     s += "A:" + sgl.gen("answer", stop="\n")
 
-states = text_qa.run(
+state = text_qa.run(
     question="What is the capital of France?",
     temperature=0.1,
     stream=True
@@ -608,8 +613,13 @@ python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port
 - Mistral
 - Mixtral
 - Qwen / Qwen 2
+- Gemma
+  - Please add a new flag `--attention-reduce-in-fp32` to avoid some precision errors.
+  - `python -m sglang.launch_server --model-path google/gemma-7b-it --port 30000 --attention-reduce-in-fp32`
 - LLaVA
   - `python3 -m sglang.launch_server --model-path liuhaotian/llava-v1.5-7b --tokenizer-path llava-hf/llava-1.5-7b-hf --chat-template vicuna_v1.1 --port 30000`
+  - `python3 -m sglang.launch_server --model-path liuhaotian/llava-v1.6-vicuna-7b --tokenizer-path llava-hf/llava-1.5-7b-hf --chat-template vicuna_v1.1 --port 30000`
+  - `python3 -m sglang.launch_server --model-path liuhaotian/llava-v1.6-34b --tokenizer-path liuhaotian/llava-v1.6-34b-tokenizer --port 3000`
 - Yi-VL
   - see [srt_example_yi_vl.py](examples/quick_start/srt_example_yi_vl.py).
 - AWQ/GPTQ quantization

{sglang-0.1.12 → sglang-0.1.14}/README.md

@@ -1,4 +1,9 @@
-# SGLang
+<div align="center">
+<img src="assets/logo.png" alt="logo" width="400"></img>
+</div>
+
+--------------------------------------------------------------------------------
+
 | [**Blog**](https://lmsys.org/blog/2024-01-17-sglang/) | [**Paper**](https://arxiv.org/abs/2312.07104) |
 
 SGLang is a structured generation language designed for large language models (LLMs).
@@ -10,7 +15,7 @@ The core features of SGLang include:
 
 ## News
 - [2024/02] 🔥 SGLang enables **3x faster JSON decoding** with compressed finite state machine ([blog](https://lmsys.org/blog/2024-02-05-compressed-fsm/)).
-- [2024/01] 🔥 SGLang powers the serving of the offical **LLaVA v1.6** release demo ([usage](https://github.com/haotian-liu/LLaVA?tab=readme-ov-file#demo)).
+- [2024/01] 🔥 SGLang powers the serving of the official **LLaVA v1.6** release demo ([usage](https://github.com/haotian-liu/LLaVA?tab=readme-ov-file#demo)).
 - [2024/01] SGLang provides up to **5x faster inference** with RadixAttention ([blog](https://lmsys.org/blog/2024-01-17-sglang/)).
 
 ## Contents
@@ -252,7 +257,7 @@ def text_qa(s, question):
     s += "Q: " + question + "\n"
     s += "A:" + sgl.gen("answer", stop="\n")
 
-states = text_qa.run(
+state = text_qa.run(
     question="What is the capital of France?",
     temperature=0.1,
     stream=True
@@ -364,8 +369,13 @@ python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port
 - Mistral
 - Mixtral
 - Qwen / Qwen 2
+- Gemma
+  - Please add a new flag `--attention-reduce-in-fp32` to avoid some precision errors.
+  - `python -m sglang.launch_server --model-path google/gemma-7b-it --port 30000 --attention-reduce-in-fp32`
 - LLaVA
   - `python3 -m sglang.launch_server --model-path liuhaotian/llava-v1.5-7b --tokenizer-path llava-hf/llava-1.5-7b-hf --chat-template vicuna_v1.1 --port 30000`
+  - `python3 -m sglang.launch_server --model-path liuhaotian/llava-v1.6-vicuna-7b --tokenizer-path llava-hf/llava-1.5-7b-hf --chat-template vicuna_v1.1 --port 30000`
+  - `python3 -m sglang.launch_server --model-path liuhaotian/llava-v1.6-34b --tokenizer-path liuhaotian/llava-v1.6-34b-tokenizer --port 3000`
 - Yi-VL
   - see [srt_example_yi_vl.py](examples/quick_start/srt_example_yi_vl.py).
 - AWQ/GPTQ quantization

{sglang-0.1.12 → sglang-0.1.14}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "sglang"
-version = "0.1.12"
+version = "0.1.14"
 description = "A structured generation langauge for LLMs." 
 readme = "README.md"
 requires-python = ">=3.8"
@@ -19,10 +19,10 @@ dependencies = [
 
 [project.optional-dependencies]
 srt = ["aiohttp", "fastapi", "psutil", "rpyc", "torch", "uvloop", "uvicorn",
-       "zmq", "vllm>=0.2.5", "interegular", "lark", "numba",
+       "zmq", "vllm>=0.3.3", "interegular", "lark", "numba",
        "pydantic", "referencing", "diskcache", "cloudpickle", "pillow", "outlines>=0.0.27"]
 openai = ["openai>=1.0", "numpy"]
-anthropic = ["anthropic", "numpy"]
+anthropic = ["anthropic>=0.20.0", "numpy"]
 all = ["sglang[srt]", "sglang[openai]", "sglang[anthropic]"]
 
 [project.urls]

{sglang-0.1.12 → sglang-0.1.14}/sglang/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.1.12"
+__version__ = "0.1.14"
 
 from sglang.api import *
 from sglang.global_config import global_config

{sglang-0.1.12 → sglang-0.1.14}/sglang/api.py

@@ -44,6 +44,20 @@ def set_default_backend(backend: BaseBackend):
     global_config.default_backend = backend
 
 
+def flush_cache(backend: BaseBackend = None):
+    backend = backend or global_config.default_backend
+    if backend is None:
+        return False
+    return backend.flush_cache()
+
+
+def get_server_args(backend: BaseBackend = None):
+    backend = backend or global_config.default_backend
+    if backend is None:
+        return None
+    return backend.get_server_args()
+
+
 def gen(
     name: Optional[str] = None,
     max_tokens: Optional[int] = None,

{sglang-0.1.12 → sglang-0.1.14}/sglang/backend/anthropic.py

@@ -30,13 +30,17 @@ class Anthropic(BaseBackend):
         s: StreamExecutor,
         sampling_params: SglSamplingParams,
     ):
-        prompt = s.text_
-        ret = anthropic.Anthropic().completions.create(
+        if s.messages_:
+            messages = s.messages_
+        else:
+            messages = [{"role": "user", "content": s.text_}]
+
+        ret = anthropic.Anthropic().messages.create(
             model=self.model_name,
-            prompt=prompt,
+            messages=messages,
             **sampling_params.to_anthropic_kwargs(),
         )
-        comp = ret.completion
+        comp = ret.content[0].text
 
         return comp, {}
 
@@ -45,13 +49,15 @@ class Anthropic(BaseBackend):
         s: StreamExecutor,
         sampling_params: SglSamplingParams,
     ):
-        prompt = s.text_
-        generator = anthropic.Anthropic().completions.create(
+        if s.messages_:
+            messages = s.messages_
+        else:
+            messages = [{"role": "user", "content": s.text_}]
+
+        with anthropic.Anthropic().messages.stream(
             model=self.model_name,
-            prompt=prompt,
-            stream=True,
+            messages=messages,
             **sampling_params.to_anthropic_kwargs(),
-        )
-
-        for ret in generator:
-            yield ret.completion, {}
+        ) as stream:
+            for text in stream.text_stream:
+                yield text, {}

{sglang-0.1.12 → sglang-0.1.14}/sglang/backend/base_backend.py

@@ -72,3 +72,9 @@ class BaseBackend:
 
     def shutdown(self):
         pass
+
+    def flush_cache(self):
+        pass
+
+    def get_server_args(self):
+        pass

{sglang-0.1.12 → sglang-0.1.14}/sglang/backend/openai.py

@@ -4,7 +4,7 @@ from typing import Callable, List, Optional, Union
 
 import numpy as np
 from sglang.backend.base_backend import BaseBackend
-from sglang.lang.chat_template import get_chat_template
+from sglang.lang.chat_template import ChatTemplate, get_chat_template_by_model_path
 from sglang.lang.interpreter import StreamExecutor
 from sglang.lang.ir import SglSamplingParams
 
@@ -41,23 +41,45 @@ INSTRUCT_MODEL_NAMES = [
 
 
 class OpenAI(BaseBackend):
-    def __init__(self, model_name, *args, **kwargs):
+    def __init__(
+        self,
+        model_name: str,
+        is_chat_model: Optional[bool] = None,
+        chat_template: Optional[ChatTemplate] = None,
+        is_azure: bool = False,
+        *args,
+        **kwargs,
+    ):
         super().__init__()
 
         if isinstance(openai, Exception):
             raise openai
 
-        self.client = openai.OpenAI(*args, **kwargs)
+        if is_azure:
+            self.client = openai.AzureOpenAI(*args, **kwargs)
+        else:
+            self.client = openai.OpenAI(*args, **kwargs)
+
         self.model_name = model_name
-        self.tokenizer = tiktoken.encoding_for_model(model_name)
+        try:
+            self.tokenizer = tiktoken.encoding_for_model(model_name)
+        except KeyError:
+            self.tokenizer = tiktoken.get_encoding("cl100k_base")
         self.logit_bias_int = create_logit_bias_int(self.tokenizer)
 
-        if model_name in INSTRUCT_MODEL_NAMES:
-            self.is_chat_model = False
+        self.chat_template = chat_template or get_chat_template_by_model_path(
+            model_name
+        )
+
+        if is_chat_model is not None:
+            self.is_chat_model = is_chat_model
         else:
-            self.is_chat_model = True
+            if model_name in INSTRUCT_MODEL_NAMES:
+                self.is_chat_model = False
+            else:
+                self.is_chat_model = True
 
-        self.chat_template = get_chat_template("default")
+        self.chat_begin_str = self.chat_template.role_prefix_and_suffix["assistant"][0]
 
     def get_chat_template(self):
         return self.chat_template
@@ -69,7 +91,7 @@ class OpenAI(BaseBackend):
     ):
         if sampling_params.dtype is None:
             if self.is_chat_model:
-                if not s.text_.endswith("ASSISTANT:"):
+                if not s.text_.endswith(self.chat_begin_str):
                     raise RuntimeError(
                         "This use case is not supported. "
                         "For OpenAI chat models, sgl.gen must be right after sgl.assistant"
@@ -122,7 +144,11 @@ class OpenAI(BaseBackend):
     ):
         if sampling_params.dtype is None:
             if self.is_chat_model:
-                assert s.text_.endswith("ASSISTANT:")
+                if not s.text_.endswith(self.chat_begin_str):
+                    raise RuntimeError(
+                        "This use case is not supported. "
+                        "For OpenAI chat models, sgl.gen must be right after sgl.assistant"
+                    )
                 prompt = s.messages_
             else:
                 prompt = s.text_
@@ -137,7 +163,7 @@ class OpenAI(BaseBackend):
             )
             return generator
         else:
-            raise ValueError(f"Unknown dtype: {dtype}")
+            raise ValueError(f"Unknown dtype: {sampling_params.dtype}")
 
     def select(
         self,
@@ -241,7 +267,10 @@ def openai_completion_stream(client, retries=3, is_chat=None, prompt=None, **kwa
                     messages=prompt, stream=True, **kwargs
                 )
                 for ret in generator:
-                    content = ret.choices[0].delta.content
+                    try:
+                        content = ret.choices[0].delta.content
+                    except IndexError:
+                        content = None
                     yield content or "", {}
             else:
                 generator = client.completions.create(

{sglang-0.1.12 → sglang-0.1.14}/sglang/backend/runtime_endpoint.py

@@ -12,15 +12,26 @@ from sglang.utils import encode_image_base64, find_printable_text, http_request
 
 
 class RuntimeEndpoint(BaseBackend):
-    def __init__(self, base_url, auth_token=None):
+    def __init__(
+        self,
+        base_url: str,
+        auth_token: Optional[str] = None,
+        api_key: Optional[str] = None,
+        verify: Optional[str] = None,
+    ):
         super().__init__()
         self.support_concate_and_append = True
 
         self.base_url = base_url
         self.auth_token = auth_token
+        self.api_key = api_key
+        self.verify = verify
 
         res = http_request(
-            self.base_url + "/get_model_info", auth_token=self.auth_token
+            self.base_url + "/get_model_info",
+            auth_token=self.auth_token,
+            api_key=self.api_key,
+            verify=self.verify,
         )
         assert res.status_code == 200
         self.model_info = res.json()
@@ -32,6 +43,22 @@ class RuntimeEndpoint(BaseBackend):
     def get_model_name(self):
         return self.model_info["model_path"]
 
+    def flush_cache(self):
+        res = http_request(
+            self.base_url + "/flush_cache",
+            auth_token=self.auth_token,
+            verify=self.verify,
+        )
+        return res.status_code == 200
+
+    def get_server_args(self):
+        res = http_request(
+            self.base_url + "/get_server_args",
+            auth_token=self.auth_token,
+            verify=self.verify,
+        )
+        return res.json()
+
     def get_chat_template(self):
         return self.chat_template
 
@@ -40,6 +67,8 @@ class RuntimeEndpoint(BaseBackend):
             self.base_url + "/generate",
             json={"text": prefix_str, "sampling_params": {"max_new_tokens": 0}},
             auth_token=self.auth_token,
+            api_key=self.api_key,
+            verify=self.verify,
         )
         assert res.status_code == 200
 
@@ -48,6 +77,8 @@ class RuntimeEndpoint(BaseBackend):
             self.base_url + "/generate",
             json={"text": s.text_, "sampling_params": {"max_new_tokens": 0}},
             auth_token=self.auth_token,
+            api_key=self.api_key,
+            verify=self.verify,
         )
         assert res.status_code == 200
 
@@ -55,7 +86,11 @@ class RuntimeEndpoint(BaseBackend):
         data = {"text": s.text_, "sampling_params": {"max_new_tokens": 0}}
         self._add_images(s, data)
         res = http_request(
-            self.base_url + "/generate", json=data, auth_token=self.auth_token
+            self.base_url + "/generate",
+            json=data,
+            auth_token=self.auth_token,
+            api_key=self.api_key,
+            verify=self.verify,
         )
         assert res.status_code == 200
 
@@ -87,7 +122,11 @@ class RuntimeEndpoint(BaseBackend):
         self._add_images(s, data)
 
         res = http_request(
-            self.base_url + "/generate", json=data, auth_token=self.auth_token
+            self.base_url + "/generate",
+            json=data,
+            auth_token=self.auth_token,
+            api_key=self.api_key,
+            verify=self.verify,
         )
         obj = res.json()
         comp = obj["text"]
@@ -126,6 +165,8 @@ class RuntimeEndpoint(BaseBackend):
             json=data,
             stream=True,
             auth_token=self.auth_token,
+            api_key=self.api_key,
+            verify=self.verify,
         )
         pos = 0
 
@@ -157,7 +198,11 @@ class RuntimeEndpoint(BaseBackend):
         data = {"text": s.text_, "sampling_params": {"max_new_tokens": 0}}
         self._add_images(s, data)
         res = http_request(
-            self.base_url + "/generate", json=data, auth_token=self.auth_token
+            self.base_url + "/generate",
+            json=data,
+            auth_token=self.auth_token,
+            api_key=self.api_key,
+            verify=self.verify,
         )
         assert res.status_code == 200
         prompt_len = res.json()["meta_info"]["prompt_tokens"]
@@ -171,7 +216,11 @@ class RuntimeEndpoint(BaseBackend):
         }
         self._add_images(s, data)
         res = http_request(
-            self.base_url + "/generate", json=data, auth_token=self.auth_token
+            self.base_url + "/generate",
+            json=data,
+            auth_token=self.auth_token,
+            api_key=self.api_key,
+            verify=self.verify,
         )
         assert res.status_code == 200
         obj = res.json()
@@ -188,6 +237,8 @@ class RuntimeEndpoint(BaseBackend):
             self.base_url + "/concate_and_append_request",
             json={"src_rids": src_rids, "dst_rid": dst_rid},
             auth_token=self.auth_token,
+            api_key=self.api_key,
+            verify=self.verify,
         )
         assert res.status_code == 200
 

{sglang-0.1.12 → sglang-0.1.14}/sglang/lang/chat_template.py

@@ -12,42 +12,43 @@ class ChatTemplateStyle(Enum):
 class ChatTemplate:
     name: str
     default_system_prompt: str
-    role_prefix_and_suffix: Dict[str, Tuple[str]]
+    role_prefix_and_suffix: Dict[str, Tuple[str, str]]
     stop_str: List[str] = ()
     image_token: str = "<image>"
     style: ChatTemplateStyle = ChatTemplateStyle.PLAIN
 
-    def get_prefix_and_suffix(self, role, hist_messages):
-        if self.style == ChatTemplateStyle.PLAIN:
-            return self.role_prefix_and_suffix[role]
-        elif self.style == ChatTemplateStyle.LLAMA2:
-            if len(hist_messages) == 0 and role == "system":
-                return (
-                    self.role_prefix_and_suffix["user"][0]
-                    + self.role_prefix_and_suffix["system"][0],
-                    self.role_prefix_and_suffix["system"][1],
+    def get_prefix_and_suffix(
+        self, role: str, hist_messages: List[Dict]
+    ) -> Tuple[str, str]:
+        prefix, suffix = self.role_prefix_and_suffix.get(role, ("", ""))
+
+        if self.style == ChatTemplateStyle.LLAMA2:
+            if role == "system" and not hist_messages:
+                user_prefix, _ = self.role_prefix_and_suffix.get("user", ("", ""))
+                system_prefix, system_suffix = self.role_prefix_and_suffix.get(
+                    "system", ("", "")
                 )
+                return (user_prefix + system_prefix, system_suffix)
             elif (
-                len(hist_messages) == 1
-                and role == "user"
+                role == "user"
+                and len(hist_messages) == 1
                 and hist_messages[0]["content"] is not None
             ):
-                return ("", self.role_prefix_and_suffix["user"][1])
-            return self.role_prefix_and_suffix[role]
-        else:
-            raise ValueError(f"Invalid style: {self.style}")
+                return ("", suffix)
+
+        return prefix, suffix
 
-    def get_prompt(self, messages):
+    def get_prompt(self, messages: List[Dict]) -> str:
         prompt = ""
-        for i in range(len(messages)):
-            role, content = messages[i]["role"], messages[i]["content"]
+        for i, message in enumerate(messages):
+            role, content = message["role"], message["content"]
             if role == "system" and content is None:
                 content = self.default_system_prompt
                 if content is None:
                     continue
 
             prefix, suffix = self.get_prefix_and_suffix(role, messages[:i])
-            prompt += prefix + content + suffix
+            prompt += f"{prefix}{content}{suffix}"
         return prompt
 
 
@@ -106,9 +107,9 @@ register_chat_template(
         name="chatml",
         default_system_prompt=None,
         role_prefix_and_suffix={
-            "system": ("<|im_start|>system\n", "\n<|im_end|>\n"),
-            "user": ("<|im_start|>user\n", "\n<|im_end|>\n"),
-            "assistant": ("<|im_start|>assistant\n", "\n<|im_end|>\n"),
+            "system": ("<|im_start|>system\n", "<|im_end|>\n"),
+            "user": ("<|im_start|>user\n", "<|im_end|>\n"),
+            "assistant": ("<|im_start|>assistant\n", "<|im_end|>\n"),
         },
         style=ChatTemplateStyle.PLAIN,
         stop_str=("<|im_end|>",),
@@ -121,9 +122,9 @@ register_chat_template(
         name="chatml-llava",
         default_system_prompt="Answer the questions.",
         role_prefix_and_suffix={
-            "system": ("<|im_start|>system\n", "\n<|im_end|>\n"),
-            "user": ("<|im_start|>user\n", "\n<|im_end|>\n"),
-            "assistant": ("<|im_start|>assistant\n", "\n<|im_end|>\n"),
+            "system": ("<|im_start|>system\n", "<|im_end|>\n"),
+            "user": ("<|im_start|>user\n", "<|im_end|>\n"),
+            "assistant": ("<|im_start|>assistant\n", "<|im_end|>\n"),
         },
         style=ChatTemplateStyle.PLAIN,
         stop_str=("<|im_end|>",),
@@ -178,6 +179,19 @@ register_chat_template(
     )
 )
 
+register_chat_template(
+    ChatTemplate(
+        name="gemma-it",
+        default_system_prompt=None,
+        role_prefix_and_suffix={
+            "system": ("", ""),
+            "user": ("<start_of_turn>user\n", "<end_of_turn>\n"),
+            "assistant": ("<start_of_turn>model\n", "<end_of_turn>\n"),
+        },
+        style=ChatTemplateStyle.PLAIN,
+    )
+)
+
 
 @register_chat_template_matching_function
 def match_vicuna(model_path: str):
@@ -218,6 +232,13 @@ def match_chat_yi(model_path: str):
         return get_chat_template("yi")
 
 
+@register_chat_template_matching_function
+def match_gemma_it(model_path: str):
+    model_path = model_path.lower()
+    if "gemma" in model_path and "it" in model_path:
+        return get_chat_template("gemma-it")
+
+
 if __name__ == "__main__":
     messages = [
         {"role": "system", "content": None},  # None means default

{sglang-0.1.12 → sglang-0.1.14}/sglang/lang/interpreter.py

@@ -245,6 +245,9 @@ class StreamExecutor:
             self.variable_event[name].wait()
         return self.variables[name]
 
+    def set_var(self, name, value):
+        self.variables[name] = value
+
     def get_meta_info(self, name):
         if name in self.variable_event:
             self.variable_event[name].wait()
@@ -583,6 +586,10 @@ class StreamExecutor:
         if self.chat_template.stop_str:
             if not clone:
                 clone = self.default_sampling_para.clone()
+            if clone.stop == ():
+                clone.stop = []
+            elif isinstance(clone.stop, str):
+                clone.stop = [clone.stop]
             clone.stop += self.chat_template.stop_str
 
         return clone or self.default_sampling_para
@@ -679,7 +686,7 @@ class ProgramState:
             if var_name is None:
                 yield self.text()
             else:
-                yield self.get_var(name)
+                yield self.get_var(var_name)
 
     async def text_async_iter(
         self, var_name: Optional[str] = None, return_meta_data: bool = False
@@ -717,11 +724,14 @@ class ProgramState:
             if var_name is None:
                 yield self.text()
             else:
-                yield self.get_var(name)
+                yield self.get_var(var_name)
 
     def get_var(self, name):
         return self.stream_executor.get_var(name)
 
+    def set_var(self, name, value):
+        return self.stream_executor.set_var(name, value)
+
     def get_meta_info(self, name):
         return self.stream_executor.get_meta_info(name)
 
@@ -732,6 +742,9 @@ class ProgramState:
     def __getitem__(self, name):
         return self.get_var(name)
 
+    def __setitem__(self, name, value):
+        self.set_var(name, value)
+
     def __del__(self):
         self.stream_executor.end()
 

{sglang-0.1.12 → sglang-0.1.14}/sglang/lang/ir.py

@@ -73,7 +73,7 @@ class SglSamplingParams:
                 "Regular expression is not supported in the Anthropic backend."
             )
         return {
-            "max_tokens_to_sample": self.max_new_tokens,
+            "max_tokens": self.max_new_tokens,
             "stop_sequences": (
                 self.stop if isinstance(self.stop, (list, tuple)) else [self.stop]
             ),