sglang 0.3.3.post1__py3-none-any.whl → 0.3.4__py3-none-any.whl

sglang/test/few_shot_gsm8k_engine.py

@@ -0,0 +1,144 @@
+import argparse
+import ast
+import asyncio
+import json
+import re
+import time
+
+import numpy as np
+
+import sglang as sgl
+from sglang.api import set_default_backend
+from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint
+from sglang.utils import download_and_cache_file, dump_state_text, read_jsonl
+
+INVALID = -9999999
+
+
+def get_one_example(lines, i, include_answer):
+    ret = "Question: " + lines[i]["question"] + "\nAnswer:"
+    if include_answer:
+        ret += " " + lines[i]["answer"]
+    return ret
+
+
+def get_few_shot_examples(lines, k):
+    ret = ""
+    for i in range(k):
+        ret += get_one_example(lines, i, True) + "\n\n"
+    return ret
+
+
+def get_answer_value(answer_str):
+    answer_str = answer_str.replace(",", "")
+    numbers = re.findall(r"\d+", answer_str)
+    if len(numbers) < 1:
+        return INVALID
+    try:
+        return ast.literal_eval(numbers[-1])
+    except SyntaxError:
+        return INVALID
+
+
+async def concurrent_generate(engine, prompts, sampling_param):
+    tasks = []
+    for prompt in prompts:
+        tasks.append(asyncio.create_task(engine.async_generate(prompt, sampling_param)))
+
+    outputs = await asyncio.gather(*tasks)
+    return outputs
+
+
+def run_eval(args):
+    # Select backend
+    engine = sgl.Engine(model_path=args.model_path, log_level="error")
+
+    if args.local_data_path is None:
+        # Read data
+        url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
+        filename = download_and_cache_file(url)
+    else:
+        filename = args.local_data_path
+
+    lines = list(read_jsonl(filename))
+
+    # Construct prompts
+    num_questions = args.num_questions
+    num_shots = args.num_shots
+    few_shot_examples = get_few_shot_examples(lines, num_shots)
+
+    questions = []
+    labels = []
+    for i in range(len(lines[:num_questions])):
+        questions.append(get_one_example(lines, i, False))
+        labels.append(get_answer_value(lines[i]["answer"]))
+    assert all(l != INVALID for l in labels)
+    arguments = [{"question": q} for q in questions]
+
+    # construct the prompts
+    prompts = []
+    for i, arg in enumerate(arguments):
+        q = arg["question"]
+        prompt = few_shot_examples + q
+        prompts.append(prompt)
+
+    sampling_param = {
+        "stop": ["Question", "Assistant:", "<|separator|>"],
+        "max_new_tokens": 512,
+        "temperature": 0,
+    }
+
+    # Run requests
+    tic = time.time()
+
+    loop = asyncio.get_event_loop()
+
+    outputs = loop.run_until_complete(
+        concurrent_generate(engine, prompts, sampling_param)
+    )
+
+    # End requests
+    latency = time.time() - tic
+
+    # Shutdown the engine
+    engine.shutdown()
+
+    # Parse output
+    preds = []
+
+    for output in outputs:
+        preds.append(get_answer_value(output["text"]))
+
+    # Compute accuracy
+    acc = np.mean(np.array(preds) == np.array(labels))
+    invalid = np.mean(np.array(preds) == INVALID)
+
+    # Compute speed
+    num_output_tokens = sum(
+        output["meta_info"]["completion_tokens"] for output in outputs
+    )
+    output_throughput = num_output_tokens / latency
+
+    # Print results
+    print(f"Accuracy: {acc:.3f}")
+    print(f"Invalid: {invalid:.3f}")
+    print(f"Latency: {latency:.3f} s")
+    print(f"Output throughput: {output_throughput:.3f} token/s")
+
+    return {
+        "accuracy": acc,
+        "latency": latency,
+        "output_throughput": output_throughput,
+    }
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--model-path", type=str, default="meta-llama/Meta-Llama-3.1-8B-Instruct"
+    )
+    parser.add_argument("--local-data-path", type=Optional[str], default=None)
+    parser.add_argument("--num-shots", type=int, default=5)
+    parser.add_argument("--num-questions", type=int, default=200)
+    args = parser.parse_args()
+    metrics = run_eval(args)

sglang/test/srt/sampling/penaltylib/utils.py

@@ -164,19 +164,20 @@ class BaseBatchedPenalizerTest(unittest.TestCase):
                             msg=f"key={key}\nactual={getattr(penalizer, key)}\nexpected={tensor}",
                         )
 
-                actual = orchestrator.apply(
-                    torch.ones(
-                        size=(len(case.test_subjects), self.vocab_size),
-                        dtype=torch.float32,
-                        device=self.device,
-                    )
+                original = torch.ones(
+                    size=(len(case.test_subjects), self.vocab_size),
+                    dtype=torch.float32,
+                    device=self.device,
                 )
+                actual = orchestrator.apply(original.clone())
                 expected = torch.cat(
                     tensors=[
                         subject.steps[0].expected_logits
                         for subject in case.test_subjects
                     ],
                 )
+                if actual is None:
+                    actual = original
                 torch.testing.assert_close(
                     actual=actual,
                     expected=expected,
@@ -226,6 +227,8 @@ class BaseBatchedPenalizerTest(unittest.TestCase):
                         device=self.device,
                     )
                 )
+                if actual_logits is None:
+                    continue
                 filtered_expected_logits = torch.cat(
                     tensors=[
                         subject.steps[0].expected_logits
@@ -317,19 +320,20 @@ class BaseBatchedPenalizerTest(unittest.TestCase):
                                 msg=f"key={key}\nactual={getattr(penalizer, key)}\nexpected={tensor}",
                             )
 
-                    actual_logits = orchestrator.apply(
-                        torch.ones(
-                            size=(len(filtered_subjects), self.vocab_size),
-                            dtype=torch.float32,
-                            device=self.device,
-                        )
+                    original = torch.ones(
+                        size=(len(filtered_subjects), self.vocab_size),
+                        dtype=torch.float32,
+                        device=self.device,
                     )
+                    actual_logits = orchestrator.apply(original.clone())
                     filtered_expected_logits = torch.cat(
                         tensors=[
                             subject.steps[i].expected_logits
                             for subject in filtered_subjects
                         ],
                     )
+                    if actual_logits is None:
+                        actual_logits = original
                     torch.testing.assert_close(
                         actual=actual_logits,
                         expected=filtered_expected_logits,

sglang/version.py

@@ -1 +1 @@
-__version__ = "0.3.3.post1"
+__version__ = "0.3.4"

{sglang-0.3.3.post1.dist-info → sglang-0.3.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sglang
-Version: 0.3.3.post1
+Version: 0.3.4
 Summary: SGLang is yet another fast serving framework for large language models and vision language models.
 License: Apache License
                                    Version 2.0, January 2004
@@ -219,36 +219,49 @@ Requires-Dist: sglang[srt]; extra == "all"
 Requires-Dist: sglang[openai]; extra == "all"
 Requires-Dist: sglang[anthropic]; extra == "all"
 Requires-Dist: sglang[litellm]; extra == "all"
+Provides-Extra: all_xpu
+Requires-Dist: sglang[srt_xpu]; extra == "all-xpu"
+Requires-Dist: sglang[openai]; extra == "all-xpu"
+Requires-Dist: sglang[anthropic]; extra == "all-xpu"
+Requires-Dist: sglang[litellm]; extra == "all-xpu"
 Provides-Extra: anthropic
 Requires-Dist: anthropic>=0.20.0; extra == "anthropic"
 Provides-Extra: dev
 Requires-Dist: sglang[all]; extra == "dev"
 Requires-Dist: sglang[test]; extra == "dev"
+Provides-Extra: dev_xpu
+Requires-Dist: sglang[all_xpu]; extra == "dev-xpu"
+Requires-Dist: sglang[test]; extra == "dev-xpu"
 Provides-Extra: litellm
 Requires-Dist: litellm>=1.0.0; extra == "litellm"
 Provides-Extra: openai
 Requires-Dist: openai>=1.0; extra == "openai"
 Requires-Dist: tiktoken; extra == "openai"
+Provides-Extra: runtime_common
+Requires-Dist: aiohttp; extra == "runtime-common"
+Requires-Dist: decord; extra == "runtime-common"
+Requires-Dist: fastapi; extra == "runtime-common"
+Requires-Dist: hf-transfer; extra == "runtime-common"
+Requires-Dist: huggingface-hub; extra == "runtime-common"
+Requires-Dist: interegular; extra == "runtime-common"
+Requires-Dist: orjson; extra == "runtime-common"
+Requires-Dist: packaging; extra == "runtime-common"
+Requires-Dist: pillow; extra == "runtime-common"
+Requires-Dist: psutil; extra == "runtime-common"
+Requires-Dist: pydantic; extra == "runtime-common"
+Requires-Dist: python-multipart; extra == "runtime-common"
+Requires-Dist: torchao; extra == "runtime-common"
+Requires-Dist: uvicorn; extra == "runtime-common"
+Requires-Dist: uvloop; extra == "runtime-common"
+Requires-Dist: zmq; extra == "runtime-common"
+Requires-Dist: outlines>=0.0.44; extra == "runtime-common"
+Requires-Dist: modelscope; extra == "runtime-common"
 Provides-Extra: srt
-Requires-Dist: aiohttp; extra == "srt"
-Requires-Dist: decord; extra == "srt"
-Requires-Dist: fastapi; extra == "srt"
-Requires-Dist: hf-transfer; extra == "srt"
-Requires-Dist: huggingface-hub; extra == "srt"
-Requires-Dist: interegular; extra == "srt"
-Requires-Dist: packaging; extra == "srt"
-Requires-Dist: pillow; extra == "srt"
-Requires-Dist: psutil; extra == "srt"
-Requires-Dist: pydantic; extra == "srt"
-Requires-Dist: python-multipart; extra == "srt"
+Requires-Dist: sglang[runtime_common]; extra == "srt"
 Requires-Dist: torch; extra == "srt"
-Requires-Dist: torchao; extra == "srt"
-Requires-Dist: uvicorn; extra == "srt"
-Requires-Dist: uvloop; extra == "srt"
-Requires-Dist: zmq; extra == "srt"
 Requires-Dist: vllm==0.5.5; extra == "srt"
-Requires-Dist: outlines>=0.0.44; extra == "srt"
-Requires-Dist: modelscope; extra == "srt"
+Provides-Extra: srt_xpu
+Requires-Dist: sglang[runtime_common]; extra == "srt-xpu"
 Provides-Extra: test
 Requires-Dist: jsonlines; extra == "test"
 Requires-Dist: matplotlib; extra == "test"
@@ -270,14 +283,13 @@ Requires-Dist: peft; extra == "test"
 
 --------------------------------------------------------------------------------
 
-| [**Blog**](https://lmsys.org/blog/2024-07-25-sglang-llama3/) | [**Paper**](https://arxiv.org/abs/2312.07104) | [**Slides**](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/amd_dev_day_v2.pptx) | [**Join Slack**](https://join.slack.com/t/sgl-fru7574/shared_invite/zt-2ngly9muu-t37XiH87qvD~6rVBTkTEHw) | [**Join Bi-Weekly Development Meeting (Oct. 19)**](https://calendar.app.google/GYW7S8QGoanCuaxW6) |
-
-## Upcoming Events
-- [Oct. 16, 2024] Online meetup for efficient LLM deployment and serving, co-hosted by SGLang, FlashInfer, and MLC LLM! Fill out the [Google form](https://forms.gle/B3YeedLxmrrhL1NM8) to receive the invite link.
+| [**Blog**](https://lmsys.org/blog/2024-07-25-sglang-llama3/) | [**Paper**](https://arxiv.org/abs/2312.07104) | [**Slides**](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/amd_dev_day_v2.pdf) | [**Learn More**](https://github.com/sgl-project/sgl-learning-materials) | [**Join Slack**](https://join.slack.com/t/sgl-fru7574/shared_invite/zt-2ngly9muu-t37XiH87qvD~6rVBTkTEHw) |
+[**Join Bi-Weekly Development Meeting (Oct. 19)**](https://calendar.app.google/GYW7S8QGoanCuaxW6) |
 
 ## News
-- [2024/09] 🔥 SGLang v0.3 Release: 7x Faster DeepSeek MLA, 1.5x Faster torch.compile, Multi-Image/Video LLaVA-OneVision ([blog](https://lmsys.org/blog/2024-09-04-sglang-v0-3/)).
-- [2024/07] 🔥 Faster Llama3 Serving with SGLang Runtime (vs. TensorRT-LLM, vLLM) ([blog](https://lmsys.org/blog/2024-07-25-sglang-llama3/)).
+- [2024/10] 🔥 The First SGLang Online Meetup ([slides](https://github.com/sgl-project/sgl-learning-materials?tab=readme-ov-file#the-first-sglang-online-meetup)).
+- [2024/09] SGLang v0.3 Release: 7x Faster DeepSeek MLA, 1.5x Faster torch.compile, Multi-Image/Video LLaVA-OneVision ([blog](https://lmsys.org/blog/2024-09-04-sglang-v0-3/)).
+- [2024/07] Faster Llama3 Serving with SGLang Runtime (vs. TensorRT-LLM, vLLM) ([blog](https://lmsys.org/blog/2024-07-25-sglang-llama3/)).
 - [2024/02] SGLang enables **3x faster JSON decoding** with compressed finite state machine ([blog](https://lmsys.org/blog/2024-02-05-compressed-fsm/)).
 
 <details>
@@ -323,7 +335,7 @@ pip install flashinfer -i https://flashinfer.ai/whl/cu121/torch2.4/
 ### Method 2: From source
 ```
 # Use the last release branch
-git clone -b v0.3.3.post1 https://github.com/sgl-project/sglang.git
+git clone -b v0.3.4 https://github.com/sgl-project/sglang.git
 cd sglang
 
 pip install --upgrade pip
@@ -500,6 +512,40 @@ python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3-8B-Instruct
 python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3-8B-Instruct --tp 4 --nccl-init sgl-dev-0:50000 --nnodes 2 --node-rank 1
 ```
  
+### Engine Without HTTP Server
+
+We also provide an inference engine **without a HTTP server**. For example,
+
+```python
+import sglang as sgl
+
+
+def main():
+    prompts = [
+        "Hello, my name is",
+        "The president of the United States is",
+        "The capital of France is",
+        "The future of AI is",
+    ]
+    sampling_params = {"temperature": 0.8, "top_p": 0.95}
+    llm = sgl.Engine(model_path="meta-llama/Meta-Llama-3.1-8B-Instruct")
+
+    outputs = llm.generate(prompts, sampling_params)
+    for prompt, output in zip(prompts, outputs):
+        print("===============================")
+        print(f"Prompt: {prompt}\nGenerated text: {output['text']}")
+
+if __name__ == "__main__":
+    main()
+```
+
+This can be used for:
+
+1. **Offline Batch Inference**
+2. **Building Custom Servers**
+
+You can view the full example [here](https://github.com/sgl-project/sglang/tree/main/examples/runtime/engine)
+
 ### Supported Models
 
 **Generative Models**
@@ -836,10 +882,7 @@ def chat_example(s):
 - The `regex` argument in `sgl.gen` is implemented through autoregressive decoding with logit bias masking, according to the constraints set by the regex. It is compatible with `temperature=0` and `temperature != 0`.
 
 ## Benchmark And Performance
-![8b_throughput](https://lmsys.org/images/blog/sglang_llama3/8b_throughput.svg)
-![70b_fp8_throughput](https://lmsys.org/images/blog/sglang_llama3/70b_fp8_throughput.svg)
-
-Learn more at this [blog](https://lmsys.org/blog/2024-07-25-sglang-llama3/).
+Learn more in our release blogs: [v0.2](https://lmsys.org/blog/2024-07-25-sglang-llama3/), [v0.3](https://lmsys.org/blog/2024-09-04-sglang-v0-3/).
 
 ## Roadmap
 [Development Roadmap (2024 Q4)](https://github.com/sgl-project/sglang/issues/1487)

{sglang-0.3.3.post1.dist-info → sglang-0.3.4.dist-info}/RECORD

@@ -1,14 +1,14 @@
 sglang/__init__.py,sha256=b_pqO9bR2fjK9En_tigfzKTiQzE8b_hUizY0DAKVk1M,1616
 sglang/api.py,sha256=5x591S4rLbmNPs75qPwGKVu1sonVGDyjPAJlHTyWw50,6956
-sglang/bench_latency.py,sha256=gCS_nPNCf3sYR83jg6_KNadm-Xy7-I1V-UdZaoKFi8M,17889
-sglang/bench_server_latency.py,sha256=rRSDqjJ5jan9AzppOGx75KRUjZCU2dUG2h06CQOdJgk,5377
-sglang/bench_serving.py,sha256=1AQzkQ8ci9-rMZEM7wap8I09oPP4AZd93RfXMQRgVro,36386
+sglang/bench_latency.py,sha256=RWSyZ-UhLV6dyPMMtK3nSOoNsjCY5xMpYKeUKRNtdcA,18276
+sglang/bench_server_latency.py,sha256=2AMPwU2_85q-Btz9UdZC-TnZJPgXcNkydvFYWn2CJlU,5892
+sglang/bench_serving.py,sha256=jcxNP7reIJPh3x1hG5TCM6wMlDXshjyMJUUjL2O7kzs,40060
 sglang/check_env.py,sha256=rGRABCgt-0SfUrow4px28b2P59aMn8eVTnN5eZc_a8s,5397
-sglang/global_config.py,sha256=38id86i3tRGCSOFZlN1LM01a3xt-V98xuNgKGG9boCk,1058
+sglang/global_config.py,sha256=1r_W9rrBxGCCc2eqESRduOMMNq46e54xLgFLifHuQm0,1014
 sglang/launch_server.py,sha256=UnjNjYuZ8TtvmRtgYEsFImkbvCwvn_tQjk0V7cHy67E,450
 sglang/launch_server_llavavid.py,sha256=olPKyhozi1coCwoRMwBRYWsTFByrgus9CwPSeNmskgc,1002
 sglang/utils.py,sha256=NA_4xUrTI7KICQ3PEACfNWKE3nxSA5QvQZJNd4TQrDc,9395
-sglang/version.py,sha256=7Z8nSxbc04sgIKYqfKxkmSnG2nnSPT9dpM3RYiFOpUc,28
+sglang/version.py,sha256=oYLGMpySamd16KLiaBTfRyrAS7_oyp-TOEHmzmeumwg,22
 sglang/lang/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sglang/lang/chat_template.py,sha256=uqI_I9zIKXGXg7-W-yjqvx1ZeS_TuwFCms6wkmC2QmY,13411
 sglang/lang/choices.py,sha256=-W1DVw9N9ZliVpvmWrzIXG4cswAah8eMQrHWzkS3D8o,6234
@@ -26,9 +26,9 @@ sglang/lang/backend/vertexai.py,sha256=O-iBLD-y3vq80UxnrAoJri7bxpgd-_eakZ88Cf8bE
 sglang/srt/conversation.py,sha256=B4QPGOUVdoXRJwWWxSm5pfifGpuBs07fDTxJ1BHUXLw,20003
 sglang/srt/hf_transformers_utils.py,sha256=rt6flb6BoYTO8fw7AKCXmQLJx5XuSUuRmZX-VJHmuLQ,6064
 sglang/srt/mm_utils.py,sha256=zox644S3IHUWmADdK4MnIbdTS2DWHOy0_Dq0gCU38QQ,12273
-sglang/srt/server.py,sha256=hb27kEsOzQeVy5HuMVRJNOG8OsFwq0KZBVsZXB2qN1U,23267
-sglang/srt/server_args.py,sha256=_Y7YLlGYOWpR1mtfN8Hmi6hsldkp4q8vLhcNatHhsuQ,24374
-sglang/srt/utils.py,sha256=PYApZ2rFU67TnJaKkkjF9Z93jBPCJkotB0kk1vHGI6Y,22858
+sglang/srt/server.py,sha256=65b39k4FN_TzL8qAimS1mRx8xdO8jmKCdUftOISUv7M,26809
+sglang/srt/server_args.py,sha256=IDuX8ZCJd_6t2xHf7wNGskVgvpAQtUcFSGBnKFnLf3U,27290
+sglang/srt/utils.py,sha256=0zalNeGrtrIyfmD7DHLRqocCY1_yNbPCD5hmionHpL0,23071
 sglang/srt/configs/__init__.py,sha256=292SuEorST-lAq2Uvsv2M7yC28uYZlssVvRDsF-bZCQ,86
 sglang/srt/configs/exaone.py,sha256=Duxd4yQoKy8GWEzZD_kCY_OzmN_67CTJL_Kgn0eXk3g,10731
 sglang/srt/configs/model_config.py,sha256=36My-o44trhWY3KYDeSFMGvv9XuUtIVI5e7F8VlOTWo,6723
@@ -42,15 +42,16 @@ sglang/srt/layers/linear.py,sha256=9rjCiSb_QOn5RgpVjIhEKdReRvSYVfcTSjbWBEbApLI,4
 sglang/srt/layers/logits_processor.py,sha256=Fq7VHwjP4iSzl_OBLo8qw_HVbIDbYB-0MGmfiD3Jk_E,12521
 sglang/srt/layers/pooler.py,sha256=rj2lygvleBnyLCBZ8I11HGMgpfIDsT0l3PIkshJwdu4,1606
 sglang/srt/layers/radix_attention.py,sha256=i07VRXPDHj-zJ1TSrXEqCxumQwYSHwAvc8DoIg-Irtg,1964
-sglang/srt/layers/sampler.py,sha256=J5vd0CcLpLfgtLniCoe2VF6hjM_ld76hbDG4p1qoAMc,4010
+sglang/srt/layers/sampler.py,sha256=23wRDw2Fs3wZfPBh6gFBz2vqwxnoDd9LAHWq7YdQWlc,4166
 sglang/srt/layers/torchao_utils.py,sha256=1nzZkSzbF4qCAMeBKAeeDpMl_mK8imiY2RL3xFEgvAw,3340
-sglang/srt/layers/attention/__init__.py,sha256=zLLwinbYLAQHfVEz0jZiVa_cYNgSYoy4wYD_0y-ErHQ,1798
-sglang/srt/layers/attention/flashinfer_backend.py,sha256=DOvm-d3XLjE6XJDD3a8aCnlpuAJZZ946YFDH_Ec4lqc,10150
-sglang/srt/layers/attention/flashinfer_utils.py,sha256=9YMt7ab6F0gEVkxdVm8vDB0LVBRYRL0XIKVrmndp4n8,7571
-sglang/srt/layers/attention/triton_backend.py,sha256=I_kw0LXdgziHAFC8Qv5n5PDFJRLvZyzVsXwjmFZ0KSc,6041
+sglang/srt/layers/attention/__init__.py,sha256=hyrPpnuiTs5VIZNyoIjZLRsHUX20gX2dvY9kkqyXIII,2158
+sglang/srt/layers/attention/double_sparsity_backend.py,sha256=owzPwLWcShZ0ezkVjBr0vV73vtQIUh8z-rcQtXLG1fk,10374
+sglang/srt/layers/attention/flashinfer_backend.py,sha256=y9saTqOhb_tJoRtjq9sishlQFGYlFkQD7QcV9x_ureo,19554
+sglang/srt/layers/attention/triton_backend.py,sha256=B6YuIZWh4Zn85Q57i0z3eZ08RCrS0rfyPJzkRr_zOIQ,6150
 sglang/srt/layers/attention/triton_ops/decode_attention.py,sha256=XCQTX0kUttT1AG5FRMgfQbiXgvoempYD0UR2r6D_vJg,16711
-sglang/srt/layers/attention/triton_ops/extend_attention.py,sha256=oyqon1KG5-ICHcCANAbrglXLYKvWHFML-4tIQI9M5VI,11063
-sglang/srt/layers/attention/triton_ops/prefill_attention.py,sha256=QkXPcT02c13zha2M4mBm2S5dh_sS-Gc4FkkrcywRqvc,5377
+sglang/srt/layers/attention/triton_ops/double_sparsity_attention.py,sha256=1pSXfY3EEaM7iRN_uElHnAfsrJMhTFbu9fj8Z0O2PbE,21480
+sglang/srt/layers/attention/triton_ops/extend_attention.py,sha256=nEG7iBh1pAy3WaqPdLZwCJwDgyk5HLQ181kBS2nxbwg,11179
+sglang/srt/layers/attention/triton_ops/prefill_attention.py,sha256=bNHHZeEowwI1wwOWj2T6bjBTBtVZUbcL-0cgfZwpHek,5471
 sglang/srt/layers/fused_moe/__init__.py,sha256=bWCrDdOy2ANEXTb8CHYO63O3Iu3eZnn0PJbgl0z5vvE,75
 sglang/srt/layers/fused_moe/fused_moe.py,sha256=1WM2cObWXcFWtqh_utGJFPnrT344rORwuQ9hJDaH2s0,23104
 sglang/srt/layers/fused_moe/layer.py,sha256=raFyvPzjYz-Fv8B3IcOxQYKKCWqXis5mXwg1GFE61y4,22243
@@ -61,65 +62,67 @@ sglang/srt/lora/lora.py,sha256=a5j_Yy0s95msVPFgOuH5PCe7sMu0AyZFQ5wL0H-YIg8,14913
 sglang/srt/lora/lora_config.py,sha256=paVB7F7SIuxr_vodvKf8zzAlH2fdVYHhXxcXV62D0Vo,1411
 sglang/srt/lora/lora_manager.py,sha256=gzBwYXZEPYj56PkGTshTbWRfl_370wb6uTcRhDaLiF8,12801
 sglang/srt/managers/data_parallel_controller.py,sha256=GJGfX1-5DoQFZ-EMh_p02nvrOtrOc0UebnULWHhFrss,5765
-sglang/srt/managers/detokenizer_manager.py,sha256=iCLPdHkL6lAp_-Qew1u4Tyt3jYRkJ8i-Bj3l8TC-uaA,7278
+sglang/srt/managers/detokenizer_manager.py,sha256=G2-Y-nDbq7LF8ZFWcXXcMkbCwzyBEh1g4UrciDlkNYY,7985
 sglang/srt/managers/image_processor.py,sha256=9Y9RqyLdbt4uOK7pnJCJIhY77791klskSrEg8U6pyS4,6910
-sglang/srt/managers/io_struct.py,sha256=PxeLOgRJR5raUXExmQHWAnvJZLU0BA_e591zthEOYAQ,12185
-sglang/srt/managers/schedule_batch.py,sha256=rev0x6tp2ex8uW4PPrcfJ6m6WgfhsNLpuPNWForYcGE,31363
-sglang/srt/managers/schedule_policy.py,sha256=PiTKvsAFwoNWNsv_SFkghIHCL452MdboRc2cmN6ITcU,11935
-sglang/srt/managers/scheduler.py,sha256=3eJjVZgLDyZWjniQf1Mkv1-1rbQyaOjyM0cRG-bNHAw,40625
-sglang/srt/managers/tokenizer_manager.py,sha256=AI1yfV5A1cpWDDuoelRx55lKMgNWccNOvyDFlKLddFA,24794
-sglang/srt/managers/tp_worker.py,sha256=fcaW-u7AAX49kQCNn_AEtdRPykRdT6Z6lx1O9LHA15E,4833
+sglang/srt/managers/io_struct.py,sha256=QJ1Eu7XNsRH35ec5nUOUS2XSjiFWyjYHKsnMqviC_Mk,12298
+sglang/srt/managers/schedule_batch.py,sha256=6LUXbAd6PvHIDk6iwpsufosahIT2z32JZjsNx5xeKcg,33930
+sglang/srt/managers/schedule_policy.py,sha256=unDmK7Y_Ti0Eiizh3_iEFMsC1KDqGMTqU8MlQgg-6qo,11951
+sglang/srt/managers/scheduler.py,sha256=Y7R-VkLt8Az2jZGrGRuhG1g4UPO5y-7b9BaOknuC2aI,43019
+sglang/srt/managers/tokenizer_manager.py,sha256=SprHC0Bs8kvtnYboDPrH587uO_sdKHyp7tVBCdbEB9c,25066
+sglang/srt/managers/tp_worker.py,sha256=nwkIXiasGA4w97pnMG32U1DN1RlLFkvETvl9q7SjGeY,8887
 sglang/srt/mem_cache/base_prefix_cache.py,sha256=qEQwEkG4E5rab2ZoTqcesf5pR_J4nV2jBxIHsBJHtIM,924
-sglang/srt/mem_cache/chunk_cache.py,sha256=CjZZYlqQzq7mYOiBMLWA5XNb6HIyh5lIMdY-K0OUZEc,2368
+sglang/srt/mem_cache/chunk_cache.py,sha256=VcCpyrf5FOQ5xoKeOouCI5ZQLkZo_pgY1SPbDDkagGg,2492
 sglang/srt/mem_cache/flush_cache.py,sha256=pTLKPRB17U6vl5RFJJvuJ4jCL2SyomgkUBNlkDpGRqo,978
-sglang/srt/mem_cache/memory_pool.py,sha256=L-5drUt7vlyvple4OcjH1jJRzt2qhVrpc9klZn-bQfE,7125
-sglang/srt/mem_cache/radix_cache.py,sha256=00bghOihUm7lA1i4gxxMYQLept9LaHg2ZSXZryuFZZI,10121
-sglang/srt/model_executor/cuda_graph_runner.py,sha256=iheZYErwFT_W4kJUE1dgbGoQQx7hyOSKa-Yv8guq0DI,10479
-sglang/srt/model_executor/forward_batch_info.py,sha256=FIQ8XIIP724mIL2l7w7mSEFH452qw-TPpqm43J4YeHM,5822
-sglang/srt/model_executor/model_runner.py,sha256=Qxp6VyL-yiDzaTQuAQPTRrHiqsZAT0ki94teubxbocc,23237
-sglang/srt/models/baichuan.py,sha256=50m43kIVo-YamHFwxyiLGG_pCbF7mzUJfhEyuuSmVC8,15100
-sglang/srt/models/chatglm.py,sha256=XaS_6-ZvRw7X-56sk9xQogqT0NzGEMVpiAdQnC5qbBY,13333
-sglang/srt/models/commandr.py,sha256=2urK7u2FiwPBl60hMmt-wfaJ8V-ilv6l1B37MUlvSxk,14121
-sglang/srt/models/dbrx.py,sha256=qTpyA1Iv56VI-ksPKt4JryX2Pn7T5FXAa0n0ZoT4qbw,14615
-sglang/srt/models/deepseek.py,sha256=4sl4YYoxqe-vif7KJKcMjMA3KgvzYHqpQBgM58lzLHc,15973
-sglang/srt/models/deepseek_v2.py,sha256=dt0FGAgW3jd7OJJnKfH-LIU13U0I9b7R9shYmAEins4,28390
-sglang/srt/models/exaone.py,sha256=9JfFhYbpcHMXIaBNn8rc_GOlkItkIgbGNslNyFD7gvU,13054
-sglang/srt/models/gemma.py,sha256=gui46inEJsrmppEMTUIQuzMxGPEBx_TjiZ5-PacjuSk,12240
-sglang/srt/models/gemma2.py,sha256=V0GjEdTqxyXvBqjgyiyONipohjOqw0pLITmZZRb2kIE,14890
-sglang/srt/models/gpt_bigcode.py,sha256=LgSm-8oxBfnzMAC4Jqqg-RJGge4E_wgJ1br7ylbTPZ0,10162
-sglang/srt/models/grok.py,sha256=lUR_SmD_KhIiZx5OVUPZp8VVdrAga6WWTdMKJ5PCFbw,14896
-sglang/srt/models/internlm2.py,sha256=4SUaeJl2dZlUowahfv7kLbz3jLXtmvdBPGURmhAeX6Q,12169
-sglang/srt/models/llama.py,sha256=5j66LmvFhOKgFZiE75mJ80XBjZ2dNx7e8Yea5lsD0P0,15828
-sglang/srt/models/llama_classification.py,sha256=Yhabu9FuBxjNo74crMsK0FqpD53ehOx_zcHgIXjvlvQ,3379
+sglang/srt/mem_cache/memory_pool.py,sha256=ihVZXlJ_Fvs1L2c2SZQaijUYSn9X6eyiFiG2NNRQS_M,9297
+sglang/srt/mem_cache/radix_cache.py,sha256=cS6G5uOW_0QICH30PXxatetka4wnELfhP4czHn8RDJE,10414
+sglang/srt/model_executor/cuda_graph_runner.py,sha256=KgSBvoF2IyCGDFNXQyN7sV3E_S2NndeMQyaQZB97Pak,10499
+sglang/srt/model_executor/forward_batch_info.py,sha256=rSazAtkWKyc2g2QILT2-AsUdaBt51hQBU1qcS2iw_Nw,5690
+sglang/srt/model_executor/model_runner.py,sha256=hn-VeLABZL4y_GcJFl2y_C7x6ZS_Xw0tDyKzOxYMtVQ,25983
+sglang/srt/models/baichuan.py,sha256=uV20fr7SqlATxoziXRtJyXFnZZoWTUy3qvQNFaEvw8M,15014
+sglang/srt/models/chatglm.py,sha256=uep4Wy_2jwn_x6Cvagt5rs3JRY_AlcM-VXvRTCFC5Tc,13172
+sglang/srt/models/commandr.py,sha256=WIMwjV3C0pRbVs4Xv9tqnHGreRvWC7zsML2hNuXw4A0,14060
+sglang/srt/models/dbrx.py,sha256=_DshXyXr_xVB7wtE28PFcb6KDIkA6gygkXYKqivSCFc,14554
+sglang/srt/models/deepseek.py,sha256=W342tVpEpkc_fvO_DTP4fX3EGF-DIFC4QOySdUGzl9w,15837
+sglang/srt/models/deepseek_v2.py,sha256=5P5678aaMT4iO4jS0dZWUiRG4o9EE98xVgs6Zjy-Mr0,28229
+sglang/srt/models/exaone.py,sha256=bIXdAXoWlCdfDdX2q47Br3QOa3jEYiiP2Hdd1T4crnM,12993
+sglang/srt/models/gemma.py,sha256=4MVHwc5Jc4CSg3HIdNJEBYk8mhspjuwvc_6Oi8Cd-g8,12202
+sglang/srt/models/gemma2.py,sha256=3VL223T_3syBG3fUInbtFaXvIs7dYjtsfX3OfDQc7m4,14777
+sglang/srt/models/gpt_bigcode.py,sha256=q9N13Js2v0VheudWssRoSjnptS6TSf7DOmC8zLRGxeo,10049
+sglang/srt/models/grok.py,sha256=vc7-E_hemNKaNORxg4rmaQcVYlpoavyaAZUG9B2dgbY,14835
+sglang/srt/models/internlm2.py,sha256=-liQB13sgR3GnXJacBSMuEbLa2N4tICx0LsNgu_nNvU,12108
+sglang/srt/models/llama.py,sha256=a43Y5mvMDmFcRcPL78vsAElaOvTqPajLPB2_BDwJ7pM,15767
+sglang/srt/models/llama_classification.py,sha256=WcHYFez7qloTCpXLy1A6-dBGHWp22ebv6yG68jFVBjc,3318
 sglang/srt/models/llama_embedding.py,sha256=4j3WNLB-x7XQnJvohdRs7VSSEabbhiE2BRHmnG5IZRU,3453
-sglang/srt/models/llama_reward.py,sha256=qQOPfn-9oqhsD0EaffXtk-EXKRdSZL1X7CYAGCDoG9A,5383
-sglang/srt/models/llava.py,sha256=zbJs1P4_Bjh2_dSbyoheJZ1wGXuKHGz6BpV766G7ZUY,25094
-sglang/srt/models/llavavid.py,sha256=qhBGHTxzGAOMgqMiwOc3mUbaK6qeXsEYSlNmlEEIdeM,12198
-sglang/srt/models/minicpm.py,sha256=5vc-Lq7ggHrRxxkciVMdZ5Vq6ThLwnhFS62UCokFC2g,13792
-sglang/srt/models/minicpm3.py,sha256=hhhgZTKQApUZpH_MYQZTk3K1Ox-xpJRxGCemoUw8x4U,25184
+sglang/srt/models/llama_reward.py,sha256=ag3eVdP38iURj81fTCa-sC2jV_eCkTIjXUQf1I96fCI,5297
+sglang/srt/models/llava.py,sha256=ny3sK2sgYwrEhawSAc1tZeltcgukphSTdxsqyq-Epkc,24857
+sglang/srt/models/llavavid.py,sha256=ztS5He-NF4fmfujdoMnKljOG1fNfPvp-6bduT7B6EMU,12137
+sglang/srt/models/minicpm.py,sha256=LpUdxKA27z79DSYAPPlfCgI4GEnWCYznhgSQl-QCsTY,13731
+sglang/srt/models/minicpm3.py,sha256=-fLZ-RRbR2jLGSsatBWV-qsSNIZCPbS_jasmrOlUdK8,25023
 sglang/srt/models/mistral.py,sha256=tiYoKjyYVzlQl52QUZ33odD2yCxj9dxcqln474VuZOw,744
-sglang/srt/models/mixtral.py,sha256=BonqX_rSB_UuBDQe3uy8-NOxB4Q4s2mTxTQItvFB9ZQ,13864
-sglang/srt/models/mixtral_quant.py,sha256=SAHBIiD5O1TnojCpqTLcPy3TEvfSCKeOe3GC47fdFSg,14039
-sglang/srt/models/olmoe.py,sha256=ghhNpZe4SzaZEpw0APYBbAmLb3LBagRC2N724RkOkH4,15312
-sglang/srt/models/qwen.py,sha256=IrOKHS7b4SL2fnJegq811eeHnAQDya2PujIgKQ9URVY,9921
-sglang/srt/models/qwen2.py,sha256=B7hXnW5uYPmpMgSN7tI3tTvMEmmQLpddsw_iNTiaHJI,12398
-sglang/srt/models/qwen2_moe.py,sha256=MK-9W6FJhXoQYayg_jpXjKKq4n5j3s2b2ZaoCBfVJ2I,17120
-sglang/srt/models/stablelm.py,sha256=ldtlRG1XGdYcjwqb48dpMTfbdh8KHUjcWrrUYNJ0MEk,11326
-sglang/srt/models/torch_native_llama.py,sha256=c5GJ_k9zbSOk0PjLCXAK8YebGEy0RUVYZ9_h6_19A3M,19215
-sglang/srt/models/xverse.py,sha256=i11wEKqqVCoVtH7yo9jfpNyGHxhw7NvTPid3ojmg79s,13634
-sglang/srt/models/xverse_moe.py,sha256=JwkBhsyusP7e_hAMnomkP8cEmKNCLJPRtwaTERQ0D0M,15818
-sglang/srt/models/yivl.py,sha256=N3noJ5M-FiZS-E_zfaJs4prQOu_ineRt11MWloYgOR8,4826
-sglang/srt/openai_api/adapter.py,sha256=bQ2lZGEQGAUkITXshdnCPzx6JN9iqYVvIpfD7uO5rN4,51519
-sglang/srt/openai_api/protocol.py,sha256=rdSwUAoO5-KLemJOE50xwSUagxY4T1QIiNyCYsTtCi0,9868
-sglang/srt/sampling/sampling_batch_info.py,sha256=ec5TMw47q2OCrkp2QwN45Ss1RZ-QYv7-KuGFKyGuvsg,6686
-sglang/srt/sampling/sampling_params.py,sha256=Xwh4_M6PP4SWyGV-zNyIhp4XbRKbeU4251ao8UOlZlI,5704
+sglang/srt/models/mixtral.py,sha256=UUqzpOBXsObirmpJz4xstlG82uu4JfXsh-gWQmiKbW0,13803
+sglang/srt/models/mixtral_quant.py,sha256=HPipVG_Gc5Ki0YXg49Rwn2_uvtCCI1IxlA7mVRVFivw,13978
+sglang/srt/models/olmo.py,sha256=lD4VewXK0rVqhttGkOOzaxoqRQgVfV90s8ElStPBBdE,11896
+sglang/srt/models/olmoe.py,sha256=3qHnY1DWBhyx9FWGJGb3a8kewcmEdYZOkYZ1JBx1LWs,15251
+sglang/srt/models/qwen.py,sha256=mjGqo3NkTYfJ2qqztFw8mjKggPT2moW15nQgrq3GxWk,9860
+sglang/srt/models/qwen2.py,sha256=I2ZzH9pVTZdjP1fHlq1qdG4JiWHt1CC6t1EK2gN5Ppc,12337
+sglang/srt/models/qwen2_moe.py,sha256=BaNq8xgZKqjr_fcEBtH4yjBSc3-p4VztPiknVwllcQk,16984
+sglang/srt/models/stablelm.py,sha256=0NWUVsYGhbc_X2eT9x38MaaUhZGmFtMgw_2PBv25Yxw,11265
+sglang/srt/models/torch_native_llama.py,sha256=dtasdhwfRPE1eOcAIFUBsHrDnkjegXvo8WhGlqvXGKk,19154
+sglang/srt/models/xverse.py,sha256=v4OaFdss9oD5YNzXsnjoXE9ffCkXL9U5o0OWLm1vHQQ,13573
+sglang/srt/models/xverse_moe.py,sha256=A8EB82NpozoBplp7Qd8B_kY_3cL-UMydAxYIrhACVPE,15682
+sglang/srt/models/yivl.py,sha256=xcWqkuZ29FmBBJY6aKetwItWIPl-kfXK-QmgdLONles,4765
+sglang/srt/openai_api/adapter.py,sha256=WkYCKVaYTkFdLrySBhlkDyHJVaaHMF7KrhNnmw3L3us,53534
+sglang/srt/openai_api/protocol.py,sha256=EZ6G209rBEDP7cepO2kAYqE8wMe1ksYdN7to1iT97Lw,10248
+sglang/srt/sampling/sampling_batch_info.py,sha256=EAdep3I5qmbDDQJ0Ktrq0ySXJ6DCrTAjniEwFu4ZRqE,7679
+sglang/srt/sampling/sampling_params.py,sha256=ZPHCQq7Bi4P_sxUzdKgYVXZpB_tC-kA7rlLwiW9Ct9A,5781
 sglang/srt/sampling/penaltylib/__init__.py,sha256=5vQw0Y5DSzmsoFg1IdMIKLwFVhYZ5ArADHVBYbSmOec,513
-sglang/srt/sampling/penaltylib/orchestrator.py,sha256=WkTNeDhj9H9rtp2ZZeX6MS2sdKSGlLboE6FcuKrwUo0,10815
+sglang/srt/sampling/penaltylib/orchestrator.py,sha256=kizcPnxtRawmDt6utRuhbk4yfNs5H5mx1DAlDVEZRv8,11328
 sglang/srt/sampling/penaltylib/penalizers/frequency_penalty.py,sha256=IvYioX53Vq_ji-0Zhcz_r5mUa3T3GaIydVS6K4FhWfE,2557
 sglang/srt/sampling/penaltylib/penalizers/min_new_tokens.py,sha256=XJZP0C4NFyXgcODbIWXxrgVEjmRgqLdZuVAtoN-LveY,3565
 sglang/srt/sampling/penaltylib/penalizers/presence_penalty.py,sha256=0PlANTrR959foTA3Nj5qBE7ndaOZgG-9X6LhzlmEUc8,2533
 sglang/srt/sampling/penaltylib/penalizers/repetition_penalty.py,sha256=v9jOgA0-I31WcrhIydiFbpy2ZJPLytFLGM98NRPd2sU,2820
-sglang/test/few_shot_gsm8k.py,sha256=To7Sdg-DLF8poIQLwiOBYKbkz-1C_gn6H79vIbyPR-o,3860
+sglang/test/few_shot_gsm8k.py,sha256=ll-gNbcv829IwSPXAZt4JIEIu8IR3APCLcX3BHOFVp8,3968
+sglang/test/few_shot_gsm8k_engine.py,sha256=QQbrwOX6-cJDD3RZC_e7zPnt6aSo8JdF8X_lRHSjdDM,3886
 sglang/test/run_eval.py,sha256=NWxeLWmInBgkCvC9Jr_QzF7GfAiBve3Gf1JQrEOlNlU,3899
 sglang/test/runners.py,sha256=VCmtH08FsAq_JTAKfKo0zB4o-osNMAxxwe4aKcSxr4c,13515
 sglang/test/simple_eval_common.py,sha256=r0G-9QLycs2ax3RMc44T_61fzMxlpTzv6pececC7lyY,12379
@@ -132,9 +135,9 @@ sglang/test/test_activation.py,sha256=jkdNRzJnbd5OgZliQaIXpxovlcky17UrweomcOcMxo
 sglang/test/test_layernorm.py,sha256=IacByD5d-stXjzBz8Ypamc7povlcedpKPbb_4JLgo3c,3720
 sglang/test/test_programs.py,sha256=1Z0umrsUu9pagzyGH5SrXl_qhKSyTfUv_kWC2mcn0qo,18208
 sglang/test/test_utils.py,sha256=NkJuezjmonjgC3_i_CTBd8KSqWh6W9CLcgoaqvTNK2U,18684
-sglang/test/srt/sampling/penaltylib/utils.py,sha256=-0p0rV-P4lNo7xAe3rQSBHTubc50a-DFyOQmLGAkgkQ,12515
-sglang-0.3.3.post1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-sglang-0.3.3.post1.dist-info/METADATA,sha256=xfzfAtRkt_PcB8Lw34-Jckq-iukmhDnhu-_8e9SZ3_Y,39186
-sglang-0.3.3.post1.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-sglang-0.3.3.post1.dist-info/top_level.txt,sha256=yxhh3pYQkcnA7v3Bg889C2jZhvtJdEincysO7PEB09M,7
-sglang-0.3.3.post1.dist-info/RECORD,,
+sglang/test/srt/sampling/penaltylib/utils.py,sha256=Koe8GYoxIBUCz71of0oHhM5t5QcEd6a1IYq5SszRFAw,12730
+sglang-0.3.4.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+sglang-0.3.4.dist-info/METADATA,sha256=rrkwX2teVdp79NEuOJfTBPUYCs_72LHIabuIesToPdI,40738
+sglang-0.3.4.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
+sglang-0.3.4.dist-info/top_level.txt,sha256=yxhh3pYQkcnA7v3Bg889C2jZhvtJdEincysO7PEB09M,7
+sglang-0.3.4.dist-info/RECORD,,

{sglang-0.3.3.post1.dist-info → sglang-0.3.4.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.1.0)
+Generator: setuptools (75.2.0)
 Root-Is-Purelib: true
 Tag: py3-none-any