sglang 0.1.12__py3-none-any.whl → 0.1.14__py3-none-any.whl

sglang/utils.py

@@ -88,26 +88,29 @@ class HttpResponse:
         return self.resp.status
 
 
-def http_request(url, json=None, stream=False, auth_token=None):
+def http_request(
+    url, json=None, stream=False, auth_token=None, api_key=None, verify=None
+):
     """A faster version of requests.post with low-level urllib API."""
+    headers = {"Content-Type": "application/json; charset=utf-8"}
+
+    # add the Authorization header if an auth token is provided
+    if auth_token is not None:
+        headers["Authorization"] = f"Bearer {auth_token}"
+
+    # add the API Key header if an API key is provided
+    if api_key is not None:
+        headers["X-API-Key"] = api_key
+
     if stream:
-        if auth_token is None:
-            return requests.post(url, json=json, stream=True)
-        headers = {
-            "Content-Type": "application/json",
-            "Authentication": f"Bearer {auth_token}",
-        }
         return requests.post(url, json=json, stream=True, headers=headers)
     else:
-        req = urllib.request.Request(url)
-        req.add_header("Content-Type", "application/json; charset=utf-8")
-        if auth_token is not None:
-            req.add_header("Authentication", f"Bearer {auth_token}")
+        req = urllib.request.Request(url, headers=headers)
         if json is None:
             data = None
         else:
             data = bytes(dumps(json), encoding="utf-8")
-        resp = urllib.request.urlopen(req, data=data)
+        resp = urllib.request.urlopen(req, data=data, cafile=verify)
         return HttpResponse(resp)
 
 

{sglang-0.1.12.dist-info → sglang-0.1.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sglang
-Version: 0.1.12
+Version: 0.1.14
 Summary: A structured generation langauge for LLMs.
 License: Apache License
                                    Version 2.0, January 2004
@@ -217,7 +217,7 @@ Requires-Dist: sglang[srt] ; extra == 'all'
 Requires-Dist: sglang[openai] ; extra == 'all'
 Requires-Dist: sglang[anthropic] ; extra == 'all'
 Provides-Extra: anthropic
-Requires-Dist: anthropic ; extra == 'anthropic'
+Requires-Dist: anthropic >=0.20.0 ; extra == 'anthropic'
 Requires-Dist: numpy ; extra == 'anthropic'
 Provides-Extra: openai
 Requires-Dist: openai >=1.0 ; extra == 'openai'
@@ -231,7 +231,7 @@ Requires-Dist: torch ; extra == 'srt'
 Requires-Dist: uvloop ; extra == 'srt'
 Requires-Dist: uvicorn ; extra == 'srt'
 Requires-Dist: zmq ; extra == 'srt'
-Requires-Dist: vllm >=0.2.5 ; extra == 'srt'
+Requires-Dist: vllm >=0.3.3 ; extra == 'srt'
 Requires-Dist: interegular ; extra == 'srt'
 Requires-Dist: lark ; extra == 'srt'
 Requires-Dist: numba ; extra == 'srt'
@@ -242,7 +242,12 @@ Requires-Dist: cloudpickle ; extra == 'srt'
 Requires-Dist: pillow ; extra == 'srt'
 Requires-Dist: outlines >=0.0.27 ; extra == 'srt'
 
-# SGLang
+<div align="center">
+<img src="assets/logo.png" alt="logo" width="400"></img>
+</div>
+
+--------------------------------------------------------------------------------
+
 | [**Blog**](https://lmsys.org/blog/2024-01-17-sglang/) | [**Paper**](https://arxiv.org/abs/2312.07104) |
 
 SGLang is a structured generation language designed for large language models (LLMs).
@@ -254,7 +259,7 @@ The core features of SGLang include:
 
 ## News
 - [2024/02] 🔥 SGLang enables **3x faster JSON decoding** with compressed finite state machine ([blog](https://lmsys.org/blog/2024-02-05-compressed-fsm/)).
-- [2024/01] 🔥 SGLang powers the serving of the offical **LLaVA v1.6** release demo ([usage](https://github.com/haotian-liu/LLaVA?tab=readme-ov-file#demo)).
+- [2024/01] 🔥 SGLang powers the serving of the official **LLaVA v1.6** release demo ([usage](https://github.com/haotian-liu/LLaVA?tab=readme-ov-file#demo)).
 - [2024/01] SGLang provides up to **5x faster inference** with RadixAttention ([blog](https://lmsys.org/blog/2024-01-17-sglang/)).
 
 ## Contents
@@ -496,7 +501,7 @@ def text_qa(s, question):
     s += "Q: " + question + "\n"
     s += "A:" + sgl.gen("answer", stop="\n")
 
-states = text_qa.run(
+state = text_qa.run(
     question="What is the capital of France?",
     temperature=0.1,
     stream=True
@@ -608,8 +613,13 @@ python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port
 - Mistral
 - Mixtral
 - Qwen / Qwen 2
+- Gemma
+  - Please add a new flag `--attention-reduce-in-fp32` to avoid some precision errors.
+  - `python -m sglang.launch_server --model-path google/gemma-7b-it --port 30000 --attention-reduce-in-fp32`
 - LLaVA
   - `python3 -m sglang.launch_server --model-path liuhaotian/llava-v1.5-7b --tokenizer-path llava-hf/llava-1.5-7b-hf --chat-template vicuna_v1.1 --port 30000`
+  - `python3 -m sglang.launch_server --model-path liuhaotian/llava-v1.6-vicuna-7b --tokenizer-path llava-hf/llava-1.5-7b-hf --chat-template vicuna_v1.1 --port 30000`
+  - `python3 -m sglang.launch_server --model-path liuhaotian/llava-v1.6-34b --tokenizer-path liuhaotian/llava-v1.6-34b-tokenizer --port 3000`
 - Yi-VL
   - see [srt_example_yi_vl.py](examples/quick_start/srt_example_yi_vl.py).
 - AWQ/GPTQ quantization

sglang-0.1.14.dist-info/RECORD

@@ -0,0 +1,64 @@
+sglang/__init__.py,sha256=Nxa2M7XCh2-e6I7VrCg7OSBL6BvEW3gyRD14ZdykpRM,96
+sglang/api.py,sha256=0-Eh7c41hWKjPXrzzvLFdLAUVkvmPGJGLAsrG9evDTE,4576
+sglang/global_config.py,sha256=PAX7TWeFcq0HBzNUWyCONAOjqIokWqw8vT7I6sBSKTc,797
+sglang/launch_server.py,sha256=jKPZRDN5bUe8Wgz5eoDkqeePhmKa8DLD4DpXQLT5auo,294
+sglang/utils.py,sha256=2dUXLMPz9VhhzbIRQABmfZnVW5yz61F3UVtb6yKyevM,6237
+sglang/backend/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+sglang/backend/anthropic.py,sha256=GJ_T1Jg0VOtajgkgczPKt5sjuVYdbAiWd2jXlJRNRmg,1677
+sglang/backend/base_backend.py,sha256=APiMht4WYECLCOGRPCEUF6lX-an1vjVe2dWoMSgymWY,1831
+sglang/backend/openai.py,sha256=nPdA88A5GISJTH88svJdww3qHWIHZcGG2NEn0XjMkLU,9578
+sglang/backend/runtime_endpoint.py,sha256=r7dTazselaudlFx8hqk-PQLYDHZhpbAKjyFF1zLuM_E,8022
+sglang/backend/vertexai.py,sha256=BLfWf_tEgoHY9srCufJM5PLe3tql2j0G6ia7cPykxCM,4713
+sglang/lang/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+sglang/lang/chat_template.py,sha256=MaCF0fvNky0nJC9OvmAeApeHYgM6Lr03mtRhF0lS31U,8000
+sglang/lang/compiler.py,sha256=wNn_UqV6Sxl22mv-PpzFUtRgiFFV-Y4OYpO4LshEoRM,7527
+sglang/lang/interpreter.py,sha256=ahRxuEJZ7b1Tts2Lr7wViWIqL-Z12T3anvgj0XdvMN8,26666
+sglang/lang/ir.py,sha256=8Ap-uEUz6K9eNQTOKtMixePuLwRFHFKcN0Z5Yn44nKk,13320
+sglang/lang/tracer.py,sha256=pFiSNzPSg0l7ZZIlGqJDLCmQALR-wyo2dFgJP73J4_Y,8260
+sglang/srt/backend_config.py,sha256=UIV6kIU2j-Xh0eoezn1aXcYIy0miftHsWFeAZwqpbGE,227
+sglang/srt/conversation.py,sha256=mTstD-SsXG5p_YhWQUPEWU-vzzDMF4RgQ7KmLkOOC7U,15496
+sglang/srt/hf_transformers_utils.py,sha256=soRyYLoCn7GxgxvonufGFkdFBA3eH5i3Izk_wi7p1l0,5285
+sglang/srt/memory_pool.py,sha256=BMoX2wvicj214mV-xvcr_Iv_Je0qs3zTuzXfQVpV8u4,3609
+sglang/srt/mm_utils.py,sha256=OptgAHDX-73Bk4jAdr2BOAJtiEXJNzPrMhaM-dy275c,8889
+sglang/srt/model_config.py,sha256=ned-odjmKBKBhVPo04FEpus9gJsUWxrFLrLxahLwSaw,1328
+sglang/srt/sampling_params.py,sha256=83Fp-4HWThC20TEh139XcIb_erBqfI7KZg5txdRBq7c,2896
+sglang/srt/server.py,sha256=WLXissKuXQI7JFb2V8D47QSF-PPHnW-JZCiQm4YW0xE,24070
+sglang/srt/server_args.py,sha256=bvbi-Rb_JudqztFFfRsuXBYtUsG9hq4zMFt7X97uDhA,8954
+sglang/srt/utils.py,sha256=IEqpmWx_hl4eXn_KoHM0EPXmxeN2wKkgK7H01_t0x5Q,7355
+sglang/srt/constrained/__init__.py,sha256=BPRNDJnWtzYJ13X4urRS5aE6wFuwAVNBA9qeWIHF8rE,1236
+sglang/srt/constrained/base_cache.py,sha256=QQjmFEiT8jlOskJoZobhrDl2TKB-B4b1LPQo9JQCP_w,1405
+sglang/srt/constrained/fsm_cache.py,sha256=20mEgtDXU1Zeoicl5KBQC3arkg-RhRWiYnchJc00m1g,901
+sglang/srt/constrained/jump_forward.py,sha256=Z-pz2Jnvk1CxSEZA65OVq0GryqdiKuOkhhc13v5T6Lo,2482
+sglang/srt/layers/context_flashattention_nopad.py,sha256=TVYQ6IjftWVXORmKpEROMqQxDOnF6n2g0G1Ci4LquYM,5209
+sglang/srt/layers/extend_attention.py,sha256=KGqQOA5mel9qScXMAQP_3Qyhp3BNbiQ7Y_6wi38Lxcs,12622
+sglang/srt/layers/logits_processor.py,sha256=MW2bpqSXyghODMojqeMSYWZhUHuAFPk_gUkyyLw9HkM,4827
+sglang/srt/layers/radix_attention.py,sha256=bqrb8H8K8RbKTr1PzVmpnUxRzMj0H-OWCi1JYZKuRDw,5597
+sglang/srt/layers/token_attention.py,sha256=waOjGsWZlvf6epFhYerRJlAaMwvDTy_Z3uzPaXsVQUU,8516
+sglang/srt/managers/detokenizer_manager.py,sha256=1lPNh_Pe6Pr0v-TzlCBBREbvz4uFWxyw31SmnEZh0s8,3292
+sglang/srt/managers/io_struct.py,sha256=nXJh3CrOvv9MdAfIFoo6SCXuNQTG3KswmRKkwF61Tek,3141
+sglang/srt/managers/openai_protocol.py,sha256=cttqg9iv3de8fhtCqDI4cYoPPZ_gULedMXstV1ok6WA,4563
+sglang/srt/managers/tokenizer_manager.py,sha256=hgsR9AMj6ic9S3-2WiELh7Hnp8Xnb_bzp7kpbjHwHtM,9733
+sglang/srt/managers/router/infer_batch.py,sha256=U-Ckt9ad1WaOQF_dW6Eo9AMIRQoOJQ-Pm-MMXnEmPP8,18399
+sglang/srt/managers/router/manager.py,sha256=TNYs0IrkZGkPvZJViwL7BMUg0VlvzeyTjDMjuvRoMDI,2529
+sglang/srt/managers/router/model_rpc.py,sha256=VlwLNpHZ92bnteQl4PhVKoAXM0C8Y4_2LBBVaffeu3g,26766
+sglang/srt/managers/router/model_runner.py,sha256=-wWv00EbB_UkkLpio6VKGBTagfzxLHfY-eKDDQ0rZQc,18292
+sglang/srt/managers/router/radix_cache.py,sha256=XGUF5mxQTSCzD7GW_ltNP2p5aelEKrMXzdezufJ7NCQ,6484
+sglang/srt/managers/router/scheduler.py,sha256=V-LAnVSzgD2ddy2eXW3jWURCeq9Lv7YxCGk4kHyytfM,2818
+sglang/srt/models/gemma.py,sha256=8XlfHPtVixPYYjz5F9T4DOAuoordWFStmyFFWGfny1k,11582
+sglang/srt/models/llama2.py,sha256=VL4iN8R3wyTNr0bDxxKdLNnVGEvdXF6iGvA768YeakA,11611
+sglang/srt/models/llava.py,sha256=42sn-AgI-6dMaTEU4aEbi4Js5epy0J3JVQoMooUOKt8,14922
+sglang/srt/models/mistral.py,sha256=XSn7fiZqspyWVTYrpVAacAnWdwAybBtyn9-Sh9AvMTM,254
+sglang/srt/models/mixtral.py,sha256=wqIwKfR90ih0gDiTZkFZcQD4PIYpZFD3CmzxRcuKIqw,13915
+sglang/srt/models/qwen.py,sha256=CvdbcF90aI1tJPSQ-3OMUaQGMuaxCGe0y29m5nU_Yj0,9225
+sglang/srt/models/qwen2.py,sha256=myPc0wvgf5ZzJyGhUGN49YjY-tMf4t8Jn_Imjg8D7Mk,11307
+sglang/srt/models/stablelm.py,sha256=vMZUNgwXKPGYr5FcdYHw5g3QifVu9owKqq51_-EBOY0,10817
+sglang/srt/models/yivl.py,sha256=Qvp-zQ93cOZGg3zVyaiQLhRsfXiLrQhxu9TyQP2FMm4,4414
+sglang/test/test_conversation.py,sha256=1zIrXcXiwEliPHgDAsqsQUA7JKzZ5fnQEU-U6L887FU,1592
+sglang/test/test_openai_protocol.py,sha256=eePzoskYR3PqfWczSVZvg8ja63qbT8TFUNEMyzDZpa8,1657
+sglang/test/test_programs.py,sha256=mrLhGuprwvx8ZJ-0Qe28E-iCw5Qv-9T0SAv1Jgo1AJw,11421
+sglang/test/test_utils.py,sha256=6PhTRi8UnR-BRNjit6aGu0M5lO0RebNQwEcDt712hE4,4830
+sglang-0.1.14.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+sglang-0.1.14.dist-info/METADATA,sha256=C5N0VOYRHixdJcsf4dExIvP-Q099kYBMKs_dA4LBXSM,28809
+sglang-0.1.14.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+sglang-0.1.14.dist-info/top_level.txt,sha256=yxhh3pYQkcnA7v3Bg889C2jZhvtJdEincysO7PEB09M,7
+sglang-0.1.14.dist-info/RECORD,,

{sglang-0.1.12.dist-info → sglang-0.1.14.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.42.0)
+Generator: bdist_wheel (0.43.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

sglang/srt/models/gpt_neox.py

@@ -1,274 +0,0 @@
-# Adapted from
-# https://github.com/vllm-project/vllm/blob/c81dddb45c71e630b907f9d84686ecd73b4105c7/vllm/model_executor/models/gpt_neox.py#L1
-"""Inference-only GPT-NeoX model compatible with HuggingFace weights."""
-from typing import List, Optional, Tuple
-
-import torch
-from torch import nn
-from transformers import GPTNeoXConfig
-
-from vllm.model_executor.layers.activation import get_act_fn
-from sglang.srt.layers.logits_processor import LogitsProcessor
-from sglang.srt.layers.radix_attention import RadixAttention
-from sglang.srt.managers.router.model_runner import InputMetadata
-from vllm.model_executor.layers.linear import (ColumnParallelLinear,
-                                               LinearMethodBase,
-                                               QKVParallelLinear,
-                                               RowParallelLinear)
-from vllm.model_executor.layers.rotary_embedding import get_rope
-from vllm.model_executor.layers.vocab_parallel_embedding import (
-    VocabParallelEmbedding, ParallelLMHead)
-from vllm.model_executor.parallel_utils.parallel_state import (
-    get_tensor_model_parallel_world_size)
-from vllm.model_executor.weight_utils import (default_weight_loader,
-                                              hf_model_weights_iterator)
-
-class GPTNeoXAttention(nn.Module):
-
-    def __init__(
-        self,
-        config: GPTNeoXConfig,
-        layer_id: int = 0,
-        linear_method: Optional[LinearMethodBase] = None,
-    ):
-        super().__init__()
-        self.total_num_heads = config.num_attention_heads
-        self.hidden_size = config.hidden_size
-        self.head_size = self.hidden_size // self.total_num_heads
-        self.bias = getattr(config, "attention_bias", True)
-
-        tensor_model_parallel_world_size = (
-            get_tensor_model_parallel_world_size())
-        assert self.total_num_heads % tensor_model_parallel_world_size == 0
-        self.num_heads = (self.total_num_heads //
-                          tensor_model_parallel_world_size)
-
-        self.query_key_value = QKVParallelLinear(
-            config.hidden_size,
-            self.head_size,
-            self.total_num_heads,
-            bias=self.bias,
-            linear_method=linear_method,
-        )
-        self.dense = RowParallelLinear(
-            config.hidden_size,
-            config.hidden_size,
-            bias=self.bias,
-            linear_method=linear_method,
-        )
-        scaling = self.head_size**-0.5
-        rotary_dim = int(self.head_size * config.rotary_pct)
-        assert rotary_dim % 2 == 0
-        rope_theta = getattr(config, "rope_theta", 10000)
-        max_position_embeddings = getattr(config, "max_position_embeddings",
-                                          8192)
-        self.rotary_emb = get_rope(
-            self.head_size,
-            rotary_dim=rotary_dim,
-            max_position=max_position_embeddings,
-            base=rope_theta,
-        )
-        self.attn = RadixAttention(self.num_heads, 
-                                   self.head_size, 
-                                   scaling, 
-                                   num_kv_heads=self.num_heads,
-                                   layer_id=layer_id)
-
-    def forward(
-        self,
-        position_ids: torch.Tensor,
-        hidden_states: torch.Tensor,
-        input_metadata: InputMetadata,
-    ) -> torch.Tensor:
-        qkv, _ = self.query_key_value(hidden_states)
-        q, k, v = qkv.chunk(chunks=3, dim=-1)
-        q, k = self.rotary_emb(position_ids, q, k)
-        attn_output = self.attn(q, k, v, input_metadata)
-        output, _ = self.dense(attn_output)
-        return output
-
-
-class GPTNeoXMLP(nn.Module):
-
-    def __init__(
-        self,
-        config: GPTNeoXConfig,
-        linear_method: Optional[LinearMethodBase] = None,
-    ):
-        super().__init__()
-        self.dense_h_to_4h = ColumnParallelLinear(
-            config.hidden_size,
-            config.intermediate_size,
-            linear_method=linear_method,
-        )
-        self.dense_4h_to_h = RowParallelLinear(
-            config.intermediate_size,
-            config.hidden_size,
-            linear_method=linear_method,
-        )
-        quant_config = getattr(linear_method, "quant_config", None)
-        self.act = get_act_fn(config.hidden_act, quant_config,
-                              config.intermediate_size)
-
-    def forward(self, hidden_states):
-        hidden_states, _ = self.dense_h_to_4h(hidden_states)
-        hidden_states = self.act(hidden_states)
-        hidden_states, _ = self.dense_4h_to_h(hidden_states)
-        return hidden_states
-
-
-class GPTNeoXLayer(nn.Module):
-
-    def __init__(
-        self,
-        config: GPTNeoXConfig,
-        layer_id: int = 0,
-        linear_method: Optional[LinearMethodBase] = None,
-    ):
-        super().__init__()
-        self.use_parallel_residual = config.use_parallel_residual
-        self.input_layernorm = nn.LayerNorm(config.hidden_size,
-                                            eps=config.layer_norm_eps)
-        self.post_attention_layernorm = nn.LayerNorm(config.hidden_size,
-                                                     eps=config.layer_norm_eps)
-        self.attention = GPTNeoXAttention(config, layer_id=layer_id, linear_method=linear_method)
-        self.mlp = GPTNeoXMLP(config, linear_method)
-
-    def forward(
-        self,
-        position_ids: torch.Tensor,
-        hidden_states: torch.Tensor,
-        input_metadata: InputMetadata,
-    ) -> torch.Tensor:
-        attn_input = self.input_layernorm(hidden_states)
-        attn_output = self.attention(
-            position_ids=position_ids,
-            hidden_states=attn_input,
-            input_metadata=input_metadata,
-        )
-
-        if self.use_parallel_residual:
-            # pseudocode:
-            # x = x + attn(ln1(x)) + mlp(ln2(x))
-            mlp_input = self.post_attention_layernorm(hidden_states)
-            mlp_output = self.mlp(mlp_input)
-            hidden_states = mlp_output + attn_output + hidden_states
-        else:
-            # pseudocode:
-            # x = x + attn(ln1(x))
-            # x = x + mlp(ln2(x))
-            attn_output = attn_output + hidden_states
-            mlp_input = self.post_attention_layernorm(attn_output)
-            mlp_output = self.mlp(mlp_input)
-            hidden_states = mlp_output + attn_output
-        return hidden_states
-
-
-class GPTNeoXModel(nn.Module):
-
-    def __init__(
-        self,
-        config: GPTNeoXConfig,
-        linear_method: Optional[LinearMethodBase] = None,
-    ):
-        super().__init__()
-        self.config = config
-
-        self.embed_in = VocabParallelEmbedding(
-            config.vocab_size,
-            config.hidden_size,
-        )
-        self.layers = nn.ModuleList([
-            GPTNeoXLayer(config, i, linear_method)
-            for i in range(config.num_hidden_layers)
-        ])
-        self.final_layer_norm = nn.LayerNorm(config.hidden_size,
-                                             eps=config.layer_norm_eps)
-
-    def forward(
-        self,
-        input_ids: torch.Tensor,
-        position_ids: torch.Tensor,
-        input_metadata: InputMetadata,
-        skip_embed: bool = False,
-    ) -> torch.Tensor:
-        if not skip_embed:
-            hidden_states = self.embed_in(input_ids)
-        else:
-            hidden_states = input_ids
-        hidden_states = self.embed_in(input_ids)
-        for i in range(len(self.layers)):
-            layer = self.layers[i]
-            hidden_states = layer(
-                position_ids,
-                hidden_states,
-                input_metadata,
-            )
-        hidden_states = self.final_layer_norm(hidden_states)
-        return hidden_states
-
-
-class GPTNeoXForCausalLM(nn.Module):
-
-    def __init__(
-        self,
-        config,
-        linear_method: Optional[LinearMethodBase] = None,
-    ):
-        super().__init__()
-        self.config = config
-        self.linear_method = linear_method
-        self.gpt_neox = GPTNeoXModel(config, linear_method)
-        self.embed_out = ParallelLMHead(
-            config.vocab_size,
-            config.hidden_size,
-        )
-        self.logits_processor = LogitsProcessor(config)
-
-    def forward(
-        self,
-        input_ids: torch.Tensor,
-        positions: torch.Tensor,
-        input_metadata: InputMetadata,
-        skip_embed: bool = False,
-    ) -> torch.Tensor:
-        hidden_states = self.gpt_neox(input_ids, positions, input_metadata, skip_embed)
-        return self.logits_processor(
-            input_ids, hidden_states, self.embed_out.weight, input_metadata
-        )
-
-    def load_weights(self,
-                     model_name_or_path: str,
-                     cache_dir: Optional[str] = None,
-                     load_format: str = "auto",
-                     revision: Optional[str] = None):
-        params_dict = dict(self.named_parameters())
-        for name, loaded_weight in hf_model_weights_iterator(
-                model_name_or_path, cache_dir, load_format, revision):
-            if ("attention.bias" in name or "attention.masked_bias" in name
-                    or "rotary_emb.inv_freq" in name):
-                continue
-            param = params_dict[name]
-
-            if "query_key_value" in name:
-                # NOTE: GPT-NeoX's fused QKV's output_dim has the shape of
-                # (num_heads * 3 * head_size), while the
-                # required shape is (3 * num_heads * head_size).
-                # Thus, we need weight conversion.
-                output_dim = getattr(param, "output_dim", None)
-                num_heads = self.config.num_attention_heads
-                if output_dim is not None:
-                    loaded_weight_shape = loaded_weight.shape
-                    loaded_weight = loaded_weight.view(
-                        loaded_weight_shape[:output_dim] + (num_heads, 3, -1) +
-                        loaded_weight_shape[output_dim + 1:])
-                    loaded_weight = loaded_weight.transpose(
-                        output_dim, output_dim + 1)
-                    loaded_weight = loaded_weight.reshape(loaded_weight_shape)
-
-            weight_loader = getattr(param, "weight_loader",
-                                    default_weight_loader)
-            weight_loader(param, loaded_weight)
-
-
-EntryClass = GPTNeoXForCausalLM

sglang-0.1.12.dist-info/RECORD

@@ -1,63 +0,0 @@
-sglang/__init__.py,sha256=MsaKtUijK193Lw2Hw8ydva_X5Le0sKvWUVeKaOjdYqY,96
-sglang/api.py,sha256=E2G93eTlM7wT451iGEDESZnt0NZjK03Xt0Lyx-NYCOc,4207
-sglang/global_config.py,sha256=PAX7TWeFcq0HBzNUWyCONAOjqIokWqw8vT7I6sBSKTc,797
-sglang/launch_server.py,sha256=jKPZRDN5bUe8Wgz5eoDkqeePhmKa8DLD4DpXQLT5auo,294
-sglang/utils.py,sha256=a3RjlWZ-K2LjO8GTwD_ExYu-QvgSrcDh-_NKeqzBziM,6231
-sglang/backend/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sglang/backend/anthropic.py,sha256=y5TN9EDrJtOH4JEUxpXu-endloeYBy7xMUr3r7Ah3MA,1462
-sglang/backend/base_backend.py,sha256=pPalZfoezxnUBs752j7lm0uMwa8tZuCWd-ijSdStMO8,1745
-sglang/backend/openai.py,sha256=L49Ga3E1rgOyxpH9NyMrKw2Exm-WyDM_pCUQZetCH_Q,8555
-sglang/backend/runtime_endpoint.py,sha256=hx3D-Dv3XAVbnAtbW975RrNN6Jaw2ZvR6XGMFz61h7A,6689
-sglang/backend/vertexai.py,sha256=BLfWf_tEgoHY9srCufJM5PLe3tql2j0G6ia7cPykxCM,4713
-sglang/lang/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sglang/lang/chat_template.py,sha256=9aiR_4euCjrSdduYYiSnVjHE48GOqaHHigwX1oXu0lU,7461
-sglang/lang/compiler.py,sha256=wNn_UqV6Sxl22mv-PpzFUtRgiFFV-Y4OYpO4LshEoRM,7527
-sglang/lang/interpreter.py,sha256=qCIssjVWltToFXpZDfNx6tiemQpXiK_NF_qUPWu_rvU,26262
-sglang/lang/ir.py,sha256=QSx0vMepQ01SaQ4EQjUqbJknHSrF557CqHuosQi6otQ,13330
-sglang/lang/tracer.py,sha256=pFiSNzPSg0l7ZZIlGqJDLCmQALR-wyo2dFgJP73J4_Y,8260
-sglang/srt/backend_config.py,sha256=UIV6kIU2j-Xh0eoezn1aXcYIy0miftHsWFeAZwqpbGE,227
-sglang/srt/conversation.py,sha256=mTstD-SsXG5p_YhWQUPEWU-vzzDMF4RgQ7KmLkOOC7U,15496
-sglang/srt/hf_transformers_utils.py,sha256=soRyYLoCn7GxgxvonufGFkdFBA3eH5i3Izk_wi7p1l0,5285
-sglang/srt/memory_pool.py,sha256=BMoX2wvicj214mV-xvcr_Iv_Je0qs3zTuzXfQVpV8u4,3609
-sglang/srt/mm_utils.py,sha256=OptgAHDX-73Bk4jAdr2BOAJtiEXJNzPrMhaM-dy275c,8889
-sglang/srt/model_config.py,sha256=MDfjfhfZxXTPrshLsZANWyCN8RPS-pCV4RTAcA8IUG8,1124
-sglang/srt/sampling_params.py,sha256=83Fp-4HWThC20TEh139XcIb_erBqfI7KZg5txdRBq7c,2896
-sglang/srt/server.py,sha256=21EdEjG_EmVs4BhL37wI5wNtcmir44CPRX5cZ-5bofM,21454
-sglang/srt/server_args.py,sha256=WihASLqvxYDX65LAtdMzQ2kqbARxFds20jF8HFfXmRc,7567
-sglang/srt/utils.py,sha256=2gCOlsErsFz969V5kaTPm0-jFwOpbIU1ufrRyWkYvYE,7287
-sglang/srt/constrained/__init__.py,sha256=wcGWZNn19jK2m-KN6P4ui1BrQy2wwUt-qG4_Orv0ouY,490
-sglang/srt/constrained/base_cache.py,sha256=QQjmFEiT8jlOskJoZobhrDl2TKB-B4b1LPQo9JQCP_w,1405
-sglang/srt/constrained/fsm_cache.py,sha256=Q0J4St3XUOt2tKFVpj0B2KIZ6z3X6cIzTcjREVqy3pg,471
-sglang/srt/constrained/jump_forward.py,sha256=Z-pz2Jnvk1CxSEZA65OVq0GryqdiKuOkhhc13v5T6Lo,2482
-sglang/srt/layers/context_flashattention_nopad.py,sha256=GkjLiTkS4px_uLcW0aDocE3_OBXtujZ-SlsN2b2U7ng,5204
-sglang/srt/layers/extend_attention.py,sha256=pWVE6ySnPiVLFON__bie73eDhmXHk4tECMK8zTiJNbI,12558
-sglang/srt/layers/logits_processor.py,sha256=MW2bpqSXyghODMojqeMSYWZhUHuAFPk_gUkyyLw9HkM,4827
-sglang/srt/layers/radix_attention.py,sha256=Tl1zE2c87Fm5qW5Ylffkgh48QQIwd93UK6IaRMZltFI,5789
-sglang/srt/layers/token_attention.py,sha256=Z3YVuFmqca3ho2NKSXjLXZNz4L67qrhaec_Pd38IA_4,8157
-sglang/srt/managers/detokenizer_manager.py,sha256=1lPNh_Pe6Pr0v-TzlCBBREbvz4uFWxyw31SmnEZh0s8,3292
-sglang/srt/managers/io_struct.py,sha256=E5Lt81n7-DkRR-pl7XoaJXIBaa2nT9swABNwXEsTsUw,3064
-sglang/srt/managers/openai_protocol.py,sha256=cttqg9iv3de8fhtCqDI4cYoPPZ_gULedMXstV1ok6WA,4563
-sglang/srt/managers/tokenizer_manager.py,sha256=B-F6diI1sV0pW1HxQgH_v8VzJwzvgkcGB1_MkOh2unE,9693
-sglang/srt/managers/router/infer_batch.py,sha256=bvUY1EmIKqdF38N5ALWVsgD2rl6GNzvUHMGkluoUpv8,18126
-sglang/srt/managers/router/manager.py,sha256=TNYs0IrkZGkPvZJViwL7BMUg0VlvzeyTjDMjuvRoMDI,2529
-sglang/srt/managers/router/model_rpc.py,sha256=dKBRzPoERK-TCgUgnaQfFFlZtB6_xWT9eSTVwizCCiA,25938
-sglang/srt/managers/router/model_runner.py,sha256=TUEqfsQedPUFGA5cpTYi8sW5whtEM-4ui3s7YoP0cBg,17604
-sglang/srt/managers/router/radix_cache.py,sha256=XGUF5mxQTSCzD7GW_ltNP2p5aelEKrMXzdezufJ7NCQ,6484
-sglang/srt/managers/router/scheduler.py,sha256=V-LAnVSzgD2ddy2eXW3jWURCeq9Lv7YxCGk4kHyytfM,2818
-sglang/srt/models/gpt_neox.py,sha256=0NwrX9hqVD9biE0bfJYFC4TMhQKhYdNowLkVYo2OG24,10271
-sglang/srt/models/llama2.py,sha256=tICX536zPcPup3KmwRmRASqBSgyY6_XpThjfMpQ1evM,11582
-sglang/srt/models/llava.py,sha256=OaJF9Lal4Txtg_FuDsQTL_kHR7PB1BUf3nhngCdFnfU,14963
-sglang/srt/models/mistral.py,sha256=XSn7fiZqspyWVTYrpVAacAnWdwAybBtyn9-Sh9AvMTM,254
-sglang/srt/models/mixtral.py,sha256=iTuuyJdT8cq6W7CCqHz5nyY6I8r_m1SLMiGfGTkPW6w,13886
-sglang/srt/models/qwen.py,sha256=xPvwO4YBhht4ROSK-ef9Zysk_UvB06GxCBPYjyElgUY,9225
-sglang/srt/models/qwen2.py,sha256=B_dH2QQtfuz38LmGOGcFnQwTRMRFUkHAaO32rnNB_-s,11278
-sglang/srt/models/yivl.py,sha256=Qvp-zQ93cOZGg3zVyaiQLhRsfXiLrQhxu9TyQP2FMm4,4414
-sglang/test/test_conversation.py,sha256=1zIrXcXiwEliPHgDAsqsQUA7JKzZ5fnQEU-U6L887FU,1592
-sglang/test/test_openai_protocol.py,sha256=eePzoskYR3PqfWczSVZvg8ja63qbT8TFUNEMyzDZpa8,1657
-sglang/test/test_programs.py,sha256=mrLhGuprwvx8ZJ-0Qe28E-iCw5Qv-9T0SAv1Jgo1AJw,11421
-sglang/test/test_utils.py,sha256=DyZAic3KIBQ0PmZeLc9uv1ckcM5jpEE5CirjHO48_sk,4829
-sglang-0.1.12.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-sglang-0.1.12.dist-info/METADATA,sha256=4Q0u9J9QUQFlAbMnzioXZ0i47F-HZznyKA7qcnrv_K4,28129
-sglang-0.1.12.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-sglang-0.1.12.dist-info/top_level.txt,sha256=yxhh3pYQkcnA7v3Bg889C2jZhvtJdEincysO7PEB09M,7
-sglang-0.1.12.dist-info/RECORD,,