sglang 0.1.16__py3-none-any.whl → 0.1.17__py3-none-any.whl

sglang-0.1.17.dist-info/RECORD

@@ -0,0 +1,81 @@
+sglang/__init__.py,sha256=yEHUYdlMU-BtdYBBPSNKnqUTfQ4cdwWwWqA1BfLVB1M,1116
+sglang/api.py,sha256=imnZeqgNmkex9Wg3B5VQ1M8FlBZzx9Wh9D0q5ibO0Bc,4548
+sglang/global_config.py,sha256=Osa7UjpAXjEcULYvMUSa93JrvNP03vR0xLGy2gQ6uJw,1233
+sglang/launch_server.py,sha256=jKPZRDN5bUe8Wgz5eoDkqeePhmKa8DLD4DpXQLT5auo,294
+sglang/launch_server_llavavid.py,sha256=UWo_qUCJ9yknp1TVPzrz4B_aZtEuQpLQq0l96FMgynI,1058
+sglang/utils.py,sha256=-IlcZtGHnOB4Gl_ltsQZPw9Epe7maUnXFTRtvMniw2k,8146
+sglang/backend/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+sglang/backend/anthropic.py,sha256=iJjXiDMZbtvX2XNG78MG9kM7SpZq9hmXVuzT_T18elw,2076
+sglang/backend/base_backend.py,sha256=APiMht4WYECLCOGRPCEUF6lX-an1vjVe2dWoMSgymWY,1831
+sglang/backend/litellm.py,sha256=Y8lfWN0z8_hKvLMJbl-Xuw7Yn_5drNusC_wJv4BOQUY,2439
+sglang/backend/openai.py,sha256=Xv_QJc6tN5W1Da2fu3kzvrrfT9RvW921_Cwq8R_Ak9Y,14711
+sglang/backend/runtime_endpoint.py,sha256=8NyWgMvhzUcA5VEsPLo1AacZ_UPVSnpxpzt6vYdVQSU,8871
+sglang/backend/vertexai.py,sha256=XNkbUzOdLIz-1qP_BBieYIfUXZf6gsfdghlaulNpBM8,4714
+sglang/lang/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+sglang/lang/chat_template.py,sha256=ogIT8iMlDcSEgcNBTh5pRLoCkdQI_ec5Hc27wFUFDIg,11532
+sglang/lang/compiler.py,sha256=wNn_UqV6Sxl22mv-PpzFUtRgiFFV-Y4OYpO4LshEoRM,7527
+sglang/lang/interpreter.py,sha256=_QIzpnfSr02JUkeaJzTcZxxF4gv0naY16fvVkDZH9xE,29493
+sglang/lang/ir.py,sha256=EMAXzC7upkx6qvKzCss8p7OSQYAXCT1hCl649s0Kp_c,13882
+sglang/lang/tracer.py,sha256=QcslAObEjepk8XmiqCobwzWaDpihofEQXjeRs_3B8NQ,8282
+sglang/srt/conversation.py,sha256=NwTVuQXd3NqPq5WCllaYUgPLG2w2pMMbzIKDQfJMMO0,15491
+sglang/srt/flush_cache.py,sha256=N0etybT9tIS8_zreJFu64j9TYHKiR3sVXMTjHwHK8X0,382
+sglang/srt/hf_transformers_utils.py,sha256=3aDNhwxaaObiMCrw9nqzBILoosIx1-Qy7COK6NIHtog,8244
+sglang/srt/memory_pool.py,sha256=5bqI8d5_JURbKwIhv1BwlcIO2IDHewHvIqezPG-b_5M,3284
+sglang/srt/mm_utils.py,sha256=OptgAHDX-73Bk4jAdr2BOAJtiEXJNzPrMhaM-dy275c,8889
+sglang/srt/model_config.py,sha256=6XJHUtev-hI-E3NAIoWiNKtpZfN2hHoaxs_r79vGDe8,1724
+sglang/srt/openai_api_adapter.py,sha256=BDUwhTQpFJHHnWsw4a9XsoGhEZkfgZqd3EUbkD5g5ko,15089
+sglang/srt/openai_protocol.py,sha256=jChImDalBjYk9tzBccb_m5eVVJExdHm9LhCJ4Cso5LU,5350
+sglang/srt/sampling_params.py,sha256=dQbVr7JmTJ9JEn_sy3clB56yT9kyr9ldWFZ-GaNXOy0,3023
+sglang/srt/server.py,sha256=O1lJq6F95ZHeVb4aantcE7SnnM3XM7JSCa6il8vf_Mg,11595
+sglang/srt/server_args.py,sha256=N5sLrpLBL6Zkfspgvanl8-9bKhMSM2Lrv9gHJ8ENmLc,10822
+sglang/srt/utils.py,sha256=pvyyPvJF6RnoR0DG0wSDo73mSS_2x2MhtKqVmXObtyA,14654
+sglang/srt/constrained/__init__.py,sha256=BPRNDJnWtzYJ13X4urRS5aE6wFuwAVNBA9qeWIHF8rE,1236
+sglang/srt/constrained/base_cache.py,sha256=QQjmFEiT8jlOskJoZobhrDl2TKB-B4b1LPQo9JQCP_w,1405
+sglang/srt/constrained/fsm_cache.py,sha256=RmAdaAAXlh_KeDiK4w3AARiEnvrbsuELROBgMzJvZKk,967
+sglang/srt/constrained/jump_forward.py,sha256=fUa4AlnGX40gYiWTLuICTJfq4b7wA3AL5dydTqT3jz4,2483
+sglang/srt/layers/context_flashattention_nopad.py,sha256=bENdVltDozccR5mLY_CcYDjqLob28tHA9f2s03D8UFQ,5210
+sglang/srt/layers/extend_attention.py,sha256=JUYuYSAhfbgOXrwIK5YHJCXPq54a6IZ7vQrze-3VvMQ,12955
+sglang/srt/layers/fused_moe.py,sha256=0JchWmMrqF4Dqn3_gcBcaS2_uypgmOiEE0vjfo-l24U,19484
+sglang/srt/layers/logits_processor.py,sha256=96WMfpBAD-nQNq4cQ4edfhqqS3HuDkAIj42EWj_8Rwo,7283
+sglang/srt/layers/radix_attention.py,sha256=xsF8G-jrXi076Xwk_7-eD-FbNJvDvGGH6Pk4EzMUduA,5818
+sglang/srt/layers/token_attention.py,sha256=rVbPlFpmLoU3nx3qtK2YZdynDxfvMKtQNTPeKi0KNP0,8823
+sglang/srt/managers/detokenizer_manager.py,sha256=XzhlONpgAQBPUWotCGJn6XnIA7YTm6JEmHxj0Zbn6_A,3452
+sglang/srt/managers/io_struct.py,sha256=oWHLvrdszhY8y5pNlFtoVYDBVslEM-rMCegIfbFYOco,4370
+sglang/srt/managers/tokenizer_manager.py,sha256=uV8JuASF2pm95Hvit9dUF4y7juowp1aZ_Yl26Wh-mr0,14827
+sglang/srt/managers/controller/dp_worker.py,sha256=xN7oQ3TG0FeX5K7nv6p3tUXCSE8wn0svdBHFePWe2ZU,3635
+sglang/srt/managers/controller/infer_batch.py,sha256=sMjntty1MPDo__QzsxBVavMFeVIBlWU1x2lfRIP3Fmg,22716
+sglang/srt/managers/controller/manager_multi.py,sha256=VmDkViOc3KFZA5HCcqC1mTmwuVda95NqELzMOrjNsp4,6629
+sglang/srt/managers/controller/manager_single.py,sha256=CMaEl304o1SvNl3t-BpBrrQeyjmfdxNkKxlZh_c49sQ,3222
+sglang/srt/managers/controller/model_runner.py,sha256=08HpdJYih-Nz_IlJ5a_53bb133ESEJ18Y_KSNJ0mTNQ,16993
+sglang/srt/managers/controller/radix_cache.py,sha256=QnScfPDzy_QgZt0nM2BzDI_hDiohmDpJ8QKlAHAspxw,8127
+sglang/srt/managers/controller/schedule_heuristic.py,sha256=DUNbv8DWSjk6I1pabfPGTYhZRz8vAFCsAh8IQcm1jxM,2276
+sglang/srt/managers/controller/tp_worker.py,sha256=7qkDHURfeEPDSbUuN_-glwdgJ66H6dXd49yV8DT5JK0,31306
+sglang/srt/managers/router/infer_batch.py,sha256=PEq_tCQNnmSDerlL6RRjJKadFwgP0r7l67OZypHq-II,22088
+sglang/srt/managers/router/manager.py,sha256=3kTf05O2ADU91wIDoFpIZJXEz1dWeMKis0hn7j1dbzo,2693
+sglang/srt/managers/router/model_rpc.py,sha256=-W-oWF1nOiWp7TwjTUo0DN4-mPdTK4S8noiVkLoQ-vo,31877
+sglang/srt/managers/router/model_runner.py,sha256=PG7iSADgk_E1Eb60mS13Gl5MgHidEmi3YnO4k_Oz-7E,16515
+sglang/srt/managers/router/radix_cache.py,sha256=QnScfPDzy_QgZt0nM2BzDI_hDiohmDpJ8QKlAHAspxw,8127
+sglang/srt/managers/router/scheduler.py,sha256=od3fjTNyTjwTDbXVfT8jEHNPvNDk6Ss9NUUkIeXyq8s,2268
+sglang/srt/models/commandr.py,sha256=JWjljtNr_t_L9PdPuymo6beUS0_EJ7NHZHrhKD3xoL0,13606
+sglang/srt/models/dbrx.py,sha256=Wr45o_DTU1YTq3h5caTAH_1R3nYCSwRyKha64Ygl4Ak,14074
+sglang/srt/models/gemma.py,sha256=rOw9WBNZqdeKfJT9wUa-y5sAj-pAj0YNfjk-dKtxEhA,11501
+sglang/srt/models/grok.py,sha256=R_Y6CptcPgYvRt9YWob-LG2D3hTCa9VxjmA2k734Xlg,26944
+sglang/srt/models/llama2.py,sha256=-IKmBoUDcZ76dRjMSNy0rUPB7NdDh4Ayc8skV0WlRCA,11959
+sglang/srt/models/llava.py,sha256=S9Kz87les4Z_nZ2KAp1ZgmaK-ntILdZHqqqadJBLAt4,17893
+sglang/srt/models/llavavid.py,sha256=8SVkICyDSvsw-5aSmGqSLT9S1xw8ouH0gJmAAeFLOPo,13029
+sglang/srt/models/mistral.py,sha256=XSn7fiZqspyWVTYrpVAacAnWdwAybBtyn9-Sh9AvMTM,254
+sglang/srt/models/mixtral.py,sha256=dDdwkxHOfZdtfr3CixjXIZwNmB5DBfZPSQGmdz2-cJQ,20727
+sglang/srt/models/mixtral_quant.py,sha256=ZP5YfMaZUfthXwSO_84o6L6Be8RhJR-1-lvG5w42wis,13636
+sglang/srt/models/qwen.py,sha256=5Q10AAzBy79SRtZinpnRQYJskjGst2jf4IhJBkmDtjE,9419
+sglang/srt/models/qwen2.py,sha256=_7wLaaDEs_RUgS1cjC8wgk7JqJ6CngHPNTMsDdH5Yok,11465
+sglang/srt/models/stablelm.py,sha256=rzkCKYC0mGg1geFTedcbtyoOFgr_s9HacYbdb_9XJMU,10781
+sglang/srt/models/yivl.py,sha256=wHaoyC2JAvhWssfgwN84BRG8CND4d7TMj1Q-pzbDea8,4367
+sglang/test/test_conversation.py,sha256=1zIrXcXiwEliPHgDAsqsQUA7JKzZ5fnQEU-U6L887FU,1592
+sglang/test/test_openai_protocol.py,sha256=eePzoskYR3PqfWczSVZvg8ja63qbT8TFUNEMyzDZpa8,1657
+sglang/test/test_programs.py,sha256=HIfIEjO6fgBmbLIy4z4zpbz6oVw2GvHP8CeVQd69YDU,13378
+sglang/test/test_utils.py,sha256=Mjn2btfmEQQ7rpsLfNo6VugXCPzUmRpNhssWvxevN4s,11038
+sglang-0.1.17.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+sglang-0.1.17.dist-info/METADATA,sha256=AZQ36_LEiRR8Bf2AmS0qQMdFBmQK8boZwnlgFaLeoUg,29242
+sglang-0.1.17.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+sglang-0.1.17.dist-info/top_level.txt,sha256=yxhh3pYQkcnA7v3Bg889C2jZhvtJdEincysO7PEB09M,7
+sglang-0.1.17.dist-info/RECORD,,

sglang/srt/backend_config.py

@@ -1,13 +0,0 @@
-"""
-Backend configurations, may vary with different serving platforms.
-"""
-
-from dataclasses import dataclass
-
-
-@dataclass
-class BackendConfig:
-    extend_dependency_time: float = 0.03
-
-
-GLOBAL_BACKEND_CONFIG = BackendConfig()

sglang/srt/models/dbrx_config.py

@@ -1,281 +0,0 @@
-# Adapted from:
-# https://github.com/vllm-project/vllm/blob/14ccd94c89d0ffd9da283545d93ab1dfea5da340/vllm/transformers_utils/configs/dbrx.py
-# yapf: disable
-# ruff: noqa: E501
-# coding=utf-8
-# Copied from
-# https://huggingface.co/databricks/dbrx-base/blob/main/configuration_dbrx.py
-"""Dbrx configuration."""
-
-# FIXME: remove this once vllm releases a new version
-
-from typing import Any, Optional
-
-from transformers.configuration_utils import PretrainedConfig
-from transformers.utils import logging
-
-logger = logging.get_logger(__name__)
-
-DBRX_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
-
-
-class DbrxAttentionConfig(PretrainedConfig):
-    """Configuration class for Dbrx Attention.
-
-    [`DbrxAttention`] class. It is used to instantiate attention layers
-    according to the specified arguments, defining the layers architecture.
-
-    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
-    documentation from [`PretrainedConfig`] for more information.
-
-    Args:
-        attn_pdrop (`float`, *optional*, defaults to 0.0):
-            The dropout probability for the attention layers.
-        clip_qkv (`float`, *optional*, defaults to None):
-            If not `None`, clip the queries, keys, and values in the attention layer to this value.
-        kv_n_heads (Optional[int]): For grouped_query_attention only, allow user to specify number of kv heads.
-        rope_theta (float): The base frequency for rope.
-    """
-
-    def __init__(
-        self,
-        attn_pdrop: float = 0,
-        clip_qkv: Optional[float] = None,
-        kv_n_heads: int = 1,
-        rope_theta: float = 10000.0,
-        **kwargs: Any,
-    ):
-        super().__init__(**kwargs)
-        self.attn_pdrop = attn_pdrop
-        self.clip_qkv = clip_qkv
-        self.kv_n_heads = kv_n_heads
-        self.rope_theta = rope_theta
-
-        for k in ["model_type"]:
-            if k in kwargs:
-                kwargs.pop(k)
-        if len(kwargs) != 0:
-            raise ValueError(f"Found unknown {kwargs=}")
-
-    @classmethod
-    def from_pretrained(
-        cls, pretrained_model_name_or_path: str, **kwargs: Any
-    ) -> "PretrainedConfig":
-        cls._set_token_in_kwargs(kwargs)
-
-        config_dict, kwargs = cls.get_config_dict(
-            pretrained_model_name_or_path, **kwargs
-        )
-
-        if config_dict.get("model_type") == "dbrx":
-            config_dict = config_dict["attn_config"]
-
-        if (
-            "model_type" in config_dict
-            and hasattr(cls, "model_type")
-            and config_dict["model_type"] != cls.model_type
-        ):
-            logger.warning(
-                f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
-                + f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
-            )
-
-        return cls.from_dict(config_dict, **kwargs)
-
-
-class DbrxFFNConfig(PretrainedConfig):
-    """Configuration class for Dbrx FFN.
-
-    [`DbrxFFN`] class. It is used to instantiate feedforward layers according to
-    the specified arguments, defining the layers architecture.
-
-    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
-    documentation from [`PretrainedConfig`] for more information.
-
-    Args:
-        ffn_act_fn (dict, optional): A dict specifying activation function for the FFN.
-            The dict should have a key 'name' with the value being the name of
-            the activation function along with any additional keyword arguments.
-        ffn_hidden_size (int, optional): The hidden size of the feedforward network.
-        moe_num_experts (int, optional): The number of experts in the mixture of experts layer.
-        moe_top_k (int, optional): The number of experts to use in the mixture of experts layer.
-        moe_jitter_eps (float, optional): The jitter epsilon for the mixture of experts layer.
-        moe_loss_weight (float, optional): The loss weight for the mixture of experts layer.
-        moe_normalize_expert_weights (float, optional): The normalization factor for the expert weights.
-        uniform_expert_assignment (bool, optional): Whether to use uniform expert assignment.
-            This should only be used for benchmarking purposes.
-    """
-
-    def __init__(
-        self,
-        ffn_act_fn: Optional[dict] = None,
-        ffn_hidden_size: int = 3584,
-        moe_num_experts: int = 4,
-        moe_top_k: int = 1,
-        moe_jitter_eps: Optional[float] = None,
-        moe_loss_weight: float = 0.01,
-        moe_normalize_expert_weights: Optional[float] = 1,
-        uniform_expert_assignment: bool = False,
-        **kwargs: Any,
-    ):
-        super().__init__()
-        if ffn_act_fn is None:
-            ffn_act_fn = {"name": "silu"}
-        self.ffn_act_fn = ffn_act_fn
-        self.ffn_hidden_size = ffn_hidden_size
-        self.moe_num_experts = moe_num_experts
-        self.moe_top_k = moe_top_k
-        self.moe_jitter_eps = moe_jitter_eps
-        self.moe_loss_weight = moe_loss_weight
-        self.moe_normalize_expert_weights = moe_normalize_expert_weights
-        self.uniform_expert_assignment = uniform_expert_assignment
-
-        for k in ["model_type"]:
-            if k in kwargs:
-                kwargs.pop(k)
-        if len(kwargs) != 0:
-            raise ValueError(f"Found unknown {kwargs=}")
-
-    @classmethod
-    def from_pretrained(
-        cls, pretrained_model_name_or_path: str, **kwargs: Any
-    ) -> "PretrainedConfig":
-        cls._set_token_in_kwargs(kwargs)
-
-        config_dict, kwargs = cls.get_config_dict(
-            pretrained_model_name_or_path, **kwargs
-        )
-
-        if config_dict.get("model_type") == "dbrx":
-            config_dict = config_dict["ffn_config"]
-
-        if (
-            "model_type" in config_dict
-            and hasattr(cls, "model_type")
-            and config_dict["model_type"] != cls.model_type
-        ):
-            logger.warning(
-                f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
-                + f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
-            )
-
-        return cls.from_dict(config_dict, **kwargs)
-
-
-class DbrxConfig(PretrainedConfig):
-    """Configuration class for Dbrx.
-
-    [`DbrxModel`]. It is used to instantiate a Dbrx model according to the
-    specified arguments, defining the model architecture.
-
-    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
-    documentation from [`PretrainedConfig`] for more information.
-
-
-    Args:
-        d_model (`int`, *optional*, defaults to 6144):
-            Dimensionality of the embeddings and hidden states.
-        n_heads (`int`, *optional*, defaults to 48):
-            Number of attention heads for each attention layer in the Transformer encoder.
-        n_layers (`int`, *optional*, defaults to 40):
-            Number of hidden layers in the Transformer encoder.
-        max_seq_len (`int`, *optional*, defaults to 32768):
-            The maximum sequence length of the model.
-        vocab_size (`int`, *optional*, defaults to 100352):
-            Vocabulary size of the Dbrx model. Defines the maximum number of different tokens that can be represented by
-            the `inputs_ids` passed when calling [`DbrxModel`].
-        resid_pdrop (`float`, *optional*, defaults to 0.0):
-            The dropout probability applied to the attention output before combining with residual.
-        emb_pdrop (`float`, *optional*, defaults to 0.0):
-            The dropout probability for the embedding layer.
-        attn_config (`dict`, *optional*):
-            A dictionary used to configure the model's attention module.
-        ffn_config (`dict`, *optional*):
-            A dictionary used to configure the model's FFN module.
-        use_cache (`bool`, *optional*, defaults to `False`):
-            Whether or not the model should return the last key/values attentions (not used by all models).
-        initializer_range (`float`, *optional*, defaults to 0.02):
-            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
-        output_router_logits (`bool`, *optional*, defaults to `False`):
-            Whether or not the router logits should be returned by the model. Enabling this will also
-            allow the model to output the auxiliary loss. See [here]() for more details
-        router_aux_loss_coef (`float`, *optional*, defaults to 0.001):
-            The aux loss factor for the total loss.
-
-
-    Example:
-    ```python
-    >>> from transformers import DbrxConfig, DbrxModel
-
-    >>> # Initializing a Dbrx configuration
-    >>> configuration = DbrxConfig()
-
-    >>> # Initializing a model (with random weights) from the configuration
-    >>> model = DbrxModel(configuration)
-
-    >>> # Accessing the model configuration
-    >>> configuration = model.config
-    ```
-    """
-
-    model_type = "dbrx"
-    attribute_map = {
-        "num_attention_heads": "n_heads",
-        "hidden_size": "d_model",
-        "num_hidden_layers": "n_layers",
-        "max_position_embeddings": "max_seq_len",
-    }
-
-    def __init__(
-        self,
-        d_model: int = 2048,
-        n_heads: int = 16,
-        n_layers: int = 24,
-        max_seq_len: int = 2048,
-        vocab_size: int = 32000,
-        resid_pdrop: float = 0.0,
-        emb_pdrop: float = 0.0,
-        attn_config: Optional[DbrxAttentionConfig] = None,
-        ffn_config: Optional[DbrxFFNConfig] = None,
-        use_cache: bool = True,
-        initializer_range: float = 0.02,
-        output_router_logits: bool = False,
-        router_aux_loss_coef: float = 0.05,
-        **kwargs: Any,
-    ):
-        if attn_config is None:
-            self.attn_config = DbrxAttentionConfig()
-        elif isinstance(attn_config, dict):
-            self.attn_config = DbrxAttentionConfig(**attn_config)
-        else:
-            self.attn_config = attn_config
-
-        if ffn_config is None:
-            self.ffn_config = DbrxFFNConfig()
-        elif isinstance(ffn_config, dict):
-            self.ffn_config = DbrxFFNConfig(**ffn_config)
-        else:
-            self.ffn_config = ffn_config
-
-        self.d_model = d_model
-        self.n_heads = n_heads
-        self.n_layers = n_layers
-        self.max_seq_len = max_seq_len
-        self.vocab_size = vocab_size
-        self.resid_pdrop = resid_pdrop
-        self.emb_pdrop = emb_pdrop
-        self.use_cache = use_cache
-        self.initializer_range = initializer_range
-        self.output_router_logits = output_router_logits
-        self.router_aux_loss_coef = router_aux_loss_coef
-
-        tie_word_embeddings = kwargs.pop("tie_word_embeddings", False)
-        if tie_word_embeddings:
-            raise ValueError(
-                "tie_word_embeddings is not supported for Dbrx models."
-            )
-
-        super().__init__(
-            tie_word_embeddings=tie_word_embeddings,
-            **kwargs,
-        )