sglang 0.1.22__py3-none-any.whl → 0.1.25__py3-none-any.whl

sglang/backend/runtime_endpoint.py

@@ -1,283 +0,0 @@
-import json
-from typing import List, Optional
-
-import numpy as np
-
-from sglang.backend.base_backend import BaseBackend
-from sglang.global_config import global_config
-from sglang.lang.chat_template import get_chat_template_by_model_path
-from sglang.lang.interpreter import StreamExecutor
-from sglang.lang.ir import SglSamplingParams
-from sglang.utils import http_request
-
-
-class RuntimeEndpoint(BaseBackend):
-    def __init__(
-        self,
-        base_url: str,
-        auth_token: Optional[str] = None,
-        api_key: Optional[str] = None,
-        verify: Optional[str] = None,
-    ):
-        super().__init__()
-        self.support_concate_and_append = True
-
-        self.base_url = base_url
-        self.auth_token = auth_token
-        self.api_key = api_key
-        self.verify = verify
-
-        res = http_request(
-            self.base_url + "/get_model_info",
-            auth_token=self.auth_token,
-            api_key=self.api_key,
-            verify=self.verify,
-        )
-        self._assert_success(res)
-        self.model_info = res.json()
-
-        self.chat_template = get_chat_template_by_model_path(
-            self.model_info["model_path"]
-        )
-
-    def get_model_name(self):
-        return self.model_info["model_path"]
-
-    def flush_cache(self):
-        res = http_request(
-            self.base_url + "/flush_cache",
-            auth_token=self.auth_token,
-            verify=self.verify,
-        )
-        self._assert_success(res)
-
-    def get_server_args(self):
-        res = http_request(
-            self.base_url + "/get_server_args",
-            auth_token=self.auth_token,
-            verify=self.verify,
-        )
-        self._assert_success(res)
-        return res.json()
-
-    def get_chat_template(self):
-        return self.chat_template
-
-    def cache_prefix(self, prefix_str: str):
-        res = http_request(
-            self.base_url + "/generate",
-            json={"text": prefix_str, "sampling_params": {"max_new_tokens": 0}},
-            auth_token=self.auth_token,
-            api_key=self.api_key,
-            verify=self.verify,
-        )
-        self._assert_success(res)
-
-    def commit_lazy_operations(self, s: StreamExecutor):
-        data = {"text": s.text_, "sampling_params": {"max_new_tokens": 0}}
-        self._add_images(s, data)
-        res = http_request(
-            self.base_url + "/generate",
-            json=data,
-            auth_token=self.auth_token,
-            api_key=self.api_key,
-            verify=self.verify,
-        )
-        self._assert_success(res)
-
-    def fill_image(self, s: StreamExecutor):
-        data = {"text": s.text_, "sampling_params": {"max_new_tokens": 0}}
-        self._add_images(s, data)
-        res = http_request(
-            self.base_url + "/generate",
-            json=data,
-            auth_token=self.auth_token,
-            api_key=self.api_key,
-            verify=self.verify,
-        )
-        self._assert_success(res)
-
-    def generate(
-        self,
-        s: StreamExecutor,
-        sampling_params: SglSamplingParams,
-    ):
-        if sampling_params.dtype is None:
-            data = {
-                "text": s.text_,
-                "sampling_params": {
-                    "skip_special_tokens": global_config.skip_special_tokens_in_output,
-                    "spaces_between_special_tokens": global_config.spaces_between_special_tokens_in_out,
-                    **sampling_params.to_srt_kwargs(),
-                },
-            }
-        elif sampling_params.dtype in [int, "int"]:
-            data = {
-                "text": s.text_,
-                "sampling_params": {
-                    "skip_special_tokens": global_config.skip_special_tokens_in_output,
-                    "spaces_between_special_tokens": global_config.spaces_between_special_tokens_in_out,
-                    "dtype": "int",
-                    **sampling_params.to_srt_kwargs(),
-                },
-            }
-        else:
-            raise RuntimeError(f"Invalid dtype: {sampling_params.dtype}")
-
-        for item in [
-            "return_logprob",
-            "logprob_start_len",
-            "top_logprobs_num",
-            "return_text_in_logprobs",
-        ]:
-            value = getattr(sampling_params, item, None)
-            if value is not None:
-                data[item] = value
-
-        self._add_images(s, data)
-
-        res = http_request(
-            self.base_url + "/generate",
-            json=data,
-            auth_token=self.auth_token,
-            api_key=self.api_key,
-            verify=self.verify,
-        )
-        self._assert_success(res)
-
-        obj = res.json()
-        comp = obj["text"]
-        return comp, obj["meta_info"]
-
-    def generate_stream(
-        self,
-        s: StreamExecutor,
-        sampling_params: SglSamplingParams,
-    ):
-        if sampling_params.dtype is None:
-            data = {
-                "text": s.text_,
-                "sampling_params": {
-                    "skip_special_tokens": global_config.skip_special_tokens_in_output,
-                    "spaces_between_special_tokens": global_config.spaces_between_special_tokens_in_out,
-                    **sampling_params.to_srt_kwargs(),
-                },
-            }
-        elif sampling_params.dtype in [int, "int"]:
-            data = {
-                "text": s.text_,
-                "sampling_params": {
-                    "skip_special_tokens": global_config.skip_special_tokens_in_output,
-                    "spaces_between_special_tokens": global_config.spaces_between_special_tokens_in_out,
-                    "dtype": "int",
-                    **sampling_params.to_srt_kwargs(),
-                },
-            }
-        else:
-            raise RuntimeError(f"Invalid dtype: {sampling_params.dtype}")
-
-        for item in [
-            "return_logprob",
-            "logprob_start_len",
-            "top_logprobs_num",
-            "return_text_in_logprobs",
-        ]:
-            value = getattr(sampling_params, item, None)
-            if value is not None:
-                data[item] = value
-
-        data["stream"] = True
-        self._add_images(s, data)
-
-        res = http_request(
-            self.base_url + "/generate",
-            json=data,
-            stream=True,
-            auth_token=self.auth_token,
-            api_key=self.api_key,
-            verify=self.verify,
-        )
-        self._assert_success(res)
-        pos = 0
-
-        for chunk in res.iter_lines(decode_unicode=False):
-            chunk = chunk.decode("utf-8")
-            if chunk and chunk.startswith("data:"):
-                if chunk == "data: [DONE]":
-                    break
-                data = json.loads(chunk[5:].strip("\n"))
-                chunk_text = data["text"][pos:]
-                meta_info = data["meta_info"]
-                pos += len(chunk_text)
-                yield chunk_text, meta_info
-
-    def select(
-        self,
-        s: StreamExecutor,
-        choices: List[str],
-        temperature: float,
-    ):
-        assert temperature <= 1e-5
-
-        # Cache common prefix
-        data = {"text": s.text_, "sampling_params": {"max_new_tokens": 0}}
-        self._add_images(s, data)
-        res = http_request(
-            self.base_url + "/generate",
-            json=data,
-            auth_token=self.auth_token,
-            api_key=self.api_key,
-            verify=self.verify,
-        )
-        self._assert_success(res)
-        prompt_len = res.json()["meta_info"]["prompt_tokens"]
-
-        # Compute logprob
-        data = {
-            "text": [s.text_ + c for c in choices],
-            "sampling_params": {"max_new_tokens": 0},
-            "return_logprob": True,
-            "logprob_start_len": max(prompt_len - 2, 0),
-        }
-        self._add_images(s, data)
-        res = http_request(
-            self.base_url + "/generate",
-            json=data,
-            auth_token=self.auth_token,
-            api_key=self.api_key,
-            verify=self.verify,
-        )
-        self._assert_success(res)
-        obj = res.json()
-        normalized_prompt_logprobs = [
-            r["meta_info"]["normalized_prompt_logprob"] for r in obj
-        ]
-        decision = choices[np.argmax(normalized_prompt_logprobs)]
-        prefill_token_logprobs = [r["meta_info"]["prefill_token_logprobs"] for r in obj]
-        decode_token_logprobs = [r["meta_info"]["decode_token_logprobs"] for r in obj]
-
-        return (
-            decision,
-            normalized_prompt_logprobs,
-            prefill_token_logprobs,
-            decode_token_logprobs,
-        )
-
-    def concatenate_and_append(self, src_rids: List[str], dst_rid: str):
-        res = http_request(
-            self.base_url + "/concate_and_append_request",
-            json={"src_rids": src_rids, "dst_rid": dst_rid},
-            auth_token=self.auth_token,
-            api_key=self.api_key,
-            verify=self.verify,
-        )
-        self._assert_success(res)
-
-    def _add_images(self, s: StreamExecutor, data):
-        if s.images_:
-            assert len(s.images_) == 1, "Only support one image."
-            data["image_data"] = s.images_[0][1]
-
-    def _assert_success(self, res):
-        if res.status_code != 200:
-            raise RuntimeError(res.json())

sglang/backend/vertexai.py

@@ -1,149 +0,0 @@
-import os
-import warnings
-from typing import Optional
-
-from sglang.backend.base_backend import BaseBackend
-from sglang.lang.chat_template import get_chat_template
-from sglang.lang.interpreter import StreamExecutor
-from sglang.lang.ir import SglSamplingParams
-
-try:
-    import vertexai
-    from vertexai.preview.generative_models import (
-        GenerationConfig,
-        GenerativeModel,
-        Image,
-    )
-except ImportError as e:
-    GenerativeModel = e
-
-
-class VertexAI(BaseBackend):
-    def __init__(self, model_name, safety_settings=None):
-        super().__init__()
-
-        if isinstance(GenerativeModel, Exception):
-            raise GenerativeModel
-
-        project_id = os.environ["GCP_PROJECT_ID"]
-        location = os.environ.get("GCP_LOCATION")
-        vertexai.init(project=project_id, location=location)
-
-        self.model_name = model_name
-        self.chat_template = get_chat_template("default")
-        self.safety_settings = safety_settings
-
-    def get_chat_template(self):
-        return self.chat_template
-
-    def generate(
-        self,
-        s: StreamExecutor,
-        sampling_params: SglSamplingParams,
-    ):
-        if s.messages_:
-            prompt = self.messages_to_vertexai_input(s.messages_)
-        else:
-            # single-turn
-            prompt = (
-                self.text_to_vertexai_input(s.text_, s.cur_images)
-                if s.cur_images
-                else s.text_
-            )
-        ret = GenerativeModel(self.model_name).generate_content(
-            prompt,
-            generation_config=GenerationConfig(**sampling_params.to_vertexai_kwargs()),
-            safety_settings=self.safety_settings,
-        )
-
-        comp = ret.text
-
-        return comp, {}
-
-    def generate_stream(
-        self,
-        s: StreamExecutor,
-        sampling_params: SglSamplingParams,
-    ):
-        if s.messages_:
-            prompt = self.messages_to_vertexai_input(s.messages_)
-        else:
-            # single-turn
-            prompt = (
-                self.text_to_vertexai_input(s.text_, s.cur_images)
-                if s.cur_images
-                else s.text_
-            )
-        generator = GenerativeModel(self.model_name).generate_content(
-            prompt,
-            stream=True,
-            generation_config=GenerationConfig(**sampling_params.to_vertexai_kwargs()),
-            safety_settings=self.safety_settings,
-        )
-        for ret in generator:
-            yield ret.text, {}
-
-    def text_to_vertexai_input(self, text, images):
-        input = []
-        # split with image token
-        text_segs = text.split(self.chat_template.image_token)
-        for image_path, image_base64_data in images:
-            text_seg = text_segs.pop(0)
-            if text_seg != "":
-                input.append(text_seg)
-            input.append(Image.from_bytes(image_base64_data))
-        text_seg = text_segs.pop(0)
-        if text_seg != "":
-            input.append(text_seg)
-        return input
-
-    def messages_to_vertexai_input(self, messages):
-        vertexai_message = []
-        # from openai message format to vertexai message format
-        for msg in messages:
-            if isinstance(msg["content"], str):
-                text = msg["content"]
-            else:
-                text = msg["content"][0]["text"]
-
-            if msg["role"] == "system":
-                warnings.warn("Warning: system prompt is not supported in VertexAI.")
-                vertexai_message.append(
-                    {
-                        "role": "user",
-                        "parts": [{"text": "System prompt: " + text}],
-                    }
-                )
-                vertexai_message.append(
-                    {
-                        "role": "model",
-                        "parts": [{"text": "Understood."}],
-                    }
-                )
-                continue
-            if msg["role"] == "user":
-                vertexai_msg = {
-                    "role": "user",
-                    "parts": [{"text": text}],
-                }
-            elif msg["role"] == "assistant":
-                vertexai_msg = {
-                    "role": "model",
-                    "parts": [{"text": text}],
-                }
-
-            # images
-            if isinstance(msg["content"], list) and len(msg["content"]) > 1:
-                for image in msg["content"][1:]:
-                    assert image["type"] == "image_url"
-                    vertexai_msg["parts"].append(
-                        {
-                            "inline_data": {
-                                "data": image["image_url"]["url"].split(",")[1],
-                                "mime_type": "image/jpeg",
-                            }
-                        }
-                    )
-
-            vertexai_message.append(vertexai_msg)
-        return vertexai_message