xinference 0.9.2__py3-none-any.whl → 0.9.4__py3-none-any.whl

xinference/model/llm/sglang/core.py

@@ -0,0 +1,365 @@
+# Copyright 2022-2024 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import time
+import uuid
+from typing import AsyncGenerator, Dict, List, Optional, TypedDict, Union
+
+from ....constants import XINFERENCE_ENABLE_SGLANG
+from ....types import (
+    ChatCompletion,
+    ChatCompletionChunk,
+    ChatCompletionMessage,
+    Completion,
+    CompletionChoice,
+    CompletionChunk,
+    CompletionUsage,
+)
+from .. import LLM, LLMFamilyV1, LLMSpecV1
+from ..llm_family import CustomLLMFamilyV1
+from ..utils import ChatModelMixin
+
+logger = logging.getLogger(__name__)
+
+
+class SGLANGModelConfig(TypedDict, total=False):
+    tokenizer_mode: str
+    trust_remote_code: bool
+    tp_size: int
+    mem_fraction_static: float
+    log_level: str
+    attention_reduce_in_fp32: bool  # For gemma
+
+
+class SGLANGGenerateConfig(TypedDict, total=False):
+    presence_penalty: float
+    frequency_penalty: float
+    temperature: float
+    top_p: float
+    top_k: int
+    max_new_tokens: int
+    stop: Optional[Union[str, List[str]]]
+    ignore_eos: bool
+    stream: bool
+
+
+try:
+    import sglang  # noqa: F401
+
+    SGLANG_INSTALLED = True
+except ImportError:
+    SGLANG_INSTALLED = False
+
+SGLANG_SUPPORTED_MODELS = ["llama-2", "mistral-v0.1", "mixtral-v0.1"]
+SGLANG_SUPPORTED_CHAT_MODELS = [
+    "llama-2-chat",
+    "qwen-chat",
+    "qwen1.5-chat",
+    "mistral-instruct-v0.1",
+    "mistral-instruct-v0.2",
+    "mixtral-instruct-v0.1",
+    "gemma-it",
+]
+
+
+class SGLANGModel(LLM):
+    def __init__(
+        self,
+        model_uid: str,
+        model_family: "LLMFamilyV1",
+        model_spec: "LLMSpecV1",
+        quantization: str,
+        model_path: str,
+        model_config: Optional[SGLANGModelConfig],
+    ):
+        super().__init__(model_uid, model_family, model_spec, quantization, model_path)
+        self._model_config = model_config
+        self._engine = None
+
+    def load(self):
+        try:
+            import sglang as sgl
+        except ImportError:
+            error_message = "Failed to import module 'sglang'"
+            installation_guide = [
+                "Please make sure 'sglang' is installed. ",
+                "You can install it by `pip install 'sglang[all]'`\n",
+            ]
+
+            raise ImportError(f"{error_message}\n\n{''.join(installation_guide)}")
+
+        self._model_config = self._sanitize_model_config(self._model_config)
+        logger.info(
+            f"Loading {self.model_uid} with following model config: {self._model_config}"
+        )
+
+        self._engine = sgl.Runtime(
+            model_path=self.model_path,
+            tokenizer_path=self.model_path,
+            **self._model_config,
+        )
+
+    def _sanitize_model_config(
+        self, model_config: Optional[SGLANGModelConfig]
+    ) -> SGLANGModelConfig:
+        if model_config is None:
+            model_config = SGLANGModelConfig()
+
+        cuda_count = self._get_cuda_count()
+        model_config.setdefault("tokenizer_mode", "auto")
+        model_config.setdefault("trust_remote_code", True)
+        model_config.setdefault("tp_size", cuda_count)
+        # See https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/server_args.py#L37
+        mem_fraction_static = model_config.pop("mem_fraction_static", None)
+        if mem_fraction_static is None:
+            tp_size = model_config.get("tp_size", cuda_count)
+            if tp_size >= 8:
+                model_config["mem_fraction_static"] = 0.80
+            elif tp_size >= 4:
+                model_config["mem_fraction_static"] = 0.82
+            elif tp_size >= 2:
+                model_config["mem_fraction_static"] = 0.85
+            else:
+                model_config["mem_fraction_static"] = 0.90
+        model_config.setdefault("log_level", "info")
+        model_config.setdefault("attention_reduce_in_fp32", False)
+
+        return model_config
+
+    @staticmethod
+    def _sanitize_generate_config(
+        generate_config: Optional[SGLANGGenerateConfig] = None,
+    ) -> SGLANGGenerateConfig:
+        if generate_config is None:
+            generate_config = SGLANGGenerateConfig()
+
+        generate_config.setdefault("presence_penalty", 0.0)
+        generate_config.setdefault("frequency_penalty", 0.0)
+        generate_config.setdefault("temperature", 1.0)
+        generate_config.setdefault("top_p", 1.0)
+        generate_config.setdefault("top_k", -1)
+        # See https://github.com/sgl-project/sglang/blob/main/python/sglang/lang/ir.py#L120
+        # 16 is too less, so here set 256 by default
+        generate_config.setdefault(
+            "max_new_tokens", generate_config.pop("max_tokens", 256)  # type: ignore
+        )
+        generate_config.setdefault("stop", [])
+        generate_config.setdefault("stream", False)
+        generate_config.setdefault("ignore_eos", False)
+
+        return generate_config
+
+    @classmethod
+    def match(
+        cls, llm_family: "LLMFamilyV1", llm_spec: "LLMSpecV1", quantization: str
+    ) -> bool:
+        if not XINFERENCE_ENABLE_SGLANG:
+            return False
+        if not cls._has_cuda_device():
+            return False
+        if not cls._is_linux():
+            return False
+        if llm_spec.model_format not in ["pytorch", "gptq", "awq"]:
+            return False
+        if llm_spec.model_format == "pytorch":
+            if quantization != "none" and not (quantization is None):
+                return False
+        if llm_spec.model_format in ["gptq", "awq"]:
+            # Currently, only 4-bit weight quantization is supported for GPTQ, but got 8 bits.
+            if "4" not in quantization:
+                return False
+        if isinstance(llm_family, CustomLLMFamilyV1):
+            if llm_family.model_family not in SGLANG_SUPPORTED_MODELS:
+                return False
+        else:
+            if llm_family.model_name not in SGLANG_SUPPORTED_MODELS:
+                return False
+        if "generate" not in llm_family.model_ability:
+            return False
+        return SGLANG_INSTALLED
+
+    @staticmethod
+    def _convert_state_to_completion_chunk(
+        request_id: str, model: str, output_text: str, meta_info: Dict
+    ) -> CompletionChunk:
+        choices: List[CompletionChoice] = [
+            CompletionChoice(
+                text=output_text,
+                index=0,
+                logprobs=None,
+                finish_reason=None,
+            )
+        ]
+        chunk = CompletionChunk(
+            id=request_id,
+            object="text_completion",
+            created=int(time.time()),
+            model=model,
+            choices=choices,
+        )
+        prompt_tokens = meta_info["prompt_tokens"]
+        completion_tokens = meta_info["completion_tokens"]
+        chunk["usage"] = CompletionUsage(
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            total_tokens=prompt_tokens + completion_tokens,
+        )
+        return chunk
+
+    @staticmethod
+    def _convert_state_to_completion(
+        request_id: str, model: str, output_text: str, meta_info: Dict
+    ) -> Completion:
+        choices = [
+            CompletionChoice(
+                text=output_text,
+                index=0,
+                logprobs=None,
+                finish_reason=None,
+            )
+        ]
+
+        usage = CompletionUsage(
+            prompt_tokens=meta_info["prompt_tokens"],
+            completion_tokens=meta_info["completion_tokens"],
+            total_tokens=meta_info["prompt_tokens"] + meta_info["completion_tokens"],
+        )
+        return Completion(
+            id=request_id,
+            object="text_completion",
+            created=int(time.time()),
+            model=model,
+            choices=choices,
+            usage=usage,
+        )
+
+    async def async_generate(
+        self,
+        prompt: str,
+        generate_config: Optional[SGLANGGenerateConfig] = None,
+    ) -> Union[Completion, AsyncGenerator[CompletionChunk, None]]:
+        try:
+            import sglang as sgl
+            from sglang import assistant, gen, user
+        except ImportError:
+            error_message = "Failed to import module 'sglang'"
+            installation_guide = [
+                "Please make sure 'sglang' is installed. ",
+                "You can install it by `pip install sglang[all]`\n",
+            ]
+
+            raise ImportError(f"{error_message}\n\n{''.join(installation_guide)}")
+
+        @sgl.function
+        def pipeline(s, question):
+            s += user(question)
+            s += assistant(gen("answer"))
+
+        sanitized_generate_config = self._sanitize_generate_config(generate_config)
+        logger.debug(
+            "Enter generate, prompt: %s, generate config: %s", prompt, generate_config
+        )
+        stream = sanitized_generate_config.pop("stream")
+        request_id = str(uuid.uuid1())
+        state = pipeline.run(
+            question=prompt,
+            backend=self._engine,
+            stream=stream,
+            **sanitized_generate_config,
+        )
+        if not stream:
+            return self._convert_state_to_completion(
+                request_id,
+                model=self.model_uid,
+                output_text=state["answer"],
+                meta_info=state.get_meta_info(name="answer"),
+            )
+        else:
+
+            async def stream_results() -> AsyncGenerator[CompletionChunk, None]:
+                async for out, meta_info in state.text_async_iter(
+                    var_name="answer", return_meta_data=True
+                ):
+                    chunk = self._convert_state_to_completion_chunk(
+                        request_id, self.model_uid, output_text=out, meta_info=meta_info
+                    )
+                    yield chunk
+
+            return stream_results()
+
+
+class SGLANGChatModel(SGLANGModel, ChatModelMixin):
+    @classmethod
+    def match(
+        cls, llm_family: "LLMFamilyV1", llm_spec: "LLMSpecV1", quantization: str
+    ) -> bool:
+        if not XINFERENCE_ENABLE_SGLANG:
+            return False
+        if llm_spec.model_format not in ["pytorch", "gptq", "awq"]:
+            return False
+        if llm_spec.model_format == "pytorch":
+            if quantization != "none" and not (quantization is None):
+                return False
+        if llm_spec.model_format in ["gptq", "awq"]:
+            # Currently, only 4-bit weight quantization is supported for GPTQ, but got 8 bits.
+            if "4" not in quantization:
+                return False
+        if isinstance(llm_family, CustomLLMFamilyV1):
+            if llm_family.model_family not in SGLANG_SUPPORTED_CHAT_MODELS:
+                return False
+        else:
+            if llm_family.model_name not in SGLANG_SUPPORTED_CHAT_MODELS:
+                return False
+        if "chat" not in llm_family.model_ability:
+            return False
+        return SGLANG_INSTALLED
+
+    def _sanitize_chat_config(
+        self,
+        generate_config: Optional[Dict] = None,
+    ) -> Dict:
+        if not generate_config:
+            generate_config = {}
+        if self.model_family.prompt_style:
+            if (
+                not generate_config.get("stop")
+            ) and self.model_family.prompt_style.stop:
+                generate_config["stop"] = self.model_family.prompt_style.stop.copy()
+        return generate_config
+
+    async def async_chat(
+        self,
+        prompt: str,
+        system_prompt: Optional[str] = None,
+        chat_history: Optional[List[ChatCompletionMessage]] = None,
+        generate_config: Optional[Dict] = None,
+    ) -> Union[ChatCompletion, AsyncGenerator[ChatCompletionChunk, None]]:
+        assert self.model_family.prompt_style is not None
+        prompt_style = self.model_family.prompt_style.copy()
+        if system_prompt:
+            prompt_style.system_prompt = system_prompt
+        chat_history = chat_history or []
+        full_prompt = self.get_prompt(prompt, chat_history, prompt_style)
+
+        generate_config = self._sanitize_chat_config(generate_config)
+        stream = generate_config.get("stream", None)
+        if stream:
+            agen = await self.async_generate(full_prompt, generate_config)  # type: ignore
+            assert isinstance(agen, AsyncGenerator)
+            return self._async_to_chat_completion_chunks(agen)
+        else:
+            c = await self.async_generate(full_prompt, generate_config)  # type: ignore
+            assert not isinstance(c, AsyncGenerator)
+            return self._to_chat_completion(c)

xinference/model/llm/utils.py

@@ -411,6 +411,16 @@ Begin!"""
                 if content:
                     ret += content + "<end_of_turn>\n"
             return ret
+        elif prompt_style.style_name == "CodeShell":
+            ret = ""
+            for message in chat_history:
+                content = message["content"]
+                role = get_role(message["role"])
+                if content:
+                    ret += f"{role}{content}|<end>|"
+                else:
+                    ret += f"{role}".rstrip()
+            return ret
         else:
             raise ValueError(f"Invalid prompt style: {prompt_style.style_name}")
 
@@ -451,6 +461,7 @@ Begin!"""
                     "index": i,
                     "delta": {
                         "role": "assistant",
+                        "content": "",
                     },
                     "finish_reason": None,
                 }
@@ -535,26 +546,39 @@ Begin!"""
             # Refer to:
             # https://github.com/QwenLM/Qwen/blob/main/examples/react_prompt.md
             # https://github.com/QwenLM/Qwen/blob/main/openai_api.py#L297
-            func_name, func_args = "", ""
+            func_name, func_args, content = "", "", ""
             i = text.rfind("\nAction:")
             j = text.rfind("\nAction Input:")
             k = text.rfind("\nObservation:")
+            t = max(
+                text.rfind("\nThought:", 0, i), text.rfind("Thought:", 0, i)
+            )  # find the last thought just before Action, considering the Thought at the very beginning
             if 0 <= i < j:  # If the text has `Action` and `Action input`,
                 if k < j:  # but does not contain `Observation`,
                     # then it is likely that `Observation` is omitted by the LLM,
                     # because the output text may have discarded the stop word.
                     text = text.rstrip() + "\nObservation:"  # Add it back.
                     k = text.rfind("\nObservation:")
-            if 0 <= i < j < k:
+            if 0 <= t < i < j < k:
                 func_name = text[i + len("\nAction:") : j].strip()
                 func_args = text[j + len("\nAction Input:") : k].strip()
+                content = text[
+                    t + len("\nThought:") : i
+                ].strip()  # len("\nThought:") and len("Thought:") both are OK since there is a space after :
             if func_name:
-                return None, func_name, json.loads(func_args)
-            z = text.rfind("\nFinal Answer: ")
-            if z >= 0:
-                text = text[z + len("\nFinal Answer: ") :]
+                return content, func_name, json.loads(func_args)
         except Exception as e:
             logger.error("Eval tool calls completion failed: %s", e)
+        t = max(text.rfind("\nThought:"), text.rfind("Thought:"))
+        z = max(text.rfind("\nFinal Answer:"), text.rfind("Final Answer:"))
+        if z >= 0:
+            text = text[
+                z + len("\nFinal Answer:") :
+            ]  # len("\nFinal Answer::") and len("Final Answer::") both are OK since there is a space after :
+        else:
+            text = text[
+                t + len("\nThought:") :
+            ]  # There is only Thought: no Final Answer:
         return text, None, None
 
     @classmethod
@@ -573,13 +597,10 @@ Begin!"""
             )
         logger.debug("Tool call content: %s, func: %s, args: %s", content, func, args)
 
-        if content:
-            m = {"role": "assistant", "content": content, "tool_calls": []}
-            finish_reason = "stop"
-        else:
+        if func:
             m = {
                 "role": "assistant",
-                "content": None,
+                "content": content,
                 "tool_calls": [
                     {
                         "id": f"call_{_id}",
@@ -592,7 +613,9 @@ Begin!"""
                 ],
             }
             finish_reason = "tool_calls"
-
+        else:
+            m = {"role": "assistant", "content": content, "tool_calls": []}
+            finish_reason = "stop"
         return {
             "id": "chat" + f"cmpl-{_id}",
             "model": model_uid,

xinference/model/llm/vllm/core.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import logging
+import multiprocessing
 import time
 import uuid
 from typing import (
@@ -102,6 +103,13 @@ VLLM_SUPPORTED_CHAT_MODELS = [
 if VLLM_INSTALLED and vllm.__version__ >= "0.3.0":
     VLLM_SUPPORTED_CHAT_MODELS.append("qwen1.5-chat")
 
+if VLLM_INSTALLED and vllm.__version__ >= "0.3.2":
+    VLLM_SUPPORTED_CHAT_MODELS.append("gemma-it")
+
+if VLLM_INSTALLED and vllm.__version__ >= "0.3.3":
+    VLLM_SUPPORTED_CHAT_MODELS.append("orion-chat")
+    VLLM_SUPPORTED_CHAT_MODELS.append("orion-chat-rag")
+
 
 class VLLMModel(LLM):
     def __init__(
@@ -119,6 +127,7 @@ class VLLMModel(LLM):
 
     def load(self):
         try:
+            import vllm
             from vllm.engine.arg_utils import AsyncEngineArgs
             from vllm.engine.async_llm_engine import AsyncLLMEngine
         except ImportError:
@@ -130,6 +139,14 @@ class VLLMModel(LLM):
 
             raise ImportError(f"{error_message}\n\n{''.join(installation_guide)}")
 
+        if vllm.__version__ >= "0.3.1":
+            # from vllm v0.3.1, it uses cupy as NCCL backend
+            # in which cupy will fork a process
+            # only for xoscar >= 0.3.0, new process is allowed in subpool
+            # besides, xinference set start method as forkserver for unix
+            # we need to set it to fork to make cupy NCCL work
+            multiprocessing.set_start_method("fork", force=True)
+
         self._model_config = self._sanitize_model_config(self._model_config)
         logger.info(
             f"Loading {self.model_uid} with following model config: {self._model_config}"

xinference/web/ui/build/asset-manifest.json

@@ -1,11 +1,11 @@
 {
   "files": {
-    "main.js": "./static/js/main.78829790.js",
+    "main.js": "./static/js/main.66b1c4fb.js",
     "static/media/icon.webp": "./static/media/icon.4603d52c63041e5dfbfd.webp",
     "index.html": "./index.html",
-    "main.78829790.js.map": "./static/js/main.78829790.js.map"
+    "main.66b1c4fb.js.map": "./static/js/main.66b1c4fb.js.map"
   },
   "entrypoints": [
-    "static/js/main.78829790.js"
+    "static/js/main.66b1c4fb.js"
   ]
 }

xinference/web/ui/build/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="Web site created using create-react-app"/><link rel="apple-touch-icon" href="./logo192.png"/><link rel="manifest" href="./manifest.json"/><title>Xinference</title><script defer="defer" src="./static/js/main.78829790.js"></script></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="Web site created using create-react-app"/><link rel="apple-touch-icon" href="./logo192.png"/><link rel="manifest" href="./manifest.json"/><title>Xinference</title><script defer="defer" src="./static/js/main.66b1c4fb.js"></script></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>