xinference 0.9.3__py3-none-any.whl → 0.9.4__py3-none-any.whl

xinference/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2024-03-15T11:03:04+0800",
+ "date": "2024-03-21T14:58:01+0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "60f098c78adb7e28dd564278a7741b56eaf062d8",
- "version": "0.9.3"
+ "full-revisionid": "2c9465ade7f358d57d4bc087277882d896a8de15",
+ "version": "0.9.4"
 }
 '''  # END VERSION_JSON
 

xinference/api/restful_api.py

@@ -59,6 +59,7 @@ from ..core.utils import json_dumps
 from ..types import (
     SPECIAL_TOOL_PROMPT,
     ChatCompletion,
+    ChatCompletionMessage,
     Completion,
     CreateChatCompletion,
     CreateCompletion,
@@ -1258,25 +1259,21 @@ class RESTfulAPI:
                 status_code=400, detail="Invalid input. Please specify the prompt."
             )
 
-        system_messages = []
+        system_messages: List["ChatCompletionMessage"] = []
+        system_messages_contents = []
         non_system_messages = []
         for msg in messages:
             assert (
                 msg.get("content") != SPECIAL_TOOL_PROMPT
             ), f"Invalid message content {SPECIAL_TOOL_PROMPT}"
             if msg["role"] == "system":
-                system_messages.append(msg)
+                system_messages_contents.append(msg["content"])
             else:
                 non_system_messages.append(msg)
+        system_messages.append(
+            {"role": "system", "content": ". ".join(system_messages_contents)}
+        )
 
-        if len(system_messages) > 1:
-            raise HTTPException(
-                status_code=400, detail="Multiple system messages are not supported."
-            )
-        if len(system_messages) == 1 and messages[0]["role"] != "system":
-            raise HTTPException(
-                status_code=400, detail="System message should be the first one."
-            )
         assert non_system_messages
 
         has_tool_message = messages[-1].get("role") == "tool"

xinference/constants.py

@@ -25,6 +25,7 @@ XINFERENCE_ENV_HEALTH_CHECK_INTERVAL = "XINFERENCE_HEALTH_CHECK_INTERVAL"
 XINFERENCE_ENV_HEALTH_CHECK_TIMEOUT = "XINFERENCE_HEALTH_CHECK_TIMEOUT"
 XINFERENCE_ENV_DISABLE_HEALTH_CHECK = "XINFERENCE_DISABLE_HEALTH_CHECK"
 XINFERENCE_ENV_DISABLE_VLLM = "XINFERENCE_DISABLE_VLLM"
+XINFERENCE_ENV_ENABLE_SGLANG = "XINFERENCE_ENABLE_SGLANG"
 
 
 def get_xinference_home() -> str:
@@ -64,3 +65,4 @@ XINFERENCE_DISABLE_HEALTH_CHECK = bool(
     int(os.environ.get(XINFERENCE_ENV_DISABLE_HEALTH_CHECK, 0))
 )
 XINFERENCE_DISABLE_VLLM = bool(int(os.environ.get(XINFERENCE_ENV_DISABLE_VLLM, 0)))
+XINFERENCE_ENABLE_SGLANG = bool(int(os.environ.get(XINFERENCE_ENV_ENABLE_SGLANG, 0)))

xinference/model/llm/__init__.py

@@ -60,6 +60,7 @@ def _install():
     from .pytorch.qwen_vl import QwenVLChatModel
     from .pytorch.vicuna import VicunaPytorchChatModel
     from .pytorch.yi_vl import YiVLChatModel
+    from .sglang.core import SGLANGChatModel, SGLANGModel
     from .vllm.core import VLLMChatModel, VLLMModel
 
     # register llm classes.
@@ -79,6 +80,7 @@ def _install():
             CtransformersModel,
         ]
     )
+    LLM_CLASSES.extend([SGLANGModel, SGLANGChatModel])
     LLM_CLASSES.extend([VLLMModel, VLLMChatModel])
     LLM_CLASSES.extend(
         [

xinference/model/llm/llm_family.json

@@ -98,6 +98,72 @@
       ]
     }
   },
+  {
+    "version": 1,
+    "context_length": 8194,
+    "model_name": "codeshell",
+    "model_lang": [
+      "en",
+      "zh"
+    ],
+    "model_ability": [
+      "generate"
+    ],
+    "model_description": "CodeShell is a multi-language code LLM developed by the Knowledge Computing Lab of Peking University. ",
+    "model_specs": [
+      {
+        "model_format": "pytorch",
+        "model_size_in_billions": 7,
+        "quantizations": [
+          "none"
+        ],
+        "model_id": "WisdomShell/CodeShell-7B",
+        "model_revision": "1c79ab7fd316a62ab41d764facd3548a23fa5dee"
+      }
+    ]
+  },
+  {
+    "version": 1,
+    "context_length": 8194,
+    "model_name": "codeshell-chat",
+    "model_lang": [
+      "en",
+      "zh"
+    ],
+    "model_ability": [
+      "chat"
+    ],
+    "model_description": "CodeShell is a multi-language code LLM developed by the Knowledge Computing Lab of Peking University.",
+    "model_specs": [
+      {
+        "model_format": "pytorch",
+        "model_size_in_billions": 7,
+        "quantizations": [
+          "none"
+        ],
+        "model_id": "WisdomShell/CodeShell-7B-Chat",
+        "model_revision": "3cb06f589b7b1e2f8e728c77280b1114191d24de"
+      }
+    ],
+    "prompt_style": {
+      "style_name": "CodeShell",
+      "system_prompt": "",
+      "roles": [
+        "## human:",
+        "## assistant: "
+      ],
+      "intra_message_sep": "",
+      "inter_message_sep": "",
+      "stop_token_ids": [
+        70000
+      ],
+      "stop": [
+        "<|endoftext|>",
+        "|||",
+        "|<end>|"
+      ]
+    }
+  },
   {
     "version": 1,
     "context_length": 2048,
@@ -573,7 +639,7 @@
         64797,
         2
       ],
-      "stop":[
+      "stop": [
         "<|user|>",
         "<|observation|>"
       ]
@@ -616,7 +682,7 @@
         64797,
         2
       ],
-      "stop":[
+      "stop": [
         "<|user|>",
         "<|observation|>"
       ]
@@ -667,7 +733,6 @@
       ]
     }
   },
-
   {
     "version": 1,
     "context_length": 2048,
@@ -715,8 +780,7 @@
         "model_revision": "7f1b7394f74c630f50612a19ba90bd021c373989"
       }
     ]
-  }
- ,
+  },
   {
     "version": 1,
     "context_length": 4096,
@@ -1606,7 +1670,10 @@
         "model_file_name_template": "qwen1_5-72b-chat-{quantization}.gguf",
         "model_file_name_split_template": "qwen1_5-72b-chat-{quantization}.gguf.{part}",
         "quantization_parts": {
-          "q4_k_m": ["a", "b"]
+          "q4_k_m": [
+            "a",
+            "b"
+          ]
         }
       }
     ],
@@ -2658,7 +2725,11 @@
     "context_length": 32768,
     "model_name": "mixtral-v0.1",
     "model_lang": [
-      "en", "fr", "it", "de", "es"
+      "en",
+      "fr",
+      "it",
+      "de",
+      "es"
     ],
     "model_ability": [
       "generate"
@@ -2699,7 +2770,11 @@
     "context_length": 32768,
     "model_name": "mixtral-instruct-v0.1",
     "model_lang": [
-      "en", "fr", "it", "de", "es"
+      "en",
+      "fr",
+      "it",
+      "de",
+      "es"
     ],
     "model_ability": [
       "chat"
@@ -3275,10 +3350,8 @@
       ],
       "intra_message_sep": "\n",
       "inter_message_sep": "\n",
-      "stop_token_ids": [
-      ],
-      "stop": [
-      ]
+      "stop_token_ids": [],
+      "stop": []
     }
   },
   {
@@ -3376,7 +3449,8 @@
     "context_length": 4096,
     "model_name": "deepseek-coder-instruct",
     "model_lang": [
-      "en", "zh"
+      "en",
+      "zh"
     ],
     "model_ability": [
       "chat"

xinference/model/llm/llm_family_modelscope.json

@@ -338,7 +338,7 @@
         64797,
         2
       ],
-      "stop":[
+      "stop": [
         "<|user|>",
         "<|observation|>"
       ]
@@ -382,13 +382,12 @@
         64797,
         2
       ],
-      "stop":[
+      "stop": [
         "<|user|>",
         "<|observation|>"
       ]
     }
   },
-
   {
     "version": 1,
     "context_length": 2048,
@@ -728,6 +727,74 @@
       }
     ]
   },
+  {
+    "version": 1,
+    "context_length": 8194,
+    "model_name": "codeshell",
+    "model_lang": [
+      "en",
+      "zh"
+    ],
+    "model_ability": [
+      "generate"
+    ],
+    "model_description": "CodeShell is a multi-language code LLM developed by the Knowledge Computing Lab of Peking University. ",
+    "model_specs": [
+      {
+        "model_format": "pytorch",
+        "model_size_in_billions": 7,
+        "quantizations": [
+          "none"
+        ],
+        "model_id": "WisdomShell/CodeShell-7B",
+        "model_revision": "master",
+        "model_hub": "modelscope"
+      }
+    ]
+  },
+  {
+    "version": 1,
+    "context_length": 8194,
+    "model_name": "codeshell-chat",
+    "model_lang": [
+      "en",
+      "zh"
+    ],
+    "model_ability": [
+      "chat"
+    ],
+    "model_description": "CodeShell is a multi-language code LLM developed by the Knowledge Computing Lab of Peking University.",
+    "model_specs": [
+      {
+        "model_format": "pytorch",
+        "model_size_in_billions": 7,
+        "quantizations": [
+          "none"
+        ],
+        "model_id": "WisdomShell/CodeShell-7B-Chat",
+        "model_revision": "master",
+        "model_hub": "modelscope"
+      }
+    ],
+    "prompt_style": {
+      "style_name": "CodeShell",
+      "system_prompt": "",
+      "roles": [
+        "## human:",
+        "## assistant: "
+      ],
+      "intra_message_sep": "",
+      "inter_message_sep": "",
+      "stop_token_ids": [
+        70000
+      ],
+      "stop": [
+        "<|endoftext|>",
+        "|||",
+        "|<end>|"
+      ]
+    }
+  },
   {
     "version": 1,
     "context_length": 100000,
@@ -970,7 +1037,11 @@
     "context_length": 32768,
     "model_name": "mixtral-v0.1",
     "model_lang": [
-      "en", "fr", "it", "de", "es"
+      "en",
+      "fr",
+      "it",
+      "de",
+      "es"
     ],
     "model_ability": [
       "generate"
@@ -996,7 +1067,11 @@
     "context_length": 32768,
     "model_name": "mixtral-instruct-v0.1",
     "model_lang": [
-      "en", "fr", "it", "de", "es"
+      "en",
+      "fr",
+      "it",
+      "de",
+      "es"
     ],
     "model_ability": [
       "chat"
@@ -1929,7 +2004,10 @@
         "model_file_name_template": "qwen1_5-72b-chat-{quantization}.gguf",
         "model_file_name_split_template": "qwen1_5-72b-chat-{quantization}.gguf.{part}",
         "quantization_parts": {
-          "q4_k_m": ["a", "b"]
+          "q4_k_m": [
+            "a",
+            "b"
+          ]
         }
       }
     ],
@@ -2008,7 +2086,8 @@
     "context_length": 4096,
     "model_name": "deepseek-coder-instruct",
     "model_lang": [
-      "en", "zh"
+      "en",
+      "zh"
     ],
     "model_ability": [
       "chat"

xinference/model/llm/pytorch/chatglm.py

@@ -148,6 +148,7 @@ class ChatglmPytorchChatModel(PytorchChatModel):
 
                 def _stream_generator():
                     last_chunk_text_length = 0
+                    chunk_id = "chat-" + str(uuid.uuid1())
                     for chunk_text, _ in self._model.stream_chat(
                         self._tokenizer, prompt, chat_history, **kwargs
                     ):
@@ -157,7 +158,7 @@ class ChatglmPytorchChatModel(PytorchChatModel):
                             text=chunk_text, index=0, logprobs=None, finish_reason=None
                         )
                         yield CompletionChunk(
-                            id=str(uuid.uuid1()),
+                            id=chunk_id,
                             object="text_completion",
                             created=int(time.time()),
                             model=self.model_uid,

xinference/model/llm/pytorch/internlm2.py

@@ -118,6 +118,7 @@ class Internlm2PytorchChatModel(PytorchChatModel):
 
             def _stream_generator():
                 last_chunk_text_length = 0
+                chunk_id = "chat-" + str(uuid.uuid1())
                 for chunk_text, _ in self._model.stream_chat(
                     self._tokenizer, prompt, input_history, **kwargs
                 ):
@@ -127,7 +128,7 @@ class Internlm2PytorchChatModel(PytorchChatModel):
                         text=chunk_text, index=0, logprobs=None, finish_reason=None
                     )
                     yield CompletionChunk(
-                        id=str(uuid.uuid1()),
+                        id=chunk_id,
                         object="text_completion",
                         created=int(time.time()),
                         model=self.model_uid,

xinference/model/llm/sglang/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2022-2024 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

xinference/model/llm/sglang/core.py

@@ -0,0 +1,365 @@
+# Copyright 2022-2024 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import time
+import uuid
+from typing import AsyncGenerator, Dict, List, Optional, TypedDict, Union
+
+from ....constants import XINFERENCE_ENABLE_SGLANG
+from ....types import (
+    ChatCompletion,
+    ChatCompletionChunk,
+    ChatCompletionMessage,
+    Completion,
+    CompletionChoice,
+    CompletionChunk,
+    CompletionUsage,
+)
+from .. import LLM, LLMFamilyV1, LLMSpecV1
+from ..llm_family import CustomLLMFamilyV1
+from ..utils import ChatModelMixin
+
+logger = logging.getLogger(__name__)
+
+
+class SGLANGModelConfig(TypedDict, total=False):
+    tokenizer_mode: str
+    trust_remote_code: bool
+    tp_size: int
+    mem_fraction_static: float
+    log_level: str
+    attention_reduce_in_fp32: bool  # For gemma
+
+
+class SGLANGGenerateConfig(TypedDict, total=False):
+    presence_penalty: float
+    frequency_penalty: float
+    temperature: float
+    top_p: float
+    top_k: int
+    max_new_tokens: int
+    stop: Optional[Union[str, List[str]]]
+    ignore_eos: bool
+    stream: bool
+
+
+try:
+    import sglang  # noqa: F401
+
+    SGLANG_INSTALLED = True
+except ImportError:
+    SGLANG_INSTALLED = False
+
+SGLANG_SUPPORTED_MODELS = ["llama-2", "mistral-v0.1", "mixtral-v0.1"]
+SGLANG_SUPPORTED_CHAT_MODELS = [
+    "llama-2-chat",
+    "qwen-chat",
+    "qwen1.5-chat",
+    "mistral-instruct-v0.1",
+    "mistral-instruct-v0.2",
+    "mixtral-instruct-v0.1",
+    "gemma-it",
+]
+
+
+class SGLANGModel(LLM):
+    def __init__(
+        self,
+        model_uid: str,
+        model_family: "LLMFamilyV1",
+        model_spec: "LLMSpecV1",
+        quantization: str,
+        model_path: str,
+        model_config: Optional[SGLANGModelConfig],
+    ):
+        super().__init__(model_uid, model_family, model_spec, quantization, model_path)
+        self._model_config = model_config
+        self._engine = None
+
+    def load(self):
+        try:
+            import sglang as sgl
+        except ImportError:
+            error_message = "Failed to import module 'sglang'"
+            installation_guide = [
+                "Please make sure 'sglang' is installed. ",
+                "You can install it by `pip install 'sglang[all]'`\n",
+            ]
+
+            raise ImportError(f"{error_message}\n\n{''.join(installation_guide)}")
+
+        self._model_config = self._sanitize_model_config(self._model_config)
+        logger.info(
+            f"Loading {self.model_uid} with following model config: {self._model_config}"
+        )
+
+        self._engine = sgl.Runtime(
+            model_path=self.model_path,
+            tokenizer_path=self.model_path,
+            **self._model_config,
+        )
+
+    def _sanitize_model_config(
+        self, model_config: Optional[SGLANGModelConfig]
+    ) -> SGLANGModelConfig:
+        if model_config is None:
+            model_config = SGLANGModelConfig()
+
+        cuda_count = self._get_cuda_count()
+        model_config.setdefault("tokenizer_mode", "auto")
+        model_config.setdefault("trust_remote_code", True)
+        model_config.setdefault("tp_size", cuda_count)
+        # See https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/server_args.py#L37
+        mem_fraction_static = model_config.pop("mem_fraction_static", None)
+        if mem_fraction_static is None:
+            tp_size = model_config.get("tp_size", cuda_count)
+            if tp_size >= 8:
+                model_config["mem_fraction_static"] = 0.80
+            elif tp_size >= 4:
+                model_config["mem_fraction_static"] = 0.82
+            elif tp_size >= 2:
+                model_config["mem_fraction_static"] = 0.85
+            else:
+                model_config["mem_fraction_static"] = 0.90
+        model_config.setdefault("log_level", "info")
+        model_config.setdefault("attention_reduce_in_fp32", False)
+
+        return model_config
+
+    @staticmethod
+    def _sanitize_generate_config(
+        generate_config: Optional[SGLANGGenerateConfig] = None,
+    ) -> SGLANGGenerateConfig:
+        if generate_config is None:
+            generate_config = SGLANGGenerateConfig()
+
+        generate_config.setdefault("presence_penalty", 0.0)
+        generate_config.setdefault("frequency_penalty", 0.0)
+        generate_config.setdefault("temperature", 1.0)
+        generate_config.setdefault("top_p", 1.0)
+        generate_config.setdefault("top_k", -1)
+        # See https://github.com/sgl-project/sglang/blob/main/python/sglang/lang/ir.py#L120
+        # 16 is too less, so here set 256 by default
+        generate_config.setdefault(
+            "max_new_tokens", generate_config.pop("max_tokens", 256)  # type: ignore
+        )
+        generate_config.setdefault("stop", [])
+        generate_config.setdefault("stream", False)
+        generate_config.setdefault("ignore_eos", False)
+
+        return generate_config
+
+    @classmethod
+    def match(
+        cls, llm_family: "LLMFamilyV1", llm_spec: "LLMSpecV1", quantization: str
+    ) -> bool:
+        if not XINFERENCE_ENABLE_SGLANG:
+            return False
+        if not cls._has_cuda_device():
+            return False
+        if not cls._is_linux():
+            return False
+        if llm_spec.model_format not in ["pytorch", "gptq", "awq"]:
+            return False
+        if llm_spec.model_format == "pytorch":
+            if quantization != "none" and not (quantization is None):
+                return False
+        if llm_spec.model_format in ["gptq", "awq"]:
+            # Currently, only 4-bit weight quantization is supported for GPTQ, but got 8 bits.
+            if "4" not in quantization:
+                return False
+        if isinstance(llm_family, CustomLLMFamilyV1):
+            if llm_family.model_family not in SGLANG_SUPPORTED_MODELS:
+                return False
+        else:
+            if llm_family.model_name not in SGLANG_SUPPORTED_MODELS:
+                return False
+        if "generate" not in llm_family.model_ability:
+            return False
+        return SGLANG_INSTALLED
+
+    @staticmethod
+    def _convert_state_to_completion_chunk(
+        request_id: str, model: str, output_text: str, meta_info: Dict
+    ) -> CompletionChunk:
+        choices: List[CompletionChoice] = [
+            CompletionChoice(
+                text=output_text,
+                index=0,
+                logprobs=None,
+                finish_reason=None,
+            )
+        ]
+        chunk = CompletionChunk(
+            id=request_id,
+            object="text_completion",
+            created=int(time.time()),
+            model=model,
+            choices=choices,
+        )
+        prompt_tokens = meta_info["prompt_tokens"]
+        completion_tokens = meta_info["completion_tokens"]
+        chunk["usage"] = CompletionUsage(
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            total_tokens=prompt_tokens + completion_tokens,
+        )
+        return chunk
+
+    @staticmethod
+    def _convert_state_to_completion(
+        request_id: str, model: str, output_text: str, meta_info: Dict
+    ) -> Completion:
+        choices = [
+            CompletionChoice(
+                text=output_text,
+                index=0,
+                logprobs=None,
+                finish_reason=None,
+            )
+        ]
+
+        usage = CompletionUsage(
+            prompt_tokens=meta_info["prompt_tokens"],
+            completion_tokens=meta_info["completion_tokens"],
+            total_tokens=meta_info["prompt_tokens"] + meta_info["completion_tokens"],
+        )
+        return Completion(
+            id=request_id,
+            object="text_completion",
+            created=int(time.time()),
+            model=model,
+            choices=choices,
+            usage=usage,
+        )
+
+    async def async_generate(
+        self,
+        prompt: str,
+        generate_config: Optional[SGLANGGenerateConfig] = None,
+    ) -> Union[Completion, AsyncGenerator[CompletionChunk, None]]:
+        try:
+            import sglang as sgl
+            from sglang import assistant, gen, user
+        except ImportError:
+            error_message = "Failed to import module 'sglang'"
+            installation_guide = [
+                "Please make sure 'sglang' is installed. ",
+                "You can install it by `pip install sglang[all]`\n",
+            ]
+
+            raise ImportError(f"{error_message}\n\n{''.join(installation_guide)}")
+
+        @sgl.function
+        def pipeline(s, question):
+            s += user(question)
+            s += assistant(gen("answer"))
+
+        sanitized_generate_config = self._sanitize_generate_config(generate_config)
+        logger.debug(
+            "Enter generate, prompt: %s, generate config: %s", prompt, generate_config
+        )
+        stream = sanitized_generate_config.pop("stream")
+        request_id = str(uuid.uuid1())
+        state = pipeline.run(
+            question=prompt,
+            backend=self._engine,
+            stream=stream,
+            **sanitized_generate_config,
+        )
+        if not stream:
+            return self._convert_state_to_completion(
+                request_id,
+                model=self.model_uid,
+                output_text=state["answer"],
+                meta_info=state.get_meta_info(name="answer"),
+            )
+        else:
+
+            async def stream_results() -> AsyncGenerator[CompletionChunk, None]:
+                async for out, meta_info in state.text_async_iter(
+                    var_name="answer", return_meta_data=True
+                ):
+                    chunk = self._convert_state_to_completion_chunk(
+                        request_id, self.model_uid, output_text=out, meta_info=meta_info
+                    )
+                    yield chunk
+
+            return stream_results()
+
+
+class SGLANGChatModel(SGLANGModel, ChatModelMixin):
+    @classmethod
+    def match(
+        cls, llm_family: "LLMFamilyV1", llm_spec: "LLMSpecV1", quantization: str
+    ) -> bool:
+        if not XINFERENCE_ENABLE_SGLANG:
+            return False
+        if llm_spec.model_format not in ["pytorch", "gptq", "awq"]:
+            return False
+        if llm_spec.model_format == "pytorch":
+            if quantization != "none" and not (quantization is None):
+                return False
+        if llm_spec.model_format in ["gptq", "awq"]:
+            # Currently, only 4-bit weight quantization is supported for GPTQ, but got 8 bits.
+            if "4" not in quantization:
+                return False
+        if isinstance(llm_family, CustomLLMFamilyV1):
+            if llm_family.model_family not in SGLANG_SUPPORTED_CHAT_MODELS:
+                return False
+        else:
+            if llm_family.model_name not in SGLANG_SUPPORTED_CHAT_MODELS:
+                return False
+        if "chat" not in llm_family.model_ability:
+            return False
+        return SGLANG_INSTALLED
+
+    def _sanitize_chat_config(
+        self,
+        generate_config: Optional[Dict] = None,
+    ) -> Dict:
+        if not generate_config:
+            generate_config = {}
+        if self.model_family.prompt_style:
+            if (
+                not generate_config.get("stop")
+            ) and self.model_family.prompt_style.stop:
+                generate_config["stop"] = self.model_family.prompt_style.stop.copy()
+        return generate_config
+
+    async def async_chat(
+        self,
+        prompt: str,
+        system_prompt: Optional[str] = None,
+        chat_history: Optional[List[ChatCompletionMessage]] = None,
+        generate_config: Optional[Dict] = None,
+    ) -> Union[ChatCompletion, AsyncGenerator[ChatCompletionChunk, None]]:
+        assert self.model_family.prompt_style is not None
+        prompt_style = self.model_family.prompt_style.copy()
+        if system_prompt:
+            prompt_style.system_prompt = system_prompt
+        chat_history = chat_history or []
+        full_prompt = self.get_prompt(prompt, chat_history, prompt_style)
+
+        generate_config = self._sanitize_chat_config(generate_config)
+        stream = generate_config.get("stream", None)
+        if stream:
+            agen = await self.async_generate(full_prompt, generate_config)  # type: ignore
+            assert isinstance(agen, AsyncGenerator)
+            return self._async_to_chat_completion_chunks(agen)
+        else:
+            c = await self.async_generate(full_prompt, generate_config)  # type: ignore
+            assert not isinstance(c, AsyncGenerator)
+            return self._to_chat_completion(c)

xinference/model/llm/utils.py

@@ -411,6 +411,16 @@ Begin!"""
                 if content:
                     ret += content + "<end_of_turn>\n"
             return ret
+        elif prompt_style.style_name == "CodeShell":
+            ret = ""
+            for message in chat_history:
+                content = message["content"]
+                role = get_role(message["role"])
+                if content:
+                    ret += f"{role}{content}|<end>|"
+                else:
+                    ret += f"{role}".rstrip()
+            return ret
         else:
             raise ValueError(f"Invalid prompt style: {prompt_style.style_name}")
 
@@ -451,6 +461,7 @@ Begin!"""
                     "index": i,
                     "delta": {
                         "role": "assistant",
+                        "content": "",
                     },
                     "finish_reason": None,
                 }
@@ -535,26 +546,39 @@ Begin!"""
             # Refer to:
             # https://github.com/QwenLM/Qwen/blob/main/examples/react_prompt.md
             # https://github.com/QwenLM/Qwen/blob/main/openai_api.py#L297
-            func_name, func_args = "", ""
+            func_name, func_args, content = "", "", ""
             i = text.rfind("\nAction:")
             j = text.rfind("\nAction Input:")
             k = text.rfind("\nObservation:")
+            t = max(
+                text.rfind("\nThought:", 0, i), text.rfind("Thought:", 0, i)
+            )  # find the last thought just before Action, considering the Thought at the very beginning
             if 0 <= i < j:  # If the text has `Action` and `Action input`,
                 if k < j:  # but does not contain `Observation`,
                     # then it is likely that `Observation` is omitted by the LLM,
                     # because the output text may have discarded the stop word.
                     text = text.rstrip() + "\nObservation:"  # Add it back.
                     k = text.rfind("\nObservation:")
-            if 0 <= i < j < k:
+            if 0 <= t < i < j < k:
                 func_name = text[i + len("\nAction:") : j].strip()
                 func_args = text[j + len("\nAction Input:") : k].strip()
+                content = text[
+                    t + len("\nThought:") : i
+                ].strip()  # len("\nThought:") and len("Thought:") both are OK since there is a space after :
             if func_name:
-                return None, func_name, json.loads(func_args)
-            z = text.rfind("\nFinal Answer: ")
-            if z >= 0:
-                text = text[z + len("\nFinal Answer: ") :]
+                return content, func_name, json.loads(func_args)
         except Exception as e:
             logger.error("Eval tool calls completion failed: %s", e)
+        t = max(text.rfind("\nThought:"), text.rfind("Thought:"))
+        z = max(text.rfind("\nFinal Answer:"), text.rfind("Final Answer:"))
+        if z >= 0:
+            text = text[
+                z + len("\nFinal Answer:") :
+            ]  # len("\nFinal Answer::") and len("Final Answer::") both are OK since there is a space after :
+        else:
+            text = text[
+                t + len("\nThought:") :
+            ]  # There is only Thought: no Final Answer:
         return text, None, None
 
     @classmethod
@@ -573,13 +597,10 @@ Begin!"""
             )
         logger.debug("Tool call content: %s, func: %s, args: %s", content, func, args)
 
-        if content:
-            m = {"role": "assistant", "content": content, "tool_calls": []}
-            finish_reason = "stop"
-        else:
+        if func:
             m = {
                 "role": "assistant",
-                "content": None,
+                "content": content,
                 "tool_calls": [
                     {
                         "id": f"call_{_id}",
@@ -592,7 +613,9 @@ Begin!"""
                 ],
             }
             finish_reason = "tool_calls"
-
+        else:
+            m = {"role": "assistant", "content": content, "tool_calls": []}
+            finish_reason = "stop"
         return {
             "id": "chat" + f"cmpl-{_id}",
             "model": model_uid,

xinference/model/llm/vllm/core.py

@@ -103,6 +103,13 @@ VLLM_SUPPORTED_CHAT_MODELS = [
 if VLLM_INSTALLED and vllm.__version__ >= "0.3.0":
     VLLM_SUPPORTED_CHAT_MODELS.append("qwen1.5-chat")
 
+if VLLM_INSTALLED and vllm.__version__ >= "0.3.2":
+    VLLM_SUPPORTED_CHAT_MODELS.append("gemma-it")
+
+if VLLM_INSTALLED and vllm.__version__ >= "0.3.3":
+    VLLM_SUPPORTED_CHAT_MODELS.append("orion-chat")
+    VLLM_SUPPORTED_CHAT_MODELS.append("orion-chat-rag")
+
 
 class VLLMModel(LLM):
     def __init__(

{xinference-0.9.3.dist-info → xinference-0.9.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xinference
-Version: 0.9.3
+Version: 0.9.4
 Summary: Model Serving Made Easy
 Home-page: https://github.com/xorbitsai/inference
 Author: Qin Xuye
@@ -62,6 +62,7 @@ Requires-Dist: orjson ; extra == 'all'
 Requires-Dist: optimum ; extra == 'all'
 Requires-Dist: auto-gptq ; (sys_platform != "darwin") and extra == 'all'
 Requires-Dist: vllm >=0.2.6 ; (sys_platform == "linux") and extra == 'all'
+Requires-Dist: sglang[all] ; (sys_platform == "linux") and extra == 'all'
 Provides-Extra: benchmark
 Requires-Dist: psutil ; extra == 'benchmark'
 Provides-Extra: dev
@@ -105,6 +106,8 @@ Requires-Dist: controlnet-aux ; extra == 'image'
 Provides-Extra: intel
 Requires-Dist: torch ==2.1.0a0 ; extra == 'intel'
 Requires-Dist: intel-extension-for-pytorch ==2.1.10+xpu ; extra == 'intel'
+Provides-Extra: sglang
+Requires-Dist: sglang[all] ; extra == 'sglang'
 Provides-Extra: transformers
 Requires-Dist: transformers >=4.34.1 ; extra == 'transformers'
 Requires-Dist: torch ; extra == 'transformers'

{xinference-0.9.3.dist-info → xinference-0.9.4.dist-info}/RECORD

@@ -1,15 +1,15 @@
 xinference/__init__.py,sha256=0LgIveLP6CXxoIaSrxhlFyOh0lOqPgJBVcBe0tkWJjc,987
 xinference/_compat.py,sha256=SQAjZMGxtBIce45qtW7ob7RWzA0zhv2yB3AxT0rb0uU,1778
-xinference/_version.py,sha256=Lg7riCQT494Ylj5YFCWP1j41UqI3CrUvWJukfkpY7ck,497
+xinference/_version.py,sha256=NA8Gd_eyeZ4pGEW181mxcRG1Ae6xSp0HasvpDwAg4iQ,497
 xinference/conftest.py,sha256=soSjfLdFYvMleUaSL4cZKPtP1WdlAb7q3Sm2pX4aTog,9523
-xinference/constants.py,sha256=81M8ATbulsB78fht4pp0H8q2dDU2qTqSmu1FAjRiOKk,2738
+xinference/constants.py,sha256=Bu_fOJUGAvvqF_6FY5OzOHl7fQ1Nomek3LY17xr9oz4,2882
 xinference/device_utils.py,sha256=WNKDD4Eni3Io3AehiyonsuoJaukT77Bc76Es7vNGvjc,2615
 xinference/fields.py,sha256=BFAVowcvvwA4i1HHHeyD1J_p3p6Vi5hVmKY0CT21blM,5075
 xinference/isolation.py,sha256=NstVRcO3dG4umHExICXAHlzVKwH8ch8MBwKwE-KFkE0,1826
 xinference/types.py,sha256=_tbGgdMTlXs6oINWbEmCesyckDDa2tsPRH9yzM8fKSI,11812
 xinference/utils.py,sha256=Z6PPDGmX4EW8OD3OfA2Wa37ZM9OdRTnR00ITMDTu4qE,716
 xinference/api/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
-xinference/api/restful_api.py,sha256=pMpT7H46viyG1i1dez6Xqmsz2G-xzWDXmkZXK-d3sxY,58898
+xinference/api/restful_api.py,sha256=FaD0PJxBWJSh_cpOIsjMfUfCP5XPuhtQu_I48xSvYfE,58762
 xinference/api/oauth2/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
 xinference/api/oauth2/auth_service.py,sha256=n88v85kkkM_4NYZBmbETM7OF7XuehWv9gyLRUmhVL1Q,4929
 xinference/api/oauth2/types.py,sha256=ltAkjPBEJVSvYVhzwjaJpe2eUbnqkx2vofLuRbcGMg4,972
@@ -65,12 +65,12 @@ xinference/model/image/model_spec_modelscope.json,sha256=KMI-2YOJoLUtN93LZiqqETW
 xinference/model/image/utils.py,sha256=gxg8jJ2nYaDknzCcSC53WCy1slbB5aWU14AbJbfm6Z4,906
 xinference/model/image/stable_diffusion/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
 xinference/model/image/stable_diffusion/core.py,sha256=YStAtcO-HVR36brvy7DK-D-Z-LT1X4tfwrVrXcLXK5Y,6010
-xinference/model/llm/__init__.py,sha256=j-e1XhrqQvizR4CU4AhURu29t4LA2qWP2hHNf7haj6Q,6417
+xinference/model/llm/__init__.py,sha256=vA1nVY-lDVB4AXuWxsvoevvrU526c4AmSt2UTjTT2pQ,6530
 xinference/model/llm/core.py,sha256=ubS7maXHr5tzTUey1D8ta49Ur1C5RMB-rnpAZNqZZ8c,9526
-xinference/model/llm/llm_family.json,sha256=D1910vsIuaY0O2t6GTAhQulcTfZIg9OriGss8l9cP4o,95056
+xinference/model/llm/llm_family.json,sha256=eGD_z3Z32a9TWh8qjCKyF6qpcjhu39ow5Kfx1zb_eyU,96692
 xinference/model/llm/llm_family.py,sha256=BoqfNX6DpIJylIOUYd1xpDybu1WyV94HMtdQRlRHVho,33447
-xinference/model/llm/llm_family_modelscope.json,sha256=aJgARoqfYKp5slOoGv-6kHi7Z-MuKiUAfBPJXStSGBo,59087
-xinference/model/llm/utils.py,sha256=Gh6CjrOnhQAZYrmRUvxLD_io3RLa-xyYQ5G34Lk7OQ0,25715
+xinference/model/llm/llm_family_modelscope.json,sha256=eGyEottnMKNwRsesOfF5tVof3jOGp4GxM6mu80uiQfg,60741
+xinference/model/llm/utils.py,sha256=js5lml_Ar8TUjAX1YWfhlfAYsFgAhPC35KEiLZbVzOE,26893
 xinference/model/llm/ggml/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
 xinference/model/llm/ggml/chatglm.py,sha256=Zrzw8K2EroI5v2JlwOAJ08tNFs871n86zRtBxuK97Z8,13044
 xinference/model/llm/ggml/ctransformers.py,sha256=n8dTItZe97cF79NkmsVPirqpBcrZiGAQfd2GRpz9-3I,9917
@@ -81,11 +81,11 @@ xinference/model/llm/ggml/tools/convert_ggml_to_gguf.py,sha256=ULvaoAKGH-L6RuRLF
 xinference/model/llm/ggml/tools/gguf.py,sha256=Hv2haR-UN7NdB1N8YId32hFoEPd-JX6_aUNWRJhyJZc,30277
 xinference/model/llm/pytorch/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
 xinference/model/llm/pytorch/baichuan.py,sha256=gh49PNue47T4XUwE5bx_ErDwagb1dYVrhBFYXUsJDQ8,2795
-xinference/model/llm/pytorch/chatglm.py,sha256=jjoYRcZmFht9hXKh8zp6JGlY-jPJTJ5nuUc1nJAFfPs,7099
+xinference/model/llm/pytorch/chatglm.py,sha256=VF4-Kzw-THpedw0EEpnJoCJi4OgDmnlQbqKmVrwCxQI,7149
 xinference/model/llm/pytorch/compression.py,sha256=U0vMJ-JaBt4oC2LffgWg6HbPj1CeUi_YdwVbjDd0mRA,8112
 xinference/model/llm/pytorch/core.py,sha256=bmLyP2GWk4mfq7LP6z8-ME5-AC3KikU2BbglqKPXNck,19228
 xinference/model/llm/pytorch/falcon.py,sha256=Dp19-Sv4mbx1PAVMEp7biATqs4T1WdwcvZZhuVndmYQ,4482
-xinference/model/llm/pytorch/internlm2.py,sha256=WvXehL7iSoL9OB_WE_3omcFBJkwnuUHfWYnS0wWPRvo,5661
+xinference/model/llm/pytorch/internlm2.py,sha256=fW-iQxo2CCyZ7dODJPb6w_NoyCfGZj6zCZ_CI1Su_os,5707
 xinference/model/llm/pytorch/llama_2.py,sha256=RiS6chAj-nh_E-GOZPV-Ze1zDevGmJkhWZMdF4Etoq4,3693
 xinference/model/llm/pytorch/qwen_vl.py,sha256=jyE9em7sMTosYPLqW08aCuEc6c2zNKLJzIcFzowKjyk,5722
 xinference/model/llm/pytorch/spec_decoding_utils.py,sha256=brSxcXkXlELprGK2cLjNTW-3h1Lk9GlgcOFYZo5NBAA,18781
@@ -93,8 +93,10 @@ xinference/model/llm/pytorch/spec_model.py,sha256=8E0jZuXp9MlNLCqgvOjNdUeYqJdMZq
 xinference/model/llm/pytorch/utils.py,sha256=frFvbT2Ka4YRsfnx6ma8IW_i8eqqRwSo5pCU0qnOJtQ,17288
 xinference/model/llm/pytorch/vicuna.py,sha256=PjLD-5BF4MVXkPs1_WncCCuziDKl5tiGXHf-zkrBz_c,2309
 xinference/model/llm/pytorch/yi_vl.py,sha256=aJZQauLmyXEtnz57ZU-Z3pmi0dFdlR_UjnZu0QsC06c,9393
+xinference/model/llm/sglang/__init__.py,sha256=-sjSIQ4K6w-TEzx49kVaWeWC443fnZqODU91GCQ_JNo,581
+xinference/model/llm/sglang/core.py,sha256=eqAczZfGJInC_jihXVeKiWQ79Llk3reHDBkdShQlH-0,12915
 xinference/model/llm/vllm/__init__.py,sha256=h_JgzSqV5lP6vQ6XX_17kE4IY4BRnvKta_7VLQAL1ms,581
-xinference/model/llm/vllm/core.py,sha256=l6-oywSkDrnW8576xdtyqNiKmE6nopMFpzZRvFB988s,15693
+xinference/model/llm/vllm/core.py,sha256=f2hsH7UayhIan47QEif8XQvgD45u9ZMn5LzeWY42oag,15955
 xinference/model/rerank/__init__.py,sha256=BXIL1uu3ZpZHX9bODhW9lxKUXudZE7-OkXFmmM5rpMU,2817
 xinference/model/rerank/core.py,sha256=A9NqcEFRzahE8fcRgRUfY8pHUm5jYG5THxI2qSKTjFY,7771
 xinference/model/rerank/custom.py,sha256=NKk7jA7p4xkuwS5WoOs2SY2wdnoAVpyCjBTvv317bBw,3917
@@ -15374,9 +15376,9 @@ xinference/web/ui/node_modules/yargs-parser/package.json,sha256=BSwbOzgetKXMK4u0
 xinference/web/ui/node_modules/yocto-queue/package.json,sha256=6U1XHQPGXJTqsiFvT953ORihUtXTblZy4fXBWP9qxC0,725
 xinference/web/ui/node_modules/yup/package.json,sha256=xRFSROB9NKxqSWHEVFvSTsPs9Ll074uo8OS1zEw0qhA,1206
 xinference/web/ui/node_modules/yup/node_modules/type-fest/package.json,sha256=JTv2zTTVgxQ2H82m1-6qEpdMv08lHjFx4Puf_MsbB_Q,1134
-xinference-0.9.3.dist-info/LICENSE,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
-xinference-0.9.3.dist-info/METADATA,sha256=yklS-7FS330guT_xiz7gXER1FNjyKI7r2UTXMly4WEI,14129
-xinference-0.9.3.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-xinference-0.9.3.dist-info/entry_points.txt,sha256=-lDyyzqWMFQF0Rgm7VxBNz0V-bMBMQLRR3pvQ-Y8XTY,226
-xinference-0.9.3.dist-info/top_level.txt,sha256=L1rQt7pl6m8tmKXpWVHzP-GtmzAxp663rXxGE7qnK00,11
-xinference-0.9.3.dist-info/RECORD,,
+xinference-0.9.4.dist-info/LICENSE,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
+xinference-0.9.4.dist-info/METADATA,sha256=n-FCXzg0OH6gsXeLERrjxAY47xVGBc3mJj5seynjvGw,14273
+xinference-0.9.4.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+xinference-0.9.4.dist-info/entry_points.txt,sha256=-lDyyzqWMFQF0Rgm7VxBNz0V-bMBMQLRR3pvQ-Y8XTY,226
+xinference-0.9.4.dist-info/top_level.txt,sha256=L1rQt7pl6m8tmKXpWVHzP-GtmzAxp663rXxGE7qnK00,11
+xinference-0.9.4.dist-info/RECORD,,