xinference 1.3.1__py3-none-any.whl → 1.4.0__py3-none-any.whl

xinference/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-03-09T12:06:50+0800",
+ "date": "2025-03-21T14:33:52+0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "5d6ec937ce2aca2511e9e0debc4c2ab06ca41f09",
- "version": "1.3.1"
+ "full-revisionid": "ac88d425e3d5fc12166e22c4032286327871f5f2",
+ "version": "1.4.0"
 }
 '''  # END VERSION_JSON
 

xinference/core/chat_interface.py

@@ -113,6 +113,7 @@ class GradioInterface:
             max_tokens: int,
             temperature: float,
             lora_name: str,
+            stream: bool,
         ) -> Generator:
             from ..client import RESTfulClient
 
@@ -123,29 +124,40 @@ class GradioInterface:
             messages = to_chat(flatten(history))
             messages.append(dict(role="user", content=message))
 
-            response_content = ""
-            for chunk in model.chat(
-                messages,
-                generate_config={
-                    "max_tokens": int(max_tokens),
-                    "temperature": temperature,
-                    "stream": True,
-                    "lora_name": lora_name,
-                },
-            ):
-                assert isinstance(chunk, dict)
-                delta = chunk["choices"][0]["delta"]
-                if "content" not in delta:
-                    continue
-                else:
-                    # some model like deepseek-r1-distill-qwen
-                    # will generate <think>...</think> ...
-                    # in gradio, no output will be rendered,
-                    # thus escape html tags in advance
-                    response_content += html.escape(delta["content"])
-                    yield response_content
-
-            yield response_content
+            if stream:
+                response_content = ""
+                for chunk in model.chat(
+                    messages,
+                    generate_config={
+                        "max_tokens": int(max_tokens),
+                        "temperature": temperature,
+                        "stream": True,
+                        "lora_name": lora_name,
+                    },
+                ):
+                    assert isinstance(chunk, dict)
+                    delta = chunk["choices"][0]["delta"]
+                    if "content" not in delta or delta["content"] is None:
+                        continue
+                    else:
+                        # some model like deepseek-r1-distill-qwen
+                        # will generate <think>...</think> ...
+                        # in gradio, no output will be rendered,
+                        # thus escape html tags in advance
+                        response_content += html.escape(delta["content"])
+                        yield response_content
+
+                yield response_content
+            else:
+                result = model.chat(
+                    messages,
+                    generate_config={
+                        "max_tokens": int(max_tokens),
+                        "temperature": temperature,
+                        "lora_name": lora_name,
+                    },
+                )
+                yield html.escape(result["choices"][0]["message"]["content"])  # type: ignore
 
         return gr.ChatInterface(
             fn=generate_wrapper,
@@ -153,7 +165,9 @@ class GradioInterface:
                 gr.Slider(
                     minimum=1,
                     maximum=self.context_length,
-                    value=512,
+                    value=512
+                    if "reasoning" not in self.model_ability
+                    else self.context_length // 2,
                     step=1,
                     label="Max Tokens",
                 ),
@@ -161,6 +175,7 @@ class GradioInterface:
                     minimum=0, maximum=2, value=1, step=0.01, label="Temperature"
                 ),
                 gr.Text(label="LoRA Name"),
+                gr.Checkbox(label="Stream", value=True),
             ],
             title=f"🚀 Xinference Chat Bot : {self.model_name} 🚀",
             css="""

xinference/model/llm/__init__.py

@@ -143,6 +143,7 @@ def _install():
         DeepSeekV2PytorchModel,
     )
     from .transformers.deepseek_vl import DeepSeekVLChatModel
+    from .transformers.gemma3 import Gemma3ChatModel, Gemma3TextChatModel
     from .transformers.glm4v import Glm4VModel
     from .transformers.glm_edge_v import GlmEdgeVModel
     from .transformers.intern_vl import InternVLChatModel
@@ -198,6 +199,8 @@ def _install():
             OptPytorchModel,
             GlmEdgeVModel,
             CogAgentChatModel,
+            Gemma3TextChatModel,
+            Gemma3ChatModel,
         ]
     )
     if OmniLMMModel:  # type: ignore

xinference/model/llm/core.py

@@ -25,8 +25,7 @@ from typing import TYPE_CHECKING, Dict, List, Literal, Optional, Tuple, Union
 from ...core.utils import parse_replica_model_uid
 from ...types import PeftModelConfig
 from ..core import ModelDescription
-from .reasoning_parsers import deepseek_r1_reasoning_parser  # noqa: F401
-from .reasoning_parsers.abs_reasoning_parsers import ReasoningParserManager
+from .reasoning_parser import ReasoningParser
 
 if TYPE_CHECKING:
     from .llm_family import LLMFamilyV1, LLMSpecV1
@@ -123,9 +122,7 @@ class LLM(abc.ABC):
     def prepare_parse_reasoning_content(self, reasoning_content):
         # Initialize reasoning parser if model has reasoning ability
         if "reasoning" in self.model_family.model_ability and reasoning_content:
-            module_name = self.model_family.model_family or self.model_family.model_name
-            self.reasoning_parser = ReasoningParserManager.get_parser(module_name)
-            self.reasoning_parser = self.reasoning_parser(
+            self.reasoning_parser = ReasoningParser(
                 self.model_family.reasoning_start_tag,
                 self.model_family.reasoning_end_tag,
             )

xinference/model/llm/llama_cpp/core.py

@@ -39,11 +39,16 @@ logger = logging.getLogger(__name__)
 USE_XLLAMACPP = bool(int(os.environ.get("USE_XLLAMACPP", 0)))
 
 
-class _Sentinel:
+class _Done:
     pass
 
 
-class XllamaCppModel(LLM):
+class _Error:
+    def __init__(self, msg):
+        self.msg = msg
+
+
+class XllamaCppModel(LLM, ChatModelMixin):
     def __init__(
         self,
         model_uid: str,
@@ -83,6 +88,7 @@ class XllamaCppModel(LLM):
             llamacpp_model_config.setdefault("n_gpu_layers", -1)
         elif self._is_linux():
             llamacpp_model_config.setdefault("n_gpu_layers", -1)
+        llamacpp_model_config.setdefault("reasoning_content", False)
 
         return llamacpp_model_config
 
@@ -131,6 +137,9 @@ class XllamaCppModel(LLM):
 
             raise ImportError(f"{error_message}\n\n{''.join(installation_guide)}")
 
+        reasoning_content = self._llamacpp_model_config.pop("reasoning_content")
+        self.prepare_parse_reasoning_content(reasoning_content)
+
         if os.path.isfile(self.model_path):
             # mostly passed from --model_path
             model_path = os.path.realpath(self.model_path)
@@ -196,7 +205,14 @@ class XllamaCppModel(LLM):
             )
             prompt_json = orjson.dumps(data)
 
-            def _res_callback(ok):
+            def _error_callback(err):
+                try:
+                    msg = orjson.loads(err)
+                    q.put(_Error(msg))
+                except Exception as e:
+                    q.put(_Error(str(e)))
+
+            def _ok_callback(ok):
                 try:
                     res = orjson.loads(ok)
                     res["model"] = self.model_uid
@@ -205,10 +221,10 @@ class XllamaCppModel(LLM):
                     logger.exception("handle_completions callback failed: %s", e)
 
             try:
-                self._llm.handle_completions(prompt_json, _res_callback, _res_callback)
+                self._llm.handle_completions(prompt_json, _error_callback, _ok_callback)
             except Exception as ex:
                 logger.exception("handle_completions failed: %s", ex)
-            q.put(_Sentinel)
+            q.put(_Done)
 
         assert self._executor
         self._executor.submit(_handle_completion)
@@ -216,12 +232,17 @@ class XllamaCppModel(LLM):
         if stream:
 
             def _to_iterator():
-                while (r := q.get()) is not _Sentinel:
+                while (r := q.get()) is not _Done:
+                    if type(r) is _Error:
+                        raise Exception("Got error in generate stream: %s", r.msg)
                     yield r
 
             return _to_iterator()
         else:
-            return q.get()
+            r = q.get()
+            if type(r) is _Error:
+                raise Exception("Got error in generate: %s", r.msg)
+            return r
 
     def chat(
         self,
@@ -249,7 +270,14 @@ class XllamaCppModel(LLM):
             )
             prompt_json = orjson.dumps(data)
 
-            def _res_callback(ok):
+            def _error_callback(err):
+                try:
+                    msg = orjson.loads(err)
+                    q.put(_Error(msg))
+                except Exception as e:
+                    q.put(_Error(str(e)))
+
+            def _ok_callback(ok):
                 try:
                     res = orjson.loads(ok)
                     res["model"] = self.model_uid
@@ -259,11 +287,11 @@ class XllamaCppModel(LLM):
 
             try:
                 self._llm.handle_chat_completions(
-                    prompt_json, _res_callback, _res_callback
+                    prompt_json, _error_callback, _ok_callback
                 )
             except Exception as ex:
                 logger.exception("handle_chat_completions failed: %s", ex)
-            q.put(_Sentinel)
+            q.put(_Done)
 
         assert self._executor
         self._executor.submit(_handle_chat_completion)
@@ -271,12 +299,19 @@ class XllamaCppModel(LLM):
         if stream:
 
             def _to_iterator():
-                while (r := q.get()) is not _Sentinel:
+                while (r := q.get()) is not _Done:
+                    if type(r) is _Error:
+                        raise Exception("Got error in chat stream: %s", r.msg)
                     yield r
 
-            return _to_iterator()
+            return self._to_chat_completion_chunks(
+                _to_iterator(), self.reasoning_parser
+            )
         else:
-            return q.get()
+            r = q.get()
+            if type(r) is _Error:
+                raise Exception("Got error in chat: %s", r.msg)
+            return self._to_chat_completion(r, self.reasoning_parser)
 
 
 class LlamaCppModel(LLM):
@@ -527,10 +562,11 @@ class LlamaCppChatModel(LlamaCppModel, ChatModelMixin):
         tools = generate_config.pop("tools", []) if generate_config else None
         full_context_kwargs = {}
         if tools:
-            if model_family in QWEN_TOOL_CALL_FAMILY:
+            if (
+                model_family in QWEN_TOOL_CALL_FAMILY
+                or model_family in DEEPSEEK_TOOL_CALL_FAMILY
+            ):
                 full_context_kwargs["tools"] = tools
-            elif model_family in DEEPSEEK_TOOL_CALL_FAMILY:
-                self._tools_to_messages_for_deepseek(messages, tools)
         assert self.model_family.chat_template is not None
         full_prompt = self.get_full_context(
             messages, self.model_family.chat_template, **full_context_kwargs