xinference 1.3.1.post1__py3-none-any.whl → 1.4.1__py3-none-any.whl

xinference/_compat.py

@@ -102,6 +102,7 @@ class CreateChatCompletionOpenAI(BaseModel):
     frequency_penalty: Optional[float]
     logit_bias: Optional[Dict[str, int]]
     logprobs: Optional[bool]
+    max_completion_tokens: Optional[int]
     max_tokens: Optional[int]
     n: Optional[int]
     parallel_tool_calls: Optional[bool]

xinference/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-03-11T12:00:36+0800",
+ "date": "2025-04-03T21:26:30+0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "2ef99fbb5450a76a6ba07a909f58b8c2e4c22a28",
- "version": "1.3.1.post1"
+ "full-revisionid": "23260be3b917e7a2e8381927721ed3de815c0a99",
+ "version": "1.4.1"
 }
 '''  # END VERSION_JSON
 

xinference/api/restful_api.py

@@ -1952,6 +1952,7 @@ class RESTfulAPI(CancelMixin):
             "logit_bias",
             "logit_bias_type",
             "user",
+            "max_completion_tokens",
         }
 
         raw_kwargs = {k: v for k, v in raw_body.items() if k not in exclude}
@@ -1964,6 +1965,9 @@ class RESTfulAPI(CancelMixin):
         if body.max_tokens is None:
             kwargs["max_tokens"] = max_tokens_field.default
 
+        if body.max_completion_tokens is not None:
+            kwargs["max_tokens"] = body.max_completion_tokens
+
         if body.logit_bias is not None:
             raise HTTPException(status_code=501, detail="Not implemented")
 

xinference/core/chat_interface.py

@@ -137,7 +137,7 @@ class GradioInterface:
                 ):
                     assert isinstance(chunk, dict)
                     delta = chunk["choices"][0]["delta"]
-                    if "content" not in delta:
+                    if "content" not in delta or delta["content"] is None:
                         continue
                     else:
                         # some model like deepseek-r1-distill-qwen

xinference/core/model.py

@@ -185,7 +185,7 @@ class ModelActor(xo.StatelessActor, CancelMixin):
                 )
 
         if hasattr(self._model, "stop") and callable(self._model.stop):
-            self._model.stop()
+            await asyncio.to_thread(self._model.stop)
 
         if isinstance(self._model, LLMVLLMModel):
             if self._transfer_ref is not None:
@@ -284,6 +284,8 @@ class ModelActor(xo.StatelessActor, CancelMixin):
     async def __post_create__(self):
         self._loop = asyncio.get_running_loop()
 
+        logger.debug("Starting ModelActor at %s, uid: %s", self.address, self.uid)
+
         self._handle_pending_requests_task = asyncio.create_task(
             self._handle_pending_requests()
         )
@@ -463,7 +465,9 @@ class ModelActor(xo.StatelessActor, CancelMixin):
         while True:
             i += 1
             try:
-                self._model.load()
+                if hasattr(self._model, "set_loop"):
+                    self._model.set_loop(asyncio.get_running_loop())
+                await asyncio.to_thread(self._model.load)
                 if hasattr(self._model, "driver_info"):
                     self._driver_info = self._model.driver_info
                 break
@@ -490,7 +494,23 @@ class ModelActor(xo.StatelessActor, CancelMixin):
 
     async def wait_for_load(self):
         if hasattr(self._model, "wait_for_load"):
-            self._model.wait_for_load()
+            await asyncio.to_thread(self._model.wait_for_load)
+
+    def need_create_pools(self):
+        return getattr(self._model, "need_create_pools", False)
+
+    def set_pool_addresses(self, pool_addresses: List[str]):
+        if hasattr(self._model, "set_pool_addresses"):
+            self._model.set_pool_addresses(pool_addresses)
+
+    def get_pool_addresses(self) -> Optional[List[str]]:
+        if hasattr(self._model, "get_pool_addresses"):
+            return self._model.get_pool_addresses()
+        return None
+
+    def set_worker_addresses(self, shard: int, worker_addresses: List[str]):
+        if hasattr(self._model, "set_worker_addresses"):
+            self._model.set_worker_addresses(shard, worker_addresses)
 
     def model_uid(self):
         return (

xinference/core/supervisor.py

@@ -1097,6 +1097,7 @@ class SupervisorActor(xo.StatelessActor):
                 xavier_config=xavier_config,
                 **kwargs,
             )
+            await worker_ref.wait_for_load(_replica_model_uid)
             self._replica_model_uid_to_worker[_replica_model_uid] = worker_ref
             return subpool_address
 
@@ -1242,6 +1243,11 @@ class SupervisorActor(xo.StatelessActor):
                 available_workers.append(worker_ip)
 
         async def _launch_model():
+            # Validation of n_worker, intercept if it is greater than the available workers.
+            if n_worker > len(available_workers):
+                raise ValueError(
+                    "n_worker cannot be larger than the number of available workers."
+                )
             try:
                 for _idx, rep_model_uid in enumerate(
                     iter_replica_model_uid(model_uid, replica)

xinference/core/worker.py

@@ -874,7 +874,7 @@ class WorkerActor(xo.StatelessActor):
             subpool_address, devices = await self._create_subpool(
                 model_uid, model_type, n_gpu=n_gpu, gpu_idx=gpu_idx
             )
-
+            all_subpool_addresses = [subpool_address]
             try:
                 xavier_config: Optional[Dict] = kwargs.pop("xavier_config", None)
                 if xavier_config is not None:
@@ -885,7 +885,7 @@ class WorkerActor(xo.StatelessActor):
                     # add a few kwargs
                     model_kwargs.update(
                         dict(
-                            address=self.address,
+                            address=subpool_address,
                             n_worker=n_worker,
                             shard=shard,
                             driver_info=driver_info,
@@ -923,11 +923,28 @@ class WorkerActor(xo.StatelessActor):
                     shard=shard,
                     driver_info=driver_info,
                 )
+                if await model_ref.need_create_pools() and (
+                    len(devices) > 1 or n_worker > 1  # type: ignore
+                ):
+                    coros = []
+                    env_name = get_available_device_env_name() or "CUDA_VISIBLE_DEVICES"
+                    env_value = ",".join(devices)
+                    for device in devices:
+                        coros.append(
+                            self._main_pool.append_sub_pool(
+                                env={env_name: env_value},
+                                start_method=self._get_start_method(),
+                            )
+                        )
+                    pool_addresses = await asyncio.gather(*coros)
+                    all_subpool_addresses.extend(pool_addresses)
+                    await model_ref.set_pool_addresses(pool_addresses)
                 await model_ref.load()
             except:
                 logger.error(f"Failed to load model {model_uid}", exc_info=True)
                 self.release_devices(model_uid=model_uid)
-                await self._main_pool.remove_sub_pool(subpool_address)
+                for addr in all_subpool_addresses:
+                    await self._main_pool.remove_sub_pool(addr)
                 raise
             self._model_uid_to_model[model_uid] = model_ref
             self._model_uid_to_model_spec[model_uid] = model_description
@@ -994,15 +1011,36 @@ class WorkerActor(xo.StatelessActor):
         if model_ref is None:
             logger.debug("Model not found, uid: %s", model_uid)
 
+        pool_addresses = None
+        if model_ref is not None:
+            try:
+                # pool addresses if model.need_create_pools()
+                pool_addresses = await model_ref.get_pool_addresses()
+            except Exception as e:
+                # process may disappear, we just ignore it.
+                logger.debug("Fail to get pool addresses, error: %s", e)
+
         try:
-            await xo.destroy_actor(model_ref)
+            logger.debug("Start to destroy model actor: %s", model_ref)
+            coro = xo.destroy_actor(model_ref)
+            await asyncio.wait_for(coro, timeout=5)
         except Exception as e:
             logger.debug(
                 "Destroy model actor failed, model uid: %s, error: %s", model_uid, e
             )
         try:
+            to_remove_addresses = []
             subpool_address = self._model_uid_to_addr[model_uid]
-            await self._main_pool.remove_sub_pool(subpool_address, force=True)
+            to_remove_addresses.append(subpool_address)
+            if pool_addresses:
+                to_remove_addresses.extend(pool_addresses)
+            logger.debug("Remove sub pools: %s", to_remove_addresses)
+            coros = []
+            for to_remove_addr in to_remove_addresses:
+                coros.append(
+                    self._main_pool.remove_sub_pool(to_remove_addr, force=True)
+                )
+            await asyncio.gather(*coros)
         except Exception as e:
             logger.debug(
                 "Remove sub pool failed, model uid: %s, error: %s", model_uid, e
@@ -1204,18 +1242,23 @@ class WorkerActor(xo.StatelessActor):
         model_ref = self._model_uid_to_model[rep_model_uid]
         await model_ref.start_transfer_for_vllm(rank_addresses)
 
-    @log_async(logger=logger, level=logging.INFO)
-    async def launch_rank0_model(
-        self, rep_model_uid: str, xavier_config: Dict[str, Any]
-    ) -> Tuple[str, int]:
-        from ..model.llm.vllm.xavier.collective_manager import Rank0ModelActor
-
+    @staticmethod
+    def _get_start_method():
         if os.name != "nt" and platform.system() != "Darwin":
             # Linux
             start_method = "forkserver"
         else:
             # Windows and macOS
             start_method = "spawn"
+        return start_method
+
+    @log_async(logger=logger, level=logging.INFO)
+    async def launch_rank0_model(
+        self, rep_model_uid: str, xavier_config: Dict[str, Any]
+    ) -> Tuple[str, int]:
+        from ..model.llm.vllm.xavier.collective_manager import Rank0ModelActor
+
+        start_method = self._get_start_method()
         subpool_address = await self._main_pool.append_sub_pool(
             start_method=start_method
         )

xinference/model/llm/__init__.py

@@ -132,7 +132,7 @@ def _install():
     from .llama_cpp.core import LlamaCppChatModel, LlamaCppModel, XllamaCppModel
     from .lmdeploy.core import LMDeployChatModel, LMDeployModel
     from .mlx.core import MLXChatModel, MLXModel, MLXVisionModel
-    from .sglang.core import SGLANGChatModel, SGLANGModel
+    from .sglang.core import SGLANGChatModel, SGLANGModel, SGLANGVisionModel
     from .transformers.chatglm import ChatglmPytorchChatModel
     from .transformers.cogagent import CogAgentChatModel
     from .transformers.cogvlm2 import CogVLM2Model
@@ -143,6 +143,8 @@ def _install():
         DeepSeekV2PytorchModel,
     )
     from .transformers.deepseek_vl import DeepSeekVLChatModel
+    from .transformers.deepseek_vl2 import DeepSeekVL2ChatModel
+    from .transformers.gemma3 import Gemma3ChatModel, Gemma3TextChatModel
     from .transformers.glm4v import Glm4VModel
     from .transformers.glm_edge_v import GlmEdgeVModel
     from .transformers.intern_vl import InternVLChatModel
@@ -172,7 +174,7 @@ def _install():
             XllamaCppModel,
         ]
     )
-    SGLANG_CLASSES.extend([SGLANGModel, SGLANGChatModel])
+    SGLANG_CLASSES.extend([SGLANGModel, SGLANGChatModel, SGLANGVisionModel])
     VLLM_CLASSES.extend([VLLMModel, VLLMChatModel, VLLMVisionModel])
     MLX_CLASSES.extend([MLXModel, MLXChatModel, MLXVisionModel])
     LMDEPLOY_CLASSES.extend([LMDeployModel, LMDeployChatModel])
@@ -186,6 +188,7 @@ def _install():
             Qwen2AudioChatModel,
             YiVLChatModel,
             DeepSeekVLChatModel,
+            DeepSeekVL2ChatModel,
             InternVLChatModel,
             PytorchModel,
             CogVLM2Model,
@@ -198,6 +201,8 @@ def _install():
             OptPytorchModel,
             GlmEdgeVModel,
             CogAgentChatModel,
+            Gemma3TextChatModel,
+            Gemma3ChatModel,
         ]
     )
     if OmniLMMModel:  # type: ignore

xinference/model/llm/core.py

@@ -54,6 +54,7 @@ class LLM(abc.ABC):
         **kwargs,
     ):
         self.model_uid, self.rep_id = parse_replica_model_uid(replica_model_uid)
+        self.raw_model_uid = replica_model_uid
         self.model_family = model_family
         self.model_spec = model_spec
         self.quantization = quantization

xinference/model/llm/llama_cpp/core.py

@@ -39,10 +39,15 @@ logger = logging.getLogger(__name__)
 USE_XLLAMACPP = bool(int(os.environ.get("USE_XLLAMACPP", 0)))
 
 
-class _Sentinel:
+class _Done:
     pass
 
 
+class _Error:
+    def __init__(self, msg):
+        self.msg = msg
+
+
 class XllamaCppModel(LLM, ChatModelMixin):
     def __init__(
         self,
@@ -200,7 +205,14 @@ class XllamaCppModel(LLM, ChatModelMixin):
             )
             prompt_json = orjson.dumps(data)
 
-            def _res_callback(ok):
+            def _error_callback(err):
+                try:
+                    msg = orjson.loads(err)
+                    q.put(_Error(msg))
+                except Exception as e:
+                    q.put(_Error(str(e)))
+
+            def _ok_callback(ok):
                 try:
                     res = orjson.loads(ok)
                     res["model"] = self.model_uid
@@ -209,10 +221,10 @@ class XllamaCppModel(LLM, ChatModelMixin):
                     logger.exception("handle_completions callback failed: %s", e)
 
             try:
-                self._llm.handle_completions(prompt_json, _res_callback, _res_callback)
+                self._llm.handle_completions(prompt_json, _error_callback, _ok_callback)
             except Exception as ex:
                 logger.exception("handle_completions failed: %s", ex)
-            q.put(_Sentinel)
+            q.put(_Done)
 
         assert self._executor
         self._executor.submit(_handle_completion)
@@ -220,12 +232,17 @@ class XllamaCppModel(LLM, ChatModelMixin):
         if stream:
 
             def _to_iterator():
-                while (r := q.get()) is not _Sentinel:
+                while (r := q.get()) is not _Done:
+                    if type(r) is _Error:
+                        raise Exception("Got error in generate stream: %s", r.msg)
                     yield r
 
             return _to_iterator()
         else:
-            return q.get()
+            r = q.get()
+            if type(r) is _Error:
+                raise Exception("Got error in generate: %s", r.msg)
+            return r
 
     def chat(
         self,
@@ -253,7 +270,14 @@ class XllamaCppModel(LLM, ChatModelMixin):
             )
             prompt_json = orjson.dumps(data)
 
-            def _res_callback(ok):
+            def _error_callback(err):
+                try:
+                    msg = orjson.loads(err)
+                    q.put(_Error(msg))
+                except Exception as e:
+                    q.put(_Error(str(e)))
+
+            def _ok_callback(ok):
                 try:
                     res = orjson.loads(ok)
                     res["model"] = self.model_uid
@@ -263,11 +287,11 @@ class XllamaCppModel(LLM, ChatModelMixin):
 
             try:
                 self._llm.handle_chat_completions(
-                    prompt_json, _res_callback, _res_callback
+                    prompt_json, _error_callback, _ok_callback
                 )
             except Exception as ex:
                 logger.exception("handle_chat_completions failed: %s", ex)
-            q.put(_Sentinel)
+            q.put(_Done)
 
         assert self._executor
         self._executor.submit(_handle_chat_completion)
@@ -275,14 +299,24 @@ class XllamaCppModel(LLM, ChatModelMixin):
         if stream:
 
             def _to_iterator():
-                while (r := q.get()) is not _Sentinel:
-                    yield r
+                while (r := q.get()) is not _Done:
+                    if type(r) is _Error:
+                        raise Exception("Got error in chat stream: %s", r.msg)
+                    # Get valid keys (O(1) lookup)
+                    chunk_keys = ChatCompletionChunk.__annotations__
+                    # The chunk may contain additional keys (e.g., system_fingerprint),
+                    # which might not conform to OpenAI/DeepSeek formats.
+                    # Filter out keys that are not part of ChatCompletionChunk.
+                    yield {key: r[key] for key in chunk_keys if key in r}
 
             return self._to_chat_completion_chunks(
                 _to_iterator(), self.reasoning_parser
             )
         else:
-            return self._to_chat_completion(q.get(), self.reasoning_parser)
+            r = q.get()
+            if type(r) is _Error:
+                raise Exception("Got error in chat: %s", r.msg)
+            return self._to_chat_completion(r, self.reasoning_parser)
 
 
 class LlamaCppModel(LLM):
@@ -533,10 +567,11 @@ class LlamaCppChatModel(LlamaCppModel, ChatModelMixin):
         tools = generate_config.pop("tools", []) if generate_config else None
         full_context_kwargs = {}
         if tools:
-            if model_family in QWEN_TOOL_CALL_FAMILY:
+            if (
+                model_family in QWEN_TOOL_CALL_FAMILY
+                or model_family in DEEPSEEK_TOOL_CALL_FAMILY
+            ):
                 full_context_kwargs["tools"] = tools
-            elif model_family in DEEPSEEK_TOOL_CALL_FAMILY:
-                self._tools_to_messages_for_deepseek(messages, tools)
         assert self.model_family.chat_template is not None
         full_prompt = self.get_full_context(
             messages, self.model_family.chat_template, **full_context_kwargs