xinference 1.3.1.post1__py3-none-any.whl → 1.4.1__py3-none-any.whl

xinference/model/llm/transformers/gemma3.py

@@ -0,0 +1,185 @@
+# Copyright 2022-2025 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import logging
+import sys
+import uuid
+from typing import Iterator, List, Optional, Union
+
+from ....model.utils import select_device
+from ....types import (
+    ChatCompletion,
+    ChatCompletionChunk,
+    ChatCompletionMessage,
+    CompletionChunk,
+)
+from ..llm_family import LLMFamilyV1, LLMSpecV1
+from ..utils import generate_chat_completion, generate_completion_chunk
+from .core import PytorchChatModel, PytorchGenerateConfig
+from .utils import cache_clean
+
+logger = logging.getLogger(__name__)
+
+
+class Gemma3TextChatModel(PytorchChatModel):
+    @classmethod
+    def match(
+        cls, model_family: "LLMFamilyV1", model_spec: "LLMSpecV1", quantization: str
+    ) -> bool:
+        if model_spec.model_format not in ["pytorch", "gptq", "awq"]:
+            return False
+        llm_family = model_family.model_family or model_family.model_name
+        if "gemma-3-1b-it".lower() in llm_family.lower():
+            return True
+        return False
+
+
+class Gemma3ChatModel(PytorchChatModel):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._tokenizer = None
+        self._model = None
+        self._device = None
+        self._processor = None
+
+    @classmethod
+    def match(
+        cls, model_family: "LLMFamilyV1", model_spec: "LLMSpecV1", quantization: str
+    ) -> bool:
+        if model_spec.model_format not in ["pytorch", "gptq", "awq"]:
+            return False
+        llm_family = model_family.model_family or model_family.model_name
+        if "gemma-3-it".lower() in llm_family.lower():
+            return True
+        return False
+
+    def load(self):
+        from transformers import AutoProcessor, Gemma3ForConditionalGeneration
+
+        device = self._pytorch_model_config.get("device", "auto")
+        device = select_device(device)
+        self._device = device
+        # for multiple GPU, set back to auto to make multiple devices work
+        device = "auto" if device == "cuda" else device
+
+        self._processor = AutoProcessor.from_pretrained(self.model_path)
+        self._tokenizer = self._processor.tokenizer
+        self._model = Gemma3ForConditionalGeneration.from_pretrained(
+            self.model_path,
+            device_map="auto",
+            torch_dtype="bfloat16",
+        )
+
+    @cache_clean
+    def chat(
+        self,
+        messages: List[ChatCompletionMessage],  # type: ignore
+        generate_config: Optional[PytorchGenerateConfig] = None,
+    ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
+        messages = self._transform_messages(messages)
+
+        generate_config = generate_config if generate_config else {}
+
+        stream = generate_config.get("stream", False) if generate_config else False
+
+        if stream:
+            it = self._generate_stream(messages, generate_config)
+            return self._to_chat_completion_chunks(it)
+        else:
+            c = self._generate(messages, generate_config)
+            return c
+
+    def _generate(
+        self, messages: List, config: PytorchGenerateConfig = {}
+    ) -> ChatCompletion:
+        inputs = self._processor.apply_chat_template(
+            messages,
+            add_generation_prompt=True,
+            tokenize=True,
+            return_dict=True,
+            return_tensors="pt",
+        ).to(self._device)
+        input_len = inputs["input_ids"].shape[-1]
+
+        generation = self._model.generate(**inputs, do_sample=False)
+        generation = generation[0][input_len:]
+
+        decoded = self._processor.decode(generation, skip_special_tokens=True)
+        return generate_chat_completion(self.model_uid, decoded)
+
+    def _generate_stream(
+        self, messages: List, config: PytorchGenerateConfig = {}
+    ) -> Iterator[CompletionChunk]:
+        from threading import Thread
+
+        from transformers import TextIteratorStreamer
+
+        inputs = self._processor.apply_chat_template(
+            messages,
+            add_generation_prompt=True,
+            tokenize=True,
+            return_dict=True,
+            return_tensors="pt",
+        ).to(self._device)
+
+        tokenizer = self._tokenizer
+        streamer = TextIteratorStreamer(
+            tokenizer, timeout=60.0, skip_prompt=True, skip_special_tokens=True
+        )
+
+        gen_kwargs = {"streamer": streamer, **inputs}
+        error = None
+
+        def model_generate():
+            try:
+                return self._model.generate(**gen_kwargs)
+            except Exception:
+                nonlocal error
+                error = sys.exc_info()
+                streamer.end()
+                raise
+
+        thread = Thread(target=model_generate)
+        thread.start()
+
+        completion_id = str(uuid.uuid1())
+        for new_text in streamer:
+            yield generate_completion_chunk(
+                chunk_text=new_text,
+                finish_reason=None,
+                chunk_id=completion_id,
+                model_uid=self.model_uid,
+                prompt_tokens=-1,
+                completion_tokens=-1,
+                total_tokens=-1,
+                has_choice=True,
+                has_content=True,
+            )
+
+        if error:
+            _, err, tb = error  # type: ignore
+            raise err.with_traceback(tb)
+
+        yield generate_completion_chunk(
+            chunk_text=None,
+            finish_reason="stop",
+            chunk_id=completion_id,
+            model_uid=self.model_uid,
+            prompt_tokens=-1,
+            completion_tokens=-1,
+            total_tokens=-1,
+            has_choice=True,
+            has_content=False,
+        )

xinference/model/llm/transformers/intern_vl.py

@@ -245,8 +245,6 @@ class InternVLChatModel(PytorchChatModel):
         family = model_family.model_family or model_family.model_name
         if "internvl" not in family.lower():
             return False
-        if "pytorch" not in model_spec.model_format:
-            return False
         return True
 
     def _get_model_class(self):

xinference/model/llm/utils.py

@@ -79,8 +79,7 @@ LLAMA3_TOOL_CALL_FAMILY = [
 ]
 
 DEEPSEEK_TOOL_CALL_FAMILY = [
-    "deepseek-r1-distill-qwen",
-    "deepseek-r1-distill-llama",
+    "deepseek-v3",
 ]
 
 TOOL_CALL_FAMILY = (
@@ -256,19 +255,26 @@ class ChatModelMixin:
             and choices
             and "delta" in choices[0]
         ):
-            if reasoning_parser is not None:
-                # process parsing reasoning content
-                assert previous_texts is not None
+            if choices[0]["finish_reason"] is None:
+                if reasoning_parser is not None:
+                    # process parsing reasoning content
+                    assert previous_texts is not None
+                    delta = choices[0]["delta"]  # type: ignore
+                    if text := delta.get("content"):
+                        current_text = previous_texts[-1] + text
+                        delta = reasoning_parser.extract_reasoning_content_streaming(
+                            previous_text=previous_texts[-1],
+                            current_text=current_text,
+                            delta_text=text,
+                        )
+                        previous_texts[-1] = current_text
+                        choices[0]["delta"] = delta  # type: ignore
+            elif choices[0]["finish_reason"] is not None:
                 delta = choices[0]["delta"]  # type: ignore
-                if text := delta.get("content"):
-                    current_text = previous_texts[-1] + text
-                    delta = reasoning_parser.extract_reasoning_content_streaming(
-                        previous_text=previous_texts[-1],
-                        current_text=current_text,
-                        delta_text=text,
-                    )
-                    previous_texts[-1] = current_text
-                    choices[0]["delta"] = delta  # type: ignore
+                if "content" not in delta:
+                    delta["content"] = ""  # type: ignore
+                if reasoning_parser is not None:
+                    delta["reasoning_content"] = None  # type: ignore
             # Already a ChatCompletionChunk, we don't need to convert chunk.
             return cast(ChatCompletionChunk, chunk)
 
@@ -287,7 +293,11 @@ class ChatModelMixin:
                         delta_text=choice["text"],
                     )
                     previous_texts[-1] = current_text
-            if "tool_calls" in choice:
+            elif "text" in choice and choice["finish_reason"] is not None:
+                delta["content"] = choice["text"]
+                if reasoning_parser is not None:
+                    delta["reasoning_content"] = None
+            elif "tool_calls" in choice:
                 delta["tool_calls"] = choice["tool_calls"]
             choices_list.append(
                 {
@@ -296,12 +306,19 @@ class ChatModelMixin:
                     "finish_reason": choice["finish_reason"],
                 }
             )
+        assert choices is not None
+        usage = (
+            chunk["usage"]
+            if choices[0]["finish_reason"] is not None and reasoning_parser is not None
+            else None
+        )
         chat_chunk = {
             "id": "chat" + chunk["id"],
             "model": chunk["model"],
             "created": chunk["created"],
             "object": "chat.completion.chunk",
             "choices": choices_list,
+            "usage": usage,
         }
         return cast(ChatCompletionChunk, chat_chunk)
 
@@ -313,12 +330,9 @@ class ChatModelMixin:
     ) -> ChatCompletionChunk:
         choices_list = []
         for i, choice in enumerate(chunk["choices"]):
-            delta = {
-                "role": "assistant",
-            }
-            if reasoning_parser is None:
-                delta["content"] = ""
-            else:
+            delta = ChatCompletionChunkDelta(role="assistant", content="")
+            if reasoning_parser is not None:
+                delta["content"] = None
                 delta["reasoning_content"] = ""
             choices_list.append(
                 {
@@ -359,9 +373,7 @@ class ChatModelMixin:
         reasoning_parse: Optional[ReasoningParser] = None,
     ) -> Iterator[ChatCompletionChunk]:
         previous_texts = [""]
-        for i, chunk in enumerate(chunks):
-            if i == 0:
-                yield cls._get_first_chat_completion_chunk(chunk, reasoning_parse)
+        for _, chunk in enumerate(chunks):
             # usage
             choices = chunk.get("choices")
             if not choices:
@@ -407,14 +419,10 @@ class ChatModelMixin:
         chunks: AsyncGenerator[CompletionChunk, None],
         reasoning_parser: Optional[ReasoningParser] = None,
     ) -> AsyncGenerator[ChatCompletionChunk, None]:
-        i = 0
         previous_texts = [""]
         async for chunk in chunks:
-            if i == 0:
-                chat_chunk = cls._get_first_chat_completion_chunk(
-                    chunk, reasoning_parser
-                )
-            elif not chunk.get("choices"):
+            choices = chunk.get("choices")
+            if not choices:
                 # usage
                 chat_chunk = cls._get_final_chat_completion_chunk(chunk)
             else:
@@ -422,7 +430,6 @@ class ChatModelMixin:
                     chunk, reasoning_parser, previous_texts
                 )
             yield chat_chunk
-            i += 1
 
     @staticmethod
     def _to_chat_completion(
@@ -533,7 +540,7 @@ class ChatModelMixin:
     @classmethod
     def _eval_deepseek_chat_arguments(cls, c) -> List[Tuple]:
         """
-        Parses tool calls from deepseek-r1 format and removes duplicates.
+        Parses tool calls from deepseek-v3 format and removes duplicates.
 
         Returns:
         List[Tuple[Optional[str], Optional[str], Optional[dict]]]
@@ -541,20 +548,24 @@ class ChatModelMixin:
         - (content, None, None) if parsing failed (content is raw JSON text).
 
         Example input:
-        <｜tool▁call｜>get_current_weather
         ```json
-        {"location": "tokyo", "unit": "fahrenheit"}
+        {
+            "name": "get_weather_and_time",
+            "parameters": {
+                "location": "Hangzhou"
+            }
+        }
         ```
 
         Output:
         [
-            (None, "get_current_weather", {"location": "tokyo", "unit": "fahrenheit"})
+            (None, "get_current_weather", {"location": "Hangzhou"})
         ]
         """
 
         text = c["choices"][0]["text"]
 
-        pattern = r"<｜tool▁call｜>(\w+)\s*```json\s*(.*?)\s*```"
+        pattern = r"\s*```json\s*(.*?)\s*```"
         matches = re.findall(pattern, text, re.DOTALL)
 
         if not matches:
@@ -563,22 +574,31 @@ class ChatModelMixin:
         tool_calls = set()  # Used for deduplication
         results = []
 
-        for function_name, args_json in matches:
+        for raw_json in matches:
+            func_and_args = None
             try:
-                arguments = json.loads(args_json)
+                func_and_args = json.loads(raw_json)
                 # Convert dictionary to frozenset for deduplication
-                arguments_hashable = frozenset(arguments.items())
-                tool_call_tuple = (None, function_name, arguments)
+                arguments_hashable = frozenset(func_and_args["parameters"])
+                tool_call_tuple = (
+                    None,
+                    func_and_args["name"],
+                    func_and_args["parameters"],
+                )
             except json.JSONDecodeError:
                 tool_call_tuple = (
-                    args_json,
+                    raw_json,
                     None,
                     None,
                 )  # If parsing fails, treat as raw content
                 arguments_hashable = None  # No need for hashing
 
             # Avoid duplicate entries
-            dedup_key = (function_name, arguments_hashable)
+            dedup_key = (
+                (func_and_args["name"], arguments_hashable)
+                if func_and_args is not None
+                else (raw_json)
+            )
             if dedup_key not in tool_calls:
                 tool_calls.add(dedup_key)
                 results.append(tool_call_tuple)

xinference/model/llm/vllm/core.py

@@ -13,12 +13,16 @@
 # limitations under the License.
 
 import asyncio
+import itertools
 import json
 import logging
 import multiprocessing
 import os
+import sys
+import threading
 import time
 import uuid
+from functools import partial
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -27,10 +31,13 @@ from typing import (
     List,
     Optional,
     Tuple,
+    Type,
     TypedDict,
     Union,
 )
 
+import xoscar as xo
+
 from ....types import (
     ChatCompletion,
     ChatCompletionChunk,
@@ -73,6 +80,7 @@ class VLLMModelConfig(TypedDict, total=False):
     guided_decoding_backend: Optional[str]
     scheduling_policy: Optional[str]
     reasoning_content: bool
+    model_quantization: Optional[str]
 
 
 class VLLMGenerateConfig(TypedDict, total=False):
@@ -161,6 +169,7 @@ if VLLM_INSTALLED and vllm.__version__ >= "0.3.0":
     VLLM_SUPPORTED_CHAT_MODELS.append("QwQ-32B")
     VLLM_SUPPORTED_CHAT_MODELS.append("marco-o1")
     VLLM_SUPPORTED_CHAT_MODELS.append("deepseek-r1-distill-qwen")
+    VLLM_SUPPORTED_CHAT_MODELS.append("fin-r1")
 
 if VLLM_INSTALLED and vllm.__version__ >= "0.3.2":
     VLLM_SUPPORTED_CHAT_MODELS.append("gemma-it")
@@ -216,6 +225,10 @@ if VLLM_INSTALLED and vllm.__version__ >= "0.7.2":
 if VLLM_INSTALLED and vllm.__version__ >= "0.7.3":
     VLLM_SUPPORTED_CHAT_MODELS.append("qwen2.5-instruct-1m")
 
+if VLLM_INSTALLED and vllm.__version__ >= "0.8.0":
+    VLLM_SUPPORTED_CHAT_MODELS.append("gemma-3-1b-it")
+    VLLM_SUPPORTED_VISION_MODEL_LIST.append("gemma-3-it")
+
 
 class VLLMModel(LLM):
     def __init__(
@@ -244,15 +257,59 @@ class VLLMModel(LLM):
         self.lora_modules = peft_model
         self.lora_requests: List[LoRARequest] = []
         self._xavier_config = None
+        # distributed inference
+        self._device_count = None
+        self._address = model_config.pop("address", None)  # type: ignore
+        self._n_worker = model_config.pop("n_worker", 1)  # type: ignore
+        self._shard = model_config.pop("shard", 0)  # type: ignore
+        self._driver_info = model_config.pop("driver_info", None)  # type: ignore
+        self._loading_thread: Optional[threading.Thread] = None
+        self._loading_error = None
+        # variables used for distributed inference and multiple GPUs
+        self._pool_addresses = None
+        self._worker_addresses: Optional[Dict[int, List[str]]] = None
+        self._all_worker_ready: Optional[threading.Event] = None
+        # used to call async
+        self._loop = None
 
     def set_xavier_config(self, value: Optional[Dict]):
         self._xavier_config = value  # type: ignore
 
+    def set_worker_addresses(self, shard: int, worker_addresses: List[str]):
+        assert self._worker_addresses is not None
+        self._worker_addresses[shard] = worker_addresses
+        if (
+            self._all_worker_ready is not None
+            and len(self._worker_addresses) == self._n_worker
+        ):
+            self._all_worker_ready.set()
+
+    @property
+    def driver_info(self) -> Optional[dict]:
+        return self._driver_info
+
+    @property
+    def need_create_pools(self):
+        return True
+
+    def set_pool_addresses(self, pool_addresses: List[str]):
+        self._pool_addresses = pool_addresses  # type: ignore
+
+    def get_pool_addresses(self) -> Optional[List[str]]:
+        return self._pool_addresses
+
+    def set_loop(self, loop: asyncio.AbstractEventLoop):
+        # loop will be passed into XinferenceDistributedExecutor,
+        # to call aynsc method with asyncio.run_coroutine_threadsafe
+        self._loop = loop  # type: ignore
+
     def load(self):
         try:
             import vllm
+            from vllm.config import VllmConfig
             from vllm.engine.arg_utils import AsyncEngineArgs
             from vllm.engine.async_llm_engine import AsyncLLMEngine
+            from vllm.executor.executor_base import ExecutorBase
             from vllm.lora.request import LoRARequest
         except ImportError:
             error_message = "Failed to import module 'vllm'"
@@ -271,6 +328,7 @@ class VLLMModel(LLM):
             # we need to set it to fork to make cupy NCCL work
             multiprocessing.set_start_method("fork", force=True)
 
+        self._device_count = self._get_cuda_count()
         self._model_config = self._sanitize_model_config(self._model_config)
         reasoning_content = self._model_config.pop("reasoning_content")
 
@@ -316,6 +374,83 @@ class VLLMModel(LLM):
             self._engine = XavierEngine.from_engine_args(
                 engine_args, xavier_config=self._xavier_config
             )
+        elif self._n_worker > 1 or (
+            self._device_count > 1 and vllm.__version__ >= "0.7.0"
+        ):
+            from .distributed_executor import XinferenceDistributedExecutor
+
+            # model across multiple workers or GPUs
+            engine_args = AsyncEngineArgs(
+                model=self.model_path,
+                enable_lora=enable_lora,
+                max_loras=max_loras,
+                **self._model_config,
+            )
+
+            assert self._loop is not None
+            self._worker_addresses = {}
+
+            def _load():
+                try:
+                    assert self._pool_addresses
+
+                    if self._shard > 0:
+                        assert self._driver_info
+                        address = self._driver_info["address"]
+
+                        coro = xo.actor_ref(address, self.raw_model_uid)
+                        model_ref = asyncio.run_coroutine_threadsafe(
+                            coro, self._loop
+                        ).result()
+                        coro = model_ref.set_worker_addresses(
+                            self._shard, self._pool_addresses
+                        )
+                        asyncio.run_coroutine_threadsafe(coro, self._loop).result()
+                    else:
+                        self.set_worker_addresses(0, self._pool_addresses)
+                        self._driver_info = {"address": self._address}
+
+                        if self._n_worker > 1:
+                            self._all_worker_ready = threading.Event()
+                            # if model across workers, wait for other workers ready
+                            self._all_worker_ready.wait()
+
+                        # gather all worker addresses
+                        worker_addresses = list(
+                            itertools.chain(
+                                *[
+                                    self._worker_addresses[shard]
+                                    for shard in range(self._n_worker)
+                                ]
+                            )
+                        )
+                        assert worker_addresses
+                        loop = self._loop
+
+                        class XinferenceAsyncLLMEngine(AsyncLLMEngine):
+                            @classmethod
+                            def _get_executor_cls(
+                                cls, engine_config: VllmConfig
+                            ) -> Type[ExecutorBase]:
+                                return partial(  # type: ignore
+                                    XinferenceDistributedExecutor,
+                                    pool_addresses=worker_addresses,
+                                    n_worker=self._n_worker,
+                                    loop=loop,
+                                )
+
+                        self._engine = XinferenceAsyncLLMEngine.from_engine_args(
+                            engine_args
+                        )
+                except:
+                    logger.exception("Creating vllm engine failed")
+                    self._loading_error = sys.exc_info()
+
+            self._loading_thread = threading.Thread(target=_load)
+            self._loading_thread.start()
+            # wait some time for init finish
+            if self._shard == 0:
+                self._loading_thread.join(1)
         else:
             engine_args = AsyncEngineArgs(
                 model=self.model_path,
@@ -328,7 +463,14 @@ class VLLMModel(LLM):
         self._check_health_task = None
         if hasattr(self._engine, "check_health"):
             # vLLM introduced `check_health` since v0.4.1
-            self._check_health_task = asyncio.create_task(self._check_healthy())
+            self._check_health_task = self._loop.create_task(self._check_healthy())
+
+    def wait_for_load(self):
+        if self._loading_thread:
+            self._loading_thread.join()
+            if self._loading_error:
+                _, err, tb = self._loading_error
+                raise err.with_traceback(tb)
 
     def stop(self):
         # though the vLLM engine will shutdown when deleted,
@@ -337,9 +479,10 @@ class VLLMModel(LLM):
         logger.info("Stopping vLLM engine")
         if self._check_health_task:
             self._check_health_task.cancel()
-        if model_executor := getattr(self._engine.engine, "model_executor", None):
-            model_executor.shutdown()
-        self._engine = None
+        if self._engine:
+            if model_executor := getattr(self._engine.engine, "model_executor", None):
+                model_executor.shutdown()
+            self._engine = None
 
     async def init_xavier(self):
         await self._engine.init_xavier()
@@ -370,16 +513,18 @@ class VLLMModel(LLM):
         if model_config is None:
             model_config = VLLMModelConfig()
 
-        cuda_count = self._get_cuda_count()
-
         model_config.setdefault("tokenizer_mode", "auto")
         model_config.setdefault("trust_remote_code", True)
-        model_config.setdefault("tensor_parallel_size", cuda_count)
+        model_config.setdefault("tensor_parallel_size", self._device_count)  # type: ignore
+        model_config.setdefault("pipeline_parallel_size", self._n_worker)  # type: ignore
         model_config.setdefault("block_size", 16)
         model_config.setdefault("swap_space", 4)
         model_config.setdefault("gpu_memory_utilization", 0.90)
         model_config.setdefault("max_num_seqs", 256)
-        model_config.setdefault("quantization", None)
+        if "model_quantization" in model_config:
+            model_config["quantization"] = model_config.pop("model_quantization")
+        else:
+            model_config.setdefault("quantization", None)
         model_config.setdefault("max_model_len", None)
         model_config.setdefault("guided_decoding_backend", "outlines")
         model_config.setdefault("reasoning_content", False)
@@ -840,10 +985,11 @@ class VLLMChatModel(VLLMModel, ChatModelMixin):
         model_family = self.model_family.model_family or self.model_family.model_name
         full_context_kwargs = {}
         if tools:
-            if model_family in QWEN_TOOL_CALL_FAMILY:
+            if (
+                model_family in QWEN_TOOL_CALL_FAMILY
+                or model_family in DEEPSEEK_TOOL_CALL_FAMILY
+            ):
                 full_context_kwargs["tools"] = tools
-            elif model_family in DEEPSEEK_TOOL_CALL_FAMILY:
-                self._tools_to_messages_for_deepseek(messages, tools)
         assert self.model_family.chat_template is not None
         full_prompt = self.get_full_context(
             messages, self.model_family.chat_template, **full_context_kwargs