xinference 1.0.1__py3-none-any.whl → 1.1.1__py3-none-any.whl

xinference/model/llm/mlx/core.py

@@ -168,9 +168,14 @@ class MLXModel(LLM):
             return False
         if "generate" not in llm_family.model_ability:
             return False
+        if "chat" in llm_family.model_ability or "vision" in llm_family.model_ability:
+            # do not process chat or vision
+            return False
         return True
 
-    def _get_prompt_cache(self, prompt, lora_name: Optional[str] = None):
+    def _get_prompt_cache(
+        self, prompt, lora_name: Optional[str] = None, model: Any = None
+    ):
         from mlx_lm.models.cache import make_prompt_cache
 
         assert self._prompt_cache is not None
@@ -182,7 +187,9 @@ class MLXModel(LLM):
             or self._prompt_cache.tokens != prompt[:cache_len]
         ):
             self._prompt_cache.model_key = model_key
-            self._prompt_cache.cache = make_prompt_cache(self._model, self._max_kv_size)
+            self._prompt_cache.cache = make_prompt_cache(
+                model or self._model, self._max_kv_size
+            )
             self._prompt_cache.tokens = []
             logger.debug("Making new prompt cache for %s", self.model_uid)
         else:
@@ -191,18 +198,35 @@ class MLXModel(LLM):
         self._prompt_cache.tokens.extend(prompt)
         return prompt
 
-    def _generate_stream(self, prompt: str, kwargs: MLXGenerateConfig):
-        import mlx.core as mx
-        from mlx_lm.utils import generate_step
+    def _generate_stream_inner(self, **kwargs):
+        from mlx_lm.utils import make_sampler, stream_generate
 
-        model = self._model
+        sampler = make_sampler(
+            temp=kwargs.pop("temperature"), top_p=kwargs.pop("top_p")
+        )
+        prompt_token_ids = kwargs.pop("prompt_token_ids")
+        yield from stream_generate(
+            self._model, self._tokenizer, prompt_token_ids, sampler=sampler, **kwargs
+        )
+
+    def _prepare_inputs(
+        self, prompt: Union[str, Dict[str, Any]], kwargs
+    ) -> Tuple[Any, int]:
+        prompt_token_ids = self._tokenizer.encode(prompt)
+        prompt_token_ids = self._get_prompt_cache(
+            prompt_token_ids, kwargs.get("lora_name")
+        )
+        return prompt_token_ids, len(prompt_token_ids)
+
+    def _generate_stream(
+        self, prompt: Union[str, Dict[str, Any]], kwargs: MLXGenerateConfig
+    ):
         model_uid = self.model_uid
         tokenizer = self._tokenizer
         max_tokens = kwargs["max_tokens"]
         chunk_id = str(uuid.uuid4())
         stop_token_ids = kwargs.get("stop_token_ids", [])
         stream = kwargs.get("stream", False)
-        lora_name = kwargs.get("lora_name")
         stream_options = kwargs.pop("stream_options", None)
         include_usage = (
             stream_options["include_usage"]
@@ -210,39 +234,28 @@ class MLXModel(LLM):
             else False
         )
 
-        prompt_token_ids = tokenizer.encode(prompt)
-        prompt_token_ids = self._get_prompt_cache(prompt_token_ids, lora_name)
-        prompt_tokens = mx.array(prompt_token_ids)
-        input_echo_len = len(prompt_tokens)
+        prompt_token_ids, input_echo_len = self._prepare_inputs(prompt, kwargs)
 
         i = 0
         start = time.time()
         output = ""
         tokens = []
-        for (token, _), i in zip(
-            generate_step(
-                prompt_tokens,
-                model,
-                temp=kwargs["temperature"],
+        for chunk_resp, i in zip(
+            self._generate_stream_inner(
+                prompt_token_ids=prompt_token_ids,
+                max_tokens=max_tokens,
+                temperature=kwargs["temperature"],
+                top_p=kwargs["top_p"],
                 repetition_penalty=kwargs["repetition_penalty"],
                 repetition_context_size=kwargs["repetition_context_size"],
-                top_p=kwargs["top_p"],
-                prompt_cache=self._prompt_cache.cache,  # type: ignore
+                prompt_cache=self._prompt_cache.cache if self._prompt_cache else None,  # type: ignore
             ),
             range(max_tokens),
         ):
+            token = chunk_resp.token
             tokens.append(token)
-            if token == tokenizer.eos_token_id or token in stop_token_ids:  # type: ignore
-                break
-
-            # Yield the last segment if streaming
-            out = tokenizer.decode(
-                token,
-                skip_special_tokens=True,
-                spaces_between_special_tokens=False,
-                clean_up_tokenization_spaces=True,
-            )
 
+            out = chunk_resp.text
             if stream:
                 # this special character is mainly for qwen
                 out = out.strip("�")
@@ -266,11 +279,15 @@ class MLXModel(LLM):
                 total_tokens=(input_echo_len + i),
             ), completion_usage
 
+            if token == tokenizer.eos_token_id or token in stop_token_ids:  # type: ignore
+                break
+
         logger.info(
             f"Average generation speed: {i / (time.time() - start):.2f} tokens/s."
         )
 
-        self._prompt_cache.tokens.extend(tokens)  # type: ignore
+        if self._prompt_cache:
+            self._prompt_cache.tokens.extend(tokens)  # type: ignore
 
         if i == max_tokens - 1:
             finish_reason = "length"
@@ -314,10 +331,12 @@ class MLXModel(LLM):
             yield completion_chunk, completion_usage
 
     def generate(
-        self, prompt: str, generate_config: Optional[MLXGenerateConfig] = None
+        self,
+        prompt: Union[str, Dict[str, Any]],
+        generate_config: Optional[MLXGenerateConfig] = None,
     ) -> Union[Completion, Iterator[CompletionChunk]]:
         def generator_wrapper(
-            prompt: str, generate_config: MLXGenerateConfig
+            prompt: Union[str, Dict[str, Any]], generate_config: MLXGenerateConfig
         ) -> Iterator[CompletionChunk]:
             for completion_chunk, completion_usage in self._generate_stream(
                 prompt,
@@ -356,26 +375,6 @@ class MLXModel(LLM):
 
 
 class MLXChatModel(MLXModel, ChatModelMixin):
-    def __init__(
-        self,
-        model_uid: str,
-        model_family: "LLMFamilyV1",
-        model_spec: "LLMSpecV1",
-        quantization: str,
-        model_path: str,
-        model_config: Optional[MLXModelConfig] = None,
-        peft_model: Optional[List[LoRA]] = None,
-    ):
-        super().__init__(
-            model_uid,
-            model_family,
-            model_spec,
-            quantization,
-            model_path,
-            model_config,
-            peft_model,
-        )
-
     def _sanitize_generate_config(
         self,
         generate_config: Optional[MLXGenerateConfig],
@@ -402,6 +401,9 @@ class MLXChatModel(MLXModel, ChatModelMixin):
             return False
         if "chat" not in llm_family.model_ability:
             return False
+        if "vision" in llm_family.model_ability:
+            # do not process vision
+            return False
         return True
 
     def chat(
@@ -432,3 +434,237 @@ class MLXChatModel(MLXModel, ChatModelMixin):
             if tools:
                 return self._tool_calls_completion(self.model_family, self.model_uid, c)
             return self._to_chat_completion(c)
+
+
+class MLXVisionModel(MLXModel, ChatModelMixin):
+    @classmethod
+    def match(
+        cls, llm_family: "LLMFamilyV1", llm_spec: "LLMSpecV1", quantization: str
+    ) -> bool:
+        if llm_spec.model_format not in ["mlx"]:
+            return False
+        if sys.platform != "darwin" or platform.processor() != "arm":
+            # only work for Mac M chips
+            return False
+        if "vision" not in llm_family.model_ability:
+            return False
+        return True
+
+    def _load_model(self, **kwargs):
+        try:
+            from mlx_vlm import load
+        except ImportError:
+            error_message = "Failed to import module 'mlx_vlm'"
+            installation_guide = [
+                "Please make sure 'mlx_vlm' is installed. ",
+                "You can install it by `pip install mlx_vlm`\n",
+            ]
+
+            raise ImportError(f"{error_message}\n\n{''.join(installation_guide)}")
+
+        self._prompt_cache = PromptCache()
+
+        return load(self.model_path)
+
+    def load(self):
+        kwargs = {}
+        kwargs["revision"] = self._model_config.get(
+            "revision", self.model_spec.model_revision
+        )
+        kwargs["trust_remote_code"] = self._model_config.get("trust_remote_code")
+        kwargs["cache_limit_gb"] = self._model_config.pop("cache_limit_gb", None)
+
+        self._model, self._processor = self._load_model(**kwargs)
+        self._tokenizer = self._processor.tokenizer
+
+    def _generate_stream_inner_no_image(self, **kwargs):
+        import mlx.nn as nn
+        from mlx_lm.utils import make_sampler, stream_generate
+
+        # For mlx-lm, the model(inputs) will return logits,
+        # but the language model in mlx-vlm will return an object
+        # https://github.com/Blaizzy/mlx-vlm/blob/3f5e1620072440afb7496940f67ac1c7fc64056f/mlx_vlm/models/base.py#L260
+        # so we cannot pass the language model to stream_generate directly
+        # we wrap here to just let model(inputs) return logits to pass stream_generate
+        class ModelWrapper(nn.Module):
+            def __init__(self, model):
+                super().__init__()
+                self._model = model.language_model
+
+            @property
+            def layers(self):
+                return self._model.layers
+
+            def __call__(self, *args, **kwargs):
+                return self._model(*args, **kwargs).logits
+
+        sampler = make_sampler(
+            temp=kwargs.pop("temperature"), top_p=kwargs.pop("top_p")
+        )
+        prompt_token_ids = kwargs.pop("prompt_token_ids")
+        yield from stream_generate(
+            ModelWrapper(self._model),
+            self._tokenizer,
+            prompt_token_ids,
+            sampler=sampler,
+            **kwargs,
+        )
+
+    def _generate_stream_inner(self, **kwargs):
+        import mlx.core as mx
+        from mlx_lm.utils import GenerationResponse
+        from mlx_vlm.utils import generate_step
+
+        inputs = kwargs["prompt_token_ids"]
+
+        if not isinstance(inputs, tuple):
+            # no images
+            yield from self._generate_stream_inner_no_image(**kwargs)
+            return
+
+        max_tokens = kwargs.pop("max_tokens")
+        input_ids, pixel_values, mask = inputs[:3]
+
+        kwargs = {
+            k: v
+            for k, v in zip(
+                [
+                    "image_grid_thw",
+                    "image_sizes",
+                    "aspect_ratio_ids",
+                    "aspect_ratio_mask",
+                    "cross_attention_mask",
+                ],
+                inputs[3:],
+            )
+        }
+
+        tokenizer = self._processor.tokenizer
+        detokenizer = self._processor.detokenizer
+
+        detokenizer.reset()
+        tic = time.perf_counter()
+        for (token, logprobs), n in zip(
+            generate_step(input_ids, self._model, pixel_values, mask, **kwargs),
+            range(max_tokens),
+        ):
+            if n == 0:
+                prompt_time = time.perf_counter() - tic
+                prompt_tps = len(input_ids) / prompt_time
+                tic = time.perf_counter()
+            if token == tokenizer.eos_token_id:
+                break
+            detokenizer.add_token(token)
+
+            # Yield the last segment if streaming
+            yield GenerationResponse(
+                text=detokenizer.last_segment,
+                token=token,
+                logprobs=logprobs,
+                prompt_tokens=len(input_ids),
+                prompt_tps=prompt_tps,
+                generation_tokens=n + 1,
+                generation_tps=(n + 1) / (time.perf_counter() - tic),
+                peak_memory=mx.metal.get_peak_memory() / 1e9,
+            )
+
+        detokenizer.finalize()
+        yield GenerationResponse(
+            text=detokenizer.last_segment,
+            token=token,
+            logprobs=logprobs,
+            prompt_tokens=len(input_ids),
+            prompt_tps=prompt_tps,
+            generation_tokens=n + 1,
+            generation_tps=(n + 1) / (time.perf_counter() - tic),
+            peak_memory=mx.metal.get_peak_memory() / 1e9,
+        )
+
+    def _prepare_inputs(
+        self, prompt: Union[str, Dict[str, Any]], kwargs
+    ) -> Tuple[Any, int]:
+        from mlx_vlm import prepare_inputs
+
+        prompt_str = prompt.get("prompt")  # type: ignore
+        images = prompt.get("multi_modal_data", {}).get("image")  # type: ignore
+        if images and not isinstance(images, list):
+            images = [images]
+        if hasattr(self._model.config, "image_token_index"):
+            image_token_index = self._model.config.image_token_index
+        else:
+            image_token_index = None
+
+        if not images:
+            prompt = prompt["prompt"]  # type: ignore
+            prompt_token_ids = self._tokenizer.encode(prompt)
+            prompt_token_ids = self._get_prompt_cache(
+                prompt_token_ids,
+                kwargs.get("lora_name"),
+                model=self._model.language_model,
+            )
+            return prompt_token_ids, len(prompt_token_ids)
+        else:
+            inputs = prepare_inputs(
+                None,
+                self._processor,
+                images,
+                prompt_str,
+                image_token_index,
+                kwargs.get("resize_shape"),
+            )
+            input_ids = inputs[0]
+            return inputs, len(input_ids)
+
+    def chat(
+        self,
+        messages: List[Dict],
+        generate_config: Optional[MLXGenerateConfig] = None,
+    ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
+        messages = self._transform_messages(messages)  # type: ignore
+        tools = generate_config.pop("tools", []) if generate_config else None
+
+        model_family = self.model_family.model_family or self.model_family.model_name
+
+        if "internvl2" not in model_family.lower():
+            from qwen_vl_utils import process_vision_info
+
+            full_context_kwargs = {}
+            if tools and model_family in QWEN_TOOL_CALL_FAMILY:
+                full_context_kwargs["tools"] = tools
+            assert self.model_family.chat_template is not None
+            prompt = self.get_full_context(
+                messages, self.model_family.chat_template, **full_context_kwargs
+            )
+            images, video_inputs = process_vision_info(messages)
+            if video_inputs:
+                raise ValueError("Not support video input now.")
+        else:
+            prompt, images = self.get_specific_prompt(model_family, messages)  # type: ignore
+
+        if not images:
+            inputs = {
+                "prompt": prompt,
+            }
+        elif len(images) == 1:
+            inputs = {
+                "prompt": prompt,
+                "multi_modal_data": {"image": images[-1]},  # type: ignore
+            }
+        else:
+            inputs = {
+                "prompt": prompt,
+                "multi_modal_data": {"image": images},  # type: ignore
+            }
+        generate_config = self._sanitize_generate_config(generate_config)
+
+        stream = generate_config.get("stream", False)
+        if stream:
+            it = self.generate(inputs, generate_config)
+            assert isinstance(it, Iterator)
+            return self._to_chat_completion_chunks(it)
+        else:
+            c = self.generate(inputs, generate_config)
+            assert not isinstance(c, Iterator)
+            if tools:
+                return self._tool_calls_completion(self.model_family, self.model_uid, c)
+            return self._to_chat_completion(c)

xinference/model/llm/sglang/core.py

@@ -75,6 +75,7 @@ SGLANG_SUPPORTED_CHAT_MODELS = [
     "llama-2-chat",
     "llama-3-instruct",
     "llama-3.1-instruct",
+    "llama-3.3-instruct",
     "qwen-chat",
     "qwen1.5-chat",
     "qwen2-instruct",

xinference/model/llm/transformers/chatglm.py

@@ -61,7 +61,7 @@ class ChatglmPytorchChatModel(PytorchChatModel):
 
     def _load_model(self, **kwargs):
         try:
-            from transformers import AutoModel, AutoTokenizer
+            from transformers import AutoModelForCausalLM, AutoTokenizer
         except ImportError:
             error_message = "Failed to import module 'transformers'"
             installation_guide = [
@@ -77,7 +77,7 @@ class ChatglmPytorchChatModel(PytorchChatModel):
             encode_special_tokens=True,
             revision=kwargs["revision"],
         )
-        model = AutoModel.from_pretrained(
+        model = AutoModelForCausalLM.from_pretrained(
             self.model_path,
             **kwargs,
         )
@@ -232,9 +232,11 @@ class ChatglmPytorchChatModel(PytorchChatModel):
                     content = {
                         "name": function_name,
                         "arguments": json.dumps(
-                            arguments_json
-                            if isinstance(arguments_json, dict)
-                            else arguments,
+                            (
+                                arguments_json
+                                if isinstance(arguments_json, dict)
+                                else arguments
+                            ),
                             ensure_ascii=False,
                         ),
                     }
@@ -331,6 +333,8 @@ class ChatglmPytorchChatModel(PytorchChatModel):
         max_new_tokens = generate_config.get("max_tokens")
         if max_new_tokens is not None:
             kwargs["max_new_tokens"] = int(max_new_tokens)
+        else:
+            kwargs["max_new_tokens"] = 1024
         do_sample = generate_config.get("do_sample")
         if do_sample is not None:
             kwargs["do_sample"] = bool(do_sample)

xinference/model/llm/transformers/core.py

@@ -69,6 +69,7 @@ NON_DEFAULT_MODEL_LIST: List[str] = [
     "deepseek-v2.5",
     "deepseek-v2-chat-0628",
     "glm-edge-v",
+    "QvQ-72B-Preview",
 ]
 
 

xinference/model/llm/transformers/qwen2_vl.py

@@ -47,6 +47,8 @@ class Qwen2VLChatModel(PytorchChatModel):
         llm_family = model_family.model_family or model_family.model_name
         if "qwen2-vl-instruct".lower() in llm_family.lower():
             return True
+        if "qvq-72b-preview".lower() in llm_family.lower():
+            return True
         return False
 
     def load(self):

xinference/model/llm/transformers/utils.py

@@ -156,6 +156,7 @@ def _get_completion(
     finish_reason: Optional[str],
     model_uid: str,
     r: InferenceRequest,
+    completion_tokens: int,
 ):
     completion_choice = CompletionChoice(
         text=output, index=0, logprobs=None, finish_reason=finish_reason
@@ -170,8 +171,8 @@ def _get_completion(
     )
     completion_usage = CompletionUsage(
         prompt_tokens=len(r.prompt_tokens),
-        completion_tokens=len(r.new_tokens),
-        total_tokens=len(r.prompt_tokens) + len(r.new_tokens),
+        completion_tokens=completion_tokens,
+        total_tokens=len(r.prompt_tokens) + completion_tokens,
     )
     completion = Completion(
         id=completion_chunk["id"],
@@ -371,7 +372,7 @@ def _batch_inference_one_step_internal(
                 r.stopped = stopped
                 r.finish_reason = finish_reason
 
-            if r.stopped and r not in stop_token_mapping and r not in output_mapping:
+            if r.stopped and r not in stop_token_mapping:
                 stop_token_mapping[r] = _i + 1
 
             if r.stream:
@@ -446,12 +447,14 @@ def _batch_inference_one_step_internal(
             else:
                 # last round, handle non-stream result
                 if r.stopped and _i == decode_round - 1:
-                    invalid_token_num = decode_round - stop_token_mapping[r]
+                    invalid_token_num = (
+                        (decode_round - stop_token_mapping[r] + 1)
+                        if r.finish_reason == "stop"
+                        else (decode_round - stop_token_mapping[r])
+                    )
                     outputs = (
                         tokenizer.decode(
-                            r.new_tokens[: -(invalid_token_num + 1)]
-                            if r.finish_reason == "stop"
-                            else r.new_tokens[:-invalid_token_num],
+                            r.new_tokens[:-invalid_token_num],
                             skip_special_tokens=True,
                             spaces_between_special_tokens=False,
                             clean_up_tokenization_spaces=True,
@@ -460,7 +463,12 @@ def _batch_inference_one_step_internal(
                         else output_mapping[r]
                     )
                     completion = _get_completion(
-                        outputs, r.chunk_id, r.finish_reason, model_uid, r
+                        outputs,
+                        r.chunk_id,
+                        r.finish_reason,
+                        model_uid,
+                        r,
+                        len(r.new_tokens) - invalid_token_num,
                     )
                     r.completion = [completion]
 

xinference/model/llm/utils.py

@@ -52,6 +52,7 @@ QWEN_TOOL_CALL_FAMILY = [
     "qwen2-instruct",
     "qwen2-moe-instruct",
     "qwen2.5-instruct",
+    "qwen2.5-coder-instruct",
 ]
 
 GLM4_TOOL_CALL_FAMILY = [
@@ -324,7 +325,10 @@ class ChatModelMixin:
         """
         try:
             if isinstance(c, dict):
-                return [(None, c["name"], c["arguments"])]
+                try:
+                    return [(None, c["name"], json.loads(c["arguments"]))]
+                except Exception:
+                    return [(None, c["name"], c["arguments"])]
         except KeyError:
             logger.error("Can't parse glm output: %s", c)
             return [(str(c), None, None)]

xinference/model/llm/vllm/core.py

@@ -70,6 +70,7 @@ class VLLMModelConfig(TypedDict, total=False):
     max_model_len: Optional[int]
     limit_mm_per_prompt: Optional[Dict[str, int]]
     guided_decoding_backend: Optional[str]
+    scheduling_policy: Optional[str]
 
 
 class VLLMGenerateConfig(TypedDict, total=False):
@@ -86,6 +87,7 @@ class VLLMGenerateConfig(TypedDict, total=False):
     stop: Optional[Union[str, List[str]]]
     stream: bool  # non-sampling param, should not be passed to the engine.
     stream_options: Optional[Union[dict, None]]
+    skip_special_tokens: Optional[bool]
     response_format: Optional[dict]
     guided_json: Optional[Union[str, dict]]
     guided_regex: Optional[str]
@@ -181,14 +183,19 @@ if VLLM_INSTALLED and vllm.__version__ >= "0.5.3":
 if VLLM_INSTALLED and vllm.__version__ > "0.5.3":
     VLLM_SUPPORTED_MODELS.append("llama-3.1")
     VLLM_SUPPORTED_CHAT_MODELS.append("llama-3.1-instruct")
+    VLLM_SUPPORTED_CHAT_MODELS.append("llama-3.3-instruct")
 
 if VLLM_INSTALLED and vllm.__version__ >= "0.6.1":
     VLLM_SUPPORTED_VISION_MODEL_LIST.append("internvl2")
 
+if VLLM_INSTALLED and vllm.__version__ >= "0.6.2":
+    VLLM_SUPPORTED_CHAT_MODELS.append("minicpm3-4b")
+
 if VLLM_INSTALLED and vllm.__version__ >= "0.6.3":
     VLLM_SUPPORTED_MODELS.append("llama-3.2-vision")
     VLLM_SUPPORTED_VISION_MODEL_LIST.append("llama-3.2-vision-instruct")
     VLLM_SUPPORTED_VISION_MODEL_LIST.append("qwen2-vl-instruct")
+    VLLM_SUPPORTED_VISION_MODEL_LIST.append("QvQ-72B-Preview")
 
 
 class VLLMModel(LLM):
@@ -242,7 +249,6 @@ class VLLMModel(LLM):
             multiprocessing.set_start_method("fork", force=True)
 
         self._model_config = self._sanitize_model_config(self._model_config)
-
         if self.lora_modules is None:
             self.lora_requests = []
         else:
@@ -325,7 +331,9 @@ class VLLMModel(LLM):
         model_config.setdefault("quantization", None)
         model_config.setdefault("max_model_len", None)
         model_config.setdefault("guided_decoding_backend", "outlines")
-
+        # Add scheduling policy if vLLM version is 0.6.3 or higher
+        if vllm.__version__ >= "0.6.3":
+            model_config.setdefault("scheduling_policy", "fcfs")
         return model_config
 
     @staticmethod
@@ -373,6 +381,9 @@ class VLLMModel(LLM):
         sanitized.setdefault(
             "stream_options", generate_config.get("stream_options", None)
         )
+        sanitized.setdefault(
+            "skip_special_tokens", generate_config.get("skip_special_tokens", True)
+        )
         sanitized.setdefault(
             "guided_json", generate_config.get("guided_json", guided_json)
         )
@@ -854,6 +865,9 @@ class VLLMVisionModel(VLLMModel, ChatModelMixin):
                 "image": 2,  # default 2 images all chat
             }
         )
+        # Add scheduling policy if vLLM version is 0.6.3 or higher
+        if vllm.__version__ >= "0.6.3":
+            model_config.setdefault("scheduling_policy", "fcfs")
 
         return model_config