xinference 1.0.0__py3-none-any.whl → 1.1.0__py3-none-any.whl

xinference/model/llm/mlx/core.py

@@ -168,6 +168,9 @@ class MLXModel(LLM):
             return False
         if "generate" not in llm_family.model_ability:
             return False
+        if "chat" in llm_family.model_ability or "vision" in llm_family.model_ability:
+            # do not process chat or vision
+            return False
         return True
 
     def _get_prompt_cache(self, prompt, lora_name: Optional[str] = None):
@@ -191,18 +194,35 @@ class MLXModel(LLM):
         self._prompt_cache.tokens.extend(prompt)
         return prompt
 
-    def _generate_stream(self, prompt: str, kwargs: MLXGenerateConfig):
-        import mlx.core as mx
-        from mlx_lm.utils import generate_step
+    def _generate_stream_inner(self, **kwargs):
+        from mlx_lm.utils import make_sampler, stream_generate
+
+        sampler = make_sampler(
+            temp=kwargs.pop("temperature"), top_p=kwargs.pop("top_p")
+        )
+        prompt_token_ids = kwargs.pop("prompt_token_ids")
+        yield from stream_generate(
+            self._model, self._tokenizer, prompt_token_ids, sampler=sampler, **kwargs
+        )
+
+    def _prepare_inputs(
+        self, prompt: Union[str, Dict[str, Any]], kwargs
+    ) -> Tuple[Any, int]:
+        prompt_token_ids = self._tokenizer.encode(prompt)
+        prompt_token_ids = self._get_prompt_cache(
+            prompt_token_ids, kwargs.get("lora_name")
+        )
+        return prompt_token_ids, len(prompt_token_ids)
 
-        model = self._model
+    def _generate_stream(
+        self, prompt: Union[str, Dict[str, Any]], kwargs: MLXGenerateConfig
+    ):
         model_uid = self.model_uid
         tokenizer = self._tokenizer
         max_tokens = kwargs["max_tokens"]
         chunk_id = str(uuid.uuid4())
         stop_token_ids = kwargs.get("stop_token_ids", [])
         stream = kwargs.get("stream", False)
-        lora_name = kwargs.get("lora_name")
         stream_options = kwargs.pop("stream_options", None)
         include_usage = (
             stream_options["include_usage"]
@@ -210,40 +230,28 @@ class MLXModel(LLM):
             else False
         )
 
-        prompt_token_ids = tokenizer.encode(prompt)
-        prompt_token_ids = self._get_prompt_cache(prompt_token_ids, lora_name)
-        prompt_tokens = mx.array(prompt_token_ids)
-        input_echo_len = len(prompt_tokens)
+        prompt_token_ids, input_echo_len = self._prepare_inputs(prompt, kwargs)
 
         i = 0
         start = time.time()
         output = ""
         tokens = []
-        for (token, _), i in zip(
-            generate_step(
-                prompt_tokens,
-                model,
-                temp=kwargs["temperature"],
+        for chunk_resp, i in zip(
+            self._generate_stream_inner(
+                prompt_token_ids=prompt_token_ids,
+                max_tokens=max_tokens,
+                temperature=kwargs["temperature"],
+                top_p=kwargs["top_p"],
                 repetition_penalty=kwargs["repetition_penalty"],
                 repetition_context_size=kwargs["repetition_context_size"],
-                top_p=kwargs["top_p"],
-                logit_bias=kwargs["logit_bias"],
-                prompt_cache=self._prompt_cache.cache,  # type: ignore
+                prompt_cache=self._prompt_cache.cache if self._prompt_cache else None,  # type: ignore
             ),
             range(max_tokens),
         ):
+            token = chunk_resp.token
             tokens.append(token)
-            if token == tokenizer.eos_token_id or token in stop_token_ids:  # type: ignore
-                break
-
-            # Yield the last segment if streaming
-            out = tokenizer.decode(
-                token,
-                skip_special_tokens=True,
-                spaces_between_special_tokens=False,
-                clean_up_tokenization_spaces=True,
-            )
 
+            out = chunk_resp.text
             if stream:
                 # this special character is mainly for qwen
                 out = out.strip("�")
@@ -267,11 +275,15 @@ class MLXModel(LLM):
                 total_tokens=(input_echo_len + i),
             ), completion_usage
 
+            if token == tokenizer.eos_token_id or token in stop_token_ids:  # type: ignore
+                break
+
         logger.info(
             f"Average generation speed: {i / (time.time() - start):.2f} tokens/s."
         )
 
-        self._prompt_cache.tokens.extend(tokens)  # type: ignore
+        if self._prompt_cache:
+            self._prompt_cache.tokens.extend(tokens)  # type: ignore
 
         if i == max_tokens - 1:
             finish_reason = "length"
@@ -315,10 +327,12 @@ class MLXModel(LLM):
             yield completion_chunk, completion_usage
 
     def generate(
-        self, prompt: str, generate_config: Optional[MLXGenerateConfig] = None
+        self,
+        prompt: Union[str, Dict[str, Any]],
+        generate_config: Optional[MLXGenerateConfig] = None,
     ) -> Union[Completion, Iterator[CompletionChunk]]:
         def generator_wrapper(
-            prompt: str, generate_config: MLXGenerateConfig
+            prompt: Union[str, Dict[str, Any]], generate_config: MLXGenerateConfig
         ) -> Iterator[CompletionChunk]:
             for completion_chunk, completion_usage in self._generate_stream(
                 prompt,
@@ -357,26 +371,6 @@ class MLXModel(LLM):
 
 
 class MLXChatModel(MLXModel, ChatModelMixin):
-    def __init__(
-        self,
-        model_uid: str,
-        model_family: "LLMFamilyV1",
-        model_spec: "LLMSpecV1",
-        quantization: str,
-        model_path: str,
-        model_config: Optional[MLXModelConfig] = None,
-        peft_model: Optional[List[LoRA]] = None,
-    ):
-        super().__init__(
-            model_uid,
-            model_family,
-            model_spec,
-            quantization,
-            model_path,
-            model_config,
-            peft_model,
-        )
-
     def _sanitize_generate_config(
         self,
         generate_config: Optional[MLXGenerateConfig],
@@ -403,6 +397,9 @@ class MLXChatModel(MLXModel, ChatModelMixin):
             return False
         if "chat" not in llm_family.model_ability:
             return False
+        if "vision" in llm_family.model_ability:
+            # do not process vision
+            return False
         return True
 
     def chat(
@@ -433,3 +430,186 @@ class MLXChatModel(MLXModel, ChatModelMixin):
             if tools:
                 return self._tool_calls_completion(self.model_family, self.model_uid, c)
             return self._to_chat_completion(c)
+
+
+class MLXVisionModel(MLXModel, ChatModelMixin):
+    @classmethod
+    def match(
+        cls, llm_family: "LLMFamilyV1", llm_spec: "LLMSpecV1", quantization: str
+    ) -> bool:
+        if llm_spec.model_format not in ["mlx"]:
+            return False
+        if sys.platform != "darwin" or platform.processor() != "arm":
+            # only work for Mac M chips
+            return False
+        if "vision" not in llm_family.model_ability:
+            return False
+        return True
+
+    def _load_model(self, **kwargs):
+        try:
+            from mlx_vlm import load
+        except ImportError:
+            error_message = "Failed to import module 'mlx_vlm'"
+            installation_guide = [
+                "Please make sure 'mlx_vlm' is installed. ",
+                "You can install it by `pip install mlx_vlm`\n",
+            ]
+
+            raise ImportError(f"{error_message}\n\n{''.join(installation_guide)}")
+
+        return load(self.model_path)
+
+    def load(self):
+        kwargs = {}
+        kwargs["revision"] = self._model_config.get(
+            "revision", self.model_spec.model_revision
+        )
+        kwargs["trust_remote_code"] = self._model_config.get("trust_remote_code")
+        kwargs["cache_limit_gb"] = self._model_config.pop("cache_limit_gb", None)
+
+        self._model, self._processor = self._load_model(**kwargs)
+        self._tokenizer = self._processor.tokenizer
+
+    def _generate_stream_inner(self, **kwargs):
+        import mlx.core as mx
+        from mlx_lm.utils import GenerationResponse
+        from mlx_vlm.utils import generate_step
+
+        max_tokens = kwargs.pop("max_tokens")
+        inputs = kwargs["prompt_token_ids"]
+        input_ids, pixel_values, mask = inputs[:3]
+
+        kwargs = {
+            k: v
+            for k, v in zip(
+                [
+                    "image_grid_thw",
+                    "image_sizes",
+                    "aspect_ratio_ids",
+                    "aspect_ratio_mask",
+                    "cross_attention_mask",
+                ],
+                inputs[3:],
+            )
+        }
+
+        tokenizer = self._processor.tokenizer
+        detokenizer = self._processor.detokenizer
+
+        detokenizer.reset()
+        tic = time.perf_counter()
+        for (token, logprobs), n in zip(
+            generate_step(input_ids, self._model, pixel_values, mask, **kwargs),
+            range(max_tokens),
+        ):
+            if n == 0:
+                prompt_time = time.perf_counter() - tic
+                prompt_tps = len(input_ids) / prompt_time
+                tic = time.perf_counter()
+            if token == tokenizer.eos_token_id:
+                break
+            detokenizer.add_token(token)
+
+            # Yield the last segment if streaming
+            yield GenerationResponse(
+                text=detokenizer.last_segment,
+                token=token,
+                logprobs=logprobs,
+                prompt_tokens=len(input_ids),
+                prompt_tps=prompt_tps,
+                generation_tokens=n + 1,
+                generation_tps=(n + 1) / (time.perf_counter() - tic),
+                peak_memory=mx.metal.get_peak_memory() / 1e9,
+            )
+
+        detokenizer.finalize()
+        yield GenerationResponse(
+            text=detokenizer.last_segment,
+            token=token,
+            logprobs=logprobs,
+            prompt_tokens=len(input_ids),
+            prompt_tps=prompt_tps,
+            generation_tokens=n + 1,
+            generation_tps=(n + 1) / (time.perf_counter() - tic),
+            peak_memory=mx.metal.get_peak_memory() / 1e9,
+        )
+
+    def _prepare_inputs(
+        self, prompt: Union[str, Dict[str, Any]], kwargs
+    ) -> Tuple[Any, int]:
+        from mlx_vlm import prepare_inputs
+
+        prompt_str = prompt.get("prompt")  # type: ignore
+        images = prompt.get("multi_modal_data", {}).get("image")  # type: ignore
+        if images and not isinstance(images, list):
+            images = [images]
+        if hasattr(self._model.config, "image_token_index"):
+            image_token_index = self._model.config.image_token_index
+        else:
+            image_token_index = None
+
+        inputs = prepare_inputs(
+            None,
+            self._processor,
+            images,
+            prompt_str,
+            image_token_index,
+            kwargs.get("resize_shape"),
+        )
+        input_ids = inputs[0]
+        return inputs, len(input_ids)
+
+    def chat(
+        self,
+        messages: List[Dict],
+        generate_config: Optional[MLXGenerateConfig] = None,
+    ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
+        messages = self._transform_messages(messages)  # type: ignore
+        tools = generate_config.pop("tools", []) if generate_config else None
+
+        model_family = self.model_family.model_family or self.model_family.model_name
+
+        if "internvl2" not in model_family.lower():
+            from qwen_vl_utils import process_vision_info
+
+            full_context_kwargs = {}
+            if tools and model_family in QWEN_TOOL_CALL_FAMILY:
+                full_context_kwargs["tools"] = tools
+            assert self.model_family.chat_template is not None
+            prompt = self.get_full_context(
+                messages, self.model_family.chat_template, **full_context_kwargs
+            )
+            images, video_inputs = process_vision_info(messages)
+            if video_inputs:
+                raise ValueError("Not support video input now.")
+        else:
+            prompt, images = self.get_specific_prompt(model_family, messages)  # type: ignore
+
+        if not images:
+            inputs = {
+                "prompt": prompt,
+            }
+        elif len(images) == 1:
+            inputs = {
+                "prompt": prompt,
+                "multi_modal_data": {"image": images[-1]},  # type: ignore
+            }
+        else:
+            inputs = {
+                "prompt": prompt,
+                "multi_modal_data": {"image": images},  # type: ignore
+            }
+        generate_config = self._sanitize_generate_config(generate_config)
+
+        stream = generate_config.get("stream", False)
+        if stream:
+            it = self.generate(inputs, generate_config)
+            assert isinstance(it, Iterator)
+            return self._to_chat_completion_chunks(it)
+        else:
+            c = self.generate(inputs, generate_config)
+            assert not isinstance(c, Iterator)
+            if tools:
+                return self._tool_calls_completion(self.model_family, self.model_uid, c)
+            return self._to_chat_completion(c)

xinference/model/llm/sglang/core.py

@@ -75,6 +75,7 @@ SGLANG_SUPPORTED_CHAT_MODELS = [
     "llama-2-chat",
     "llama-3-instruct",
     "llama-3.1-instruct",
+    "llama-3.3-instruct",
     "qwen-chat",
     "qwen1.5-chat",
     "qwen2-instruct",
@@ -89,6 +90,7 @@ SGLANG_SUPPORTED_CHAT_MODELS = [
     "deepseek-v2-chat-0628",
     "qwen2.5-instruct",
     "qwen2.5-coder-instruct",
+    "QwQ-32B-Preview",
 ]
 
 

xinference/model/llm/transformers/chatglm.py

@@ -61,7 +61,7 @@ class ChatglmPytorchChatModel(PytorchChatModel):
 
     def _load_model(self, **kwargs):
         try:
-            from transformers import AutoModel, AutoTokenizer
+            from transformers import AutoModelForCausalLM, AutoTokenizer
         except ImportError:
             error_message = "Failed to import module 'transformers'"
             installation_guide = [
@@ -77,7 +77,7 @@ class ChatglmPytorchChatModel(PytorchChatModel):
             encode_special_tokens=True,
             revision=kwargs["revision"],
         )
-        model = AutoModel.from_pretrained(
+        model = AutoModelForCausalLM.from_pretrained(
             self.model_path,
             **kwargs,
         )
@@ -232,9 +232,11 @@ class ChatglmPytorchChatModel(PytorchChatModel):
                     content = {
                         "name": function_name,
                         "arguments": json.dumps(
-                            arguments_json
-                            if isinstance(arguments_json, dict)
-                            else arguments,
+                            (
+                                arguments_json
+                                if isinstance(arguments_json, dict)
+                                else arguments
+                            ),
                             ensure_ascii=False,
                         ),
                     }
@@ -331,6 +333,8 @@ class ChatglmPytorchChatModel(PytorchChatModel):
         max_new_tokens = generate_config.get("max_tokens")
         if max_new_tokens is not None:
             kwargs["max_new_tokens"] = int(max_new_tokens)
+        else:
+            kwargs["max_new_tokens"] = 1024
         do_sample = generate_config.get("do_sample")
         if do_sample is not None:
             kwargs["do_sample"] = bool(do_sample)

xinference/model/llm/transformers/core.py

@@ -68,6 +68,7 @@ NON_DEFAULT_MODEL_LIST: List[str] = [
     "deepseek-v2-chat",
     "deepseek-v2.5",
     "deepseek-v2-chat-0628",
+    "glm-edge-v",
 ]
 
 

xinference/model/llm/transformers/glm_edge_v.py

@@ -0,0 +1,230 @@
+# Copyright 2022-2023 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+from threading import Thread
+from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
+
+import torch
+
+from ....types import ChatCompletion, ChatCompletionChunk, CompletionChunk
+from ...utils import select_device
+from ..llm_family import LLMFamilyV1, LLMSpecV1
+from ..utils import (
+    _decode_image_without_rgb,
+    generate_chat_completion,
+    generate_completion_chunk,
+)
+from .core import PytorchChatModel, PytorchGenerateConfig
+from .utils import cache_clean
+
+logger = logging.getLogger(__name__)
+
+
+class GlmEdgeVModel(PytorchChatModel):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._device = None
+        self._tokenizer = None
+        self._model = None
+        self._processor = None
+
+    @classmethod
+    def match(
+        cls, model_family: "LLMFamilyV1", model_spec: "LLMSpecV1", quantization: str
+    ) -> bool:
+        family = model_family.model_family or model_family.model_name
+        if "glm-edge-v" in family.lower():
+            return True
+        return False
+
+    def load(self):
+        from transformers import AutoImageProcessor, AutoModelForCausalLM, AutoTokenizer
+
+        device = self._pytorch_model_config.get("device", "auto")
+        self._device = select_device(device)
+
+        kwargs = {"device_map": self._device}
+        quantization = self.quantization
+
+        # referenced from PytorchModel.load
+        if quantization != "none":
+            if self._device == "cuda" and self._is_linux():
+                kwargs["device_map"] = "auto"
+                if quantization == "4-bit":
+                    kwargs["load_in_4bit"] = True
+                elif quantization == "8-bit":
+                    kwargs["load_in_8bit"] = True
+                else:
+                    raise ValueError(
+                        f"Quantization {quantization} is not supported in temporary"
+                    )
+            else:
+                if quantization != "8-bit":
+                    raise ValueError(
+                        f"Only 8-bit quantization is supported if it is not linux system or cuda device"
+                    )
+
+        processor = AutoImageProcessor.from_pretrained(
+            self.model_path, trust_remote_code=True
+        )
+        self._processor = processor
+
+        model = AutoModelForCausalLM.from_pretrained(
+            self.model_path,
+            trust_remote_code=True,
+            torch_dtype=torch.bfloat16,
+            device_map="auto",
+        )
+
+        self._model = model
+
+        tokenizer = AutoTokenizer.from_pretrained(
+            self.model_path, trust_remote_code=True
+        )
+        self._tokenizer = tokenizer
+
+    @staticmethod
+    def _get_processed_msgs(
+        messages: List[Dict],
+    ) -> Tuple[List[Dict[str, Any]], List[Any]]:
+        res = []
+        img = []
+        for message in messages:
+            role = message["role"]
+            content = message["content"]
+            if isinstance(content, str):
+                res.append({"role": role, "content": content})
+            else:
+                texts = []
+                image_urls = []
+                for c in content:
+                    c_type = c.get("type")
+                    if c_type == "text":
+                        texts.append(c["text"])
+                    else:
+                        assert (
+                            c_type == "image_url"
+                        ), "Please follow the image input of the OpenAI API."
+                        image_urls.append(c["image_url"]["url"])
+                if len(image_urls) > 1:
+                    raise RuntimeError("Only one image per message is supported")
+                image_futures = []
+                with ThreadPoolExecutor() as executor:
+                    for image_url in image_urls:
+                        fut = executor.submit(_decode_image_without_rgb, image_url)
+                        image_futures.append(fut)
+                images = [fut.result() for fut in image_futures]
+                assert len(images) <= 1
+                text = " ".join(texts)
+                img.extend(images)
+                if images:
+                    res.append(
+                        {
+                            "role": role,
+                            "content": [
+                                {"type": "image"},
+                                {"type": "text", "text": text},
+                            ],
+                        }
+                    )
+                else:
+                    res.append({"role": role, "content": text})
+        return res, img
+
+    @cache_clean
+    def chat(
+        self,
+        messages: List[Dict],
+        generate_config: Optional[PytorchGenerateConfig] = None,
+    ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
+        from transformers import TextIteratorStreamer
+
+        if not generate_config:
+            generate_config = {}
+
+        stream = generate_config.get("stream", False)
+        msgs, imgs = self._get_processed_msgs(messages)
+
+        inputs = self._tokenizer.apply_chat_template(
+            msgs,
+            add_generation_prompt=True,
+            tokenize=True,
+            return_tensors="pt",
+            return_dict=True,
+        )  # chat mode
+        inputs = inputs.to(self._model.device)
+
+        generate_kwargs = {
+            **inputs,
+        }
+        if len(imgs) > 0:
+            generate_kwargs["pixel_values"] = torch.tensor(
+                self._processor(imgs[-1]).pixel_values
+            ).to(self._model.device)
+        stop_str = "<|endoftext|>"
+
+        if stream:
+            streamer = TextIteratorStreamer(
+                tokenizer=self._tokenizer,
+                timeout=60,
+                skip_prompt=True,
+                skip_special_tokens=True,
+            )
+            generate_kwargs = {
+                **generate_kwargs,
+                "streamer": streamer,
+            }
+            t = Thread(target=self._model.generate, kwargs=generate_kwargs)
+            t.start()
+
+            it = self.chat_stream(streamer, stop_str)
+            return self._to_chat_completion_chunks(it)
+        else:
+            with torch.no_grad():
+                outputs = self._model.generate(**generate_kwargs)
+                outputs = outputs[0][len(inputs["input_ids"][0]) :]
+                response = self._tokenizer.decode(outputs)
+                if response.endswith(stop_str):
+                    response = response[: -len(stop_str)]
+            return generate_chat_completion(self.model_uid, response)
+
+    def chat_stream(self, streamer, stop_str) -> Iterator[CompletionChunk]:
+        completion_id = str(uuid.uuid1())
+        for new_text in streamer:
+            if not new_text.endswith(stop_str):
+                yield generate_completion_chunk(
+                    chunk_text=new_text,
+                    finish_reason=None,
+                    chunk_id=completion_id,
+                    model_uid=self.model_uid,
+                    prompt_tokens=-1,
+                    completion_tokens=-1,
+                    total_tokens=-1,
+                    has_choice=True,
+                    has_content=True,
+                )
+
+        yield generate_completion_chunk(
+            chunk_text=None,
+            finish_reason="stop",
+            chunk_id=completion_id,
+            model_uid=self.model_uid,
+            prompt_tokens=-1,
+            completion_tokens=-1,
+            total_tokens=-1,
+            has_choice=True,
+            has_content=False,
+        )