xinference 1.0.1__py3-none-any.whl → 1.1.0__py3-none-any.whl

xinference/model/llm/llm_family_modelscope.json

@@ -454,6 +454,72 @@
       }
     ]
   },
+  {
+    "version": 1,
+    "context_length": 131072,
+    "model_name": "llama-3.3-instruct",
+    "model_lang": [
+      "en",
+      "de",
+      "fr",
+      "it",
+      "pt",
+      "hi",
+      "es",
+      "th"
+    ],
+    "model_ability": [
+      "chat",
+      "tools"
+    ],
+    "model_description": "The Llama 3.3 instruction tuned models are optimized for dialogue use cases and outperform many of the available open source chat models on common industry benchmarks..",
+    "model_specs": [
+      {
+        "model_format": "pytorch",
+        "model_size_in_billions": 70,
+        "quantizations": [
+          "none"
+        ],
+        "model_id": "LLM-Research/Llama-3.3-70B-Instruct",
+        "model_hub": "modelscope"
+      },
+      {
+        "model_format": "ggufv2",
+        "model_size_in_billions": 70,
+        "quantizations": [
+          "Q3_K_L",
+          "Q4_K_M",
+          "Q6_K",
+          "Q8_0"
+        ],
+        "quantization_parts": {
+          "Q6_K": [
+            "00001-of-00002",
+            "00002-of-00002"
+          ],
+          "Q8_0": [
+            "00001-of-00002",
+            "00002-of-00002"
+          ]
+        },
+        "model_id": "lmstudio-community/Llama-3.3-70B-Instruct-GGUF",
+        "model_file_name_template": "Llama-3.3-70B-Instruct-{quantization}.gguf",
+        "model_file_name_split_template": "Llama-3.3-70B-Instruct-{quantization}-{part}.gguf",
+        "model_hub": "modelscope"
+      }
+    ],
+    "chat_template": "{{- bos_token }}\n{%- if custom_tools is defined %}\n    {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n    {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n    {%- set date_string = \"26 Jul 2024\" %}\n{%- endif %}\n{%- if not tools is defined %}\n    {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content']|trim %}\n    {%- set messages = messages[1:] %}\n{%- else %}\n    {%- set system_message = \"\" %}\n{%- endif %}\n\n{#- System message + builtin tools #}\n{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if builtin_tools is defined or tools is not none %}\n    {{- \"Environment: ipython\\n\" }}\n{%- endif %}\n{%- if builtin_tools is defined %}\n    {{- \"Tools: \" + builtin_tools | reject('equalto', 'code_interpreter') | join(\", \") + \"\\n\\n\"}}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\" }}\n{{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n{%- if tools is not none and not tools_in_user_message %}\n    {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n    {#- Extract the first user message so we can plug it in here #}\n    {%- if messages | length != 0 %}\n        {%- set first_user_message = messages[0]['content']|trim %}\n        {%- set messages = messages[1:] %}\n    {%- else %}\n        {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n    {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n    {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n    {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n    {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n    {%- elif 'tool_calls' in message %}\n        {%- if not message.tool_calls|length == 1 %}\n            {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n        {%- endif %}\n        {%- set tool_call = message.tool_calls[0].function %}\n        {%- if builtin_tools is defined and tool_call.name in builtin_tools %}\n            {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n            {{- \"<|python_tag|>\" + tool_call.name + \".call(\" }}\n            {%- for arg_name, arg_val in tool_call.arguments | items %}\n                {{- arg_name + '=\"' + arg_val + '\"' }}\n                {%- if not loop.last %}\n                    {{- \", \" }}\n                {%- endif %}\n                {%- endfor %}\n            {{- \")\" }}\n        {%- else  %}\n            {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n            {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n            {{- '\"parameters\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- \"}\" }}\n        {%- endif %}\n        {%- if builtin_tools is defined %}\n            {#- This means we're in ipython mode #}\n            {{- \"<|eom_id|>\" }}\n        {%- else %}\n            {{- \"<|eot_id|>\" }}\n        {%- endif %}\n    {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n        {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n        {%- if message.content is mapping or message.content is iterable %}\n            {{- message.content | tojson }}\n        {%- else %}\n            {{- message.content }}\n        {%- endif %}\n        {{- \"<|eot_id|>\" }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n",
+    "stop_token_ids": [
+      128001,
+      128008,
+      128009
+    ],
+    "stop": [
+      "<|end_of_text|>",
+      "<|eot_id|>",
+      "<|eom_id|>"
+    ]
+  },
   {
     "version": 1,
     "context_length": 2048,
@@ -586,7 +652,7 @@
           "none"
         ],
         "model_hub": "modelscope",
-        "model_id": "ZhipuAI/glm-4-9b-chat",
+        "model_id": "ZhipuAI/glm-4-9b-chat-hf",
         "model_revision": "master"
       },
       {
@@ -652,7 +718,7 @@
           "none"
         ],
         "model_hub": "modelscope",
-        "model_id": "ZhipuAI/glm-4-9b-chat-1m",
+        "model_id": "ZhipuAI/glm-4-9b-chat-1m-hf",
         "model_revision": "master"
       },
       {
@@ -2837,7 +2903,7 @@
         "model_format": "mlx",
         "model_size_in_billions": "0_5",
         "quantizations": [
-          "4-bit"
+          "4bit"
         ],
         "model_id": "qwen/Qwen2-0.5B-Instruct-MLX",
         "model_hub": "modelscope"
@@ -2846,7 +2912,7 @@
         "model_format": "mlx",
         "model_size_in_billions": "1_5",
         "quantizations": [
-          "4-bit"
+          "4bit"
         ],
         "model_id": "qwen/Qwen2-1.5B-Instruct-MLX",
         "model_hub": "modelscope"
@@ -2855,7 +2921,7 @@
         "model_format": "mlx",
         "model_size_in_billions": 7,
         "quantizations": [
-          "4-bit"
+          "4bit"
         ],
         "model_id": "qwen/Qwen2-7B-Instruct-MLX",
         "model_hub": "modelscope"
@@ -4649,6 +4715,16 @@
         "model_id":"qwen/Qwen2-VL-7B-Instruct-AWQ",
         "model_revision":"master"
       },
+      {
+        "model_format":"mlx",
+        "model_size_in_billions":7,
+        "quantizations":[
+          "8bit"
+        ],
+        "model_hub": "modelscope",
+        "model_id":"okwinds/Qwen2-VL-7B-Instruct-MLX-8bit",
+        "model_revision":"master"
+      },
       {
         "model_format":"pytorch",
         "model_size_in_billions":2,
@@ -4689,6 +4765,16 @@
         "model_id":"qwen/Qwen2-VL-2B-Instruct-AWQ",
         "model_revision":"master"
       },
+      {
+        "model_format":"mlx",
+        "model_size_in_billions":2,
+        "quantizations":[
+          "8bit"
+        ],
+        "model_hub": "modelscope",
+        "model_id":"okwinds/Qwen2-VL-2B-Instruct-MLX-8bit",
+        "model_revision":"master"
+      },
       {
         "model_format":"pytorch",
         "model_size_in_billions":72,
@@ -4716,6 +4802,17 @@
         ],
         "model_id":"qwen/Qwen2-VL-72B-Instruct-GPTQ-{quantization}",
         "model_hub": "modelscope"
+      },
+      {
+        "model_format":"mlx",
+        "model_size_in_billions":72,
+        "quantizations":[
+          "4bit",
+          "8bit"
+        ],
+        "model_hub": "modelscope",
+        "model_id":"okwinds/Qwen2-VL-72B-Instruct-MLX-{quantization}",
+        "model_revision":"master"
       }
     ],
     "chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}",
@@ -5777,7 +5874,7 @@
         "model_format": "mlx",
         "model_size_in_billions": 3,
         "quantizations": [
-          "4-bit"
+          "4bit"
         ],
         "model_id": "okwinds/Qwen2.5-3B-Instruct-MLX-4bit",
         "model_hub": "modelscope"
@@ -5786,7 +5883,7 @@
         "model_format": "mlx",
         "model_size_in_billions": 3,
         "quantizations": [
-          "8-bit"
+          "8bit"
         ],
         "model_id": "okwinds/Qwen2.5-3B-Instruct-MLX-8bit",
         "model_hub": "modelscope"
@@ -5795,7 +5892,7 @@
         "model_format": "mlx",
         "model_size_in_billions": 7,
         "quantizations": [
-          "4-bit"
+          "4bit"
         ],
         "model_id": "okwinds/Qwen2.5-7B-Instruct-MLX-4bit",
         "model_hub": "modelscope"
@@ -5804,7 +5901,7 @@
         "model_format": "mlx",
         "model_size_in_billions": 7,
         "quantizations": [
-          "8-bit"
+          "8bit"
         ],
         "model_id": "okwinds/Qwen2.5-7B-Instruct-MLX-8bit",
         "model_hub": "modelscope"
@@ -5813,7 +5910,7 @@
         "model_format": "mlx",
         "model_size_in_billions": 14,
         "quantizations": [
-          "4-bit"
+          "4bit"
         ],
         "model_id": "okwinds/Qwen2.5-14B-Instruct-MLX-4bit",
         "model_hub": "modelscope"
@@ -5822,7 +5919,7 @@
         "model_format": "mlx",
         "model_size_in_billions": 14,
         "quantizations": [
-          "8-bit"
+          "8bit"
         ],
         "model_id": "okwinds/Qwen2.5-14B-Instruct-MLX-8bit",
         "model_hub": "modelscope"
@@ -5831,7 +5928,7 @@
         "model_format": "mlx",
         "model_size_in_billions": 32,
         "quantizations": [
-          "2-bit"
+          "2bit"
         ],
         "model_id": "okwinds/Qwen2.5-32B-Instruct-MLX-2bit",
         "model_hub": "modelscope"
@@ -5840,7 +5937,7 @@
         "model_format": "mlx",
         "model_size_in_billions": 32,
         "quantizations": [
-          "4-bit"
+          "4bit"
         ],
         "model_id": "okwinds/Qwen2.5-32B-Instruct-MLX-4bit",
         "model_hub": "modelscope"
@@ -5849,7 +5946,7 @@
         "model_format": "mlx",
         "model_size_in_billions": 32,
         "quantizations": [
-          "8-bit"
+          "8bit"
         ],
         "model_id": "okwinds/Qwen2.5-32B-Instruct-MLX-8bit",
         "model_hub": "modelscope"
@@ -5858,7 +5955,7 @@
         "model_format": "mlx",
         "model_size_in_billions": 72,
         "quantizations": [
-          "2-bit"
+          "2bit"
         ],
         "model_id": "okwinds/Qwen2.5-32B-Instruct-MLX-2bit",
         "model_hub": "modelscope"
@@ -5867,7 +5964,7 @@
         "model_format": "mlx",
         "model_size_in_billions": 72,
         "quantizations": [
-          "4-bit"
+          "4bit"
         ],
         "model_id": "okwinds/Qwen2.5-72B-Instruct-MLX-4bit",
         "model_hub": "modelscope"
@@ -5876,7 +5973,7 @@
         "model_format": "mlx",
         "model_size_in_billions": 72,
         "quantizations": [
-          "8-bit"
+          "8bit"
         ],
         "model_id": "okwinds/Qwen2.5-72B-Instruct-MLX-8bit",
         "model_hub": "modelscope"
@@ -6296,7 +6393,7 @@
         "model_format": "mlx",
         "model_size_in_billions": 32,
         "quantizations": [
-          "4-bit"
+          "4bit"
         ],
         "model_id": "okwinds/QwQ-32B-Preview-MLX-4bit",
         "model_hub": "modelscope"
@@ -6305,7 +6402,7 @@
         "model_format": "mlx",
         "model_size_in_billions": 32,
         "quantizations": [
-          "8-bit"
+          "8bit"
         ],
         "model_id": "okwinds/QwQ-32B-Preview-MLX-8bit",
         "model_hub": "modelscope"
@@ -6320,7 +6417,8 @@
           "Q8_0"
         ],
         "model_id": "AI-ModelScope/QwQ-32B-Preview-GGUF",
-        "model_file_name_template": "QwQ-32B-Preview-{quantization}.gguf"
+        "model_file_name_template": "QwQ-32B-Preview-{quantization}.gguf",
+	"model_hub": "modelscope"
       }
     ],
     "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",

xinference/model/llm/mlx/core.py

@@ -168,6 +168,9 @@ class MLXModel(LLM):
             return False
         if "generate" not in llm_family.model_ability:
             return False
+        if "chat" in llm_family.model_ability or "vision" in llm_family.model_ability:
+            # do not process chat or vision
+            return False
         return True
 
     def _get_prompt_cache(self, prompt, lora_name: Optional[str] = None):
@@ -191,18 +194,35 @@ class MLXModel(LLM):
         self._prompt_cache.tokens.extend(prompt)
         return prompt
 
-    def _generate_stream(self, prompt: str, kwargs: MLXGenerateConfig):
-        import mlx.core as mx
-        from mlx_lm.utils import generate_step
+    def _generate_stream_inner(self, **kwargs):
+        from mlx_lm.utils import make_sampler, stream_generate
+
+        sampler = make_sampler(
+            temp=kwargs.pop("temperature"), top_p=kwargs.pop("top_p")
+        )
+        prompt_token_ids = kwargs.pop("prompt_token_ids")
+        yield from stream_generate(
+            self._model, self._tokenizer, prompt_token_ids, sampler=sampler, **kwargs
+        )
+
+    def _prepare_inputs(
+        self, prompt: Union[str, Dict[str, Any]], kwargs
+    ) -> Tuple[Any, int]:
+        prompt_token_ids = self._tokenizer.encode(prompt)
+        prompt_token_ids = self._get_prompt_cache(
+            prompt_token_ids, kwargs.get("lora_name")
+        )
+        return prompt_token_ids, len(prompt_token_ids)
 
-        model = self._model
+    def _generate_stream(
+        self, prompt: Union[str, Dict[str, Any]], kwargs: MLXGenerateConfig
+    ):
         model_uid = self.model_uid
         tokenizer = self._tokenizer
         max_tokens = kwargs["max_tokens"]
         chunk_id = str(uuid.uuid4())
         stop_token_ids = kwargs.get("stop_token_ids", [])
         stream = kwargs.get("stream", False)
-        lora_name = kwargs.get("lora_name")
         stream_options = kwargs.pop("stream_options", None)
         include_usage = (
             stream_options["include_usage"]
@@ -210,39 +230,28 @@ class MLXModel(LLM):
             else False
         )
 
-        prompt_token_ids = tokenizer.encode(prompt)
-        prompt_token_ids = self._get_prompt_cache(prompt_token_ids, lora_name)
-        prompt_tokens = mx.array(prompt_token_ids)
-        input_echo_len = len(prompt_tokens)
+        prompt_token_ids, input_echo_len = self._prepare_inputs(prompt, kwargs)
 
         i = 0
         start = time.time()
         output = ""
         tokens = []
-        for (token, _), i in zip(
-            generate_step(
-                prompt_tokens,
-                model,
-                temp=kwargs["temperature"],
+        for chunk_resp, i in zip(
+            self._generate_stream_inner(
+                prompt_token_ids=prompt_token_ids,
+                max_tokens=max_tokens,
+                temperature=kwargs["temperature"],
+                top_p=kwargs["top_p"],
                 repetition_penalty=kwargs["repetition_penalty"],
                 repetition_context_size=kwargs["repetition_context_size"],
-                top_p=kwargs["top_p"],
-                prompt_cache=self._prompt_cache.cache,  # type: ignore
+                prompt_cache=self._prompt_cache.cache if self._prompt_cache else None,  # type: ignore
             ),
             range(max_tokens),
         ):
+            token = chunk_resp.token
             tokens.append(token)
-            if token == tokenizer.eos_token_id or token in stop_token_ids:  # type: ignore
-                break
-
-            # Yield the last segment if streaming
-            out = tokenizer.decode(
-                token,
-                skip_special_tokens=True,
-                spaces_between_special_tokens=False,
-                clean_up_tokenization_spaces=True,
-            )
 
+            out = chunk_resp.text
             if stream:
                 # this special character is mainly for qwen
                 out = out.strip("�")
@@ -266,11 +275,15 @@ class MLXModel(LLM):
                 total_tokens=(input_echo_len + i),
             ), completion_usage
 
+            if token == tokenizer.eos_token_id or token in stop_token_ids:  # type: ignore
+                break
+
         logger.info(
             f"Average generation speed: {i / (time.time() - start):.2f} tokens/s."
         )
 
-        self._prompt_cache.tokens.extend(tokens)  # type: ignore
+        if self._prompt_cache:
+            self._prompt_cache.tokens.extend(tokens)  # type: ignore
 
         if i == max_tokens - 1:
             finish_reason = "length"
@@ -314,10 +327,12 @@ class MLXModel(LLM):
             yield completion_chunk, completion_usage
 
     def generate(
-        self, prompt: str, generate_config: Optional[MLXGenerateConfig] = None
+        self,
+        prompt: Union[str, Dict[str, Any]],
+        generate_config: Optional[MLXGenerateConfig] = None,
     ) -> Union[Completion, Iterator[CompletionChunk]]:
         def generator_wrapper(
-            prompt: str, generate_config: MLXGenerateConfig
+            prompt: Union[str, Dict[str, Any]], generate_config: MLXGenerateConfig
         ) -> Iterator[CompletionChunk]:
             for completion_chunk, completion_usage in self._generate_stream(
                 prompt,
@@ -356,26 +371,6 @@ class MLXModel(LLM):
 
 
 class MLXChatModel(MLXModel, ChatModelMixin):
-    def __init__(
-        self,
-        model_uid: str,
-        model_family: "LLMFamilyV1",
-        model_spec: "LLMSpecV1",
-        quantization: str,
-        model_path: str,
-        model_config: Optional[MLXModelConfig] = None,
-        peft_model: Optional[List[LoRA]] = None,
-    ):
-        super().__init__(
-            model_uid,
-            model_family,
-            model_spec,
-            quantization,
-            model_path,
-            model_config,
-            peft_model,
-        )
-
     def _sanitize_generate_config(
         self,
         generate_config: Optional[MLXGenerateConfig],
@@ -402,6 +397,9 @@ class MLXChatModel(MLXModel, ChatModelMixin):
             return False
         if "chat" not in llm_family.model_ability:
             return False
+        if "vision" in llm_family.model_ability:
+            # do not process vision
+            return False
         return True
 
     def chat(
@@ -432,3 +430,186 @@ class MLXChatModel(MLXModel, ChatModelMixin):
             if tools:
                 return self._tool_calls_completion(self.model_family, self.model_uid, c)
             return self._to_chat_completion(c)
+
+
+class MLXVisionModel(MLXModel, ChatModelMixin):
+    @classmethod
+    def match(
+        cls, llm_family: "LLMFamilyV1", llm_spec: "LLMSpecV1", quantization: str
+    ) -> bool:
+        if llm_spec.model_format not in ["mlx"]:
+            return False
+        if sys.platform != "darwin" or platform.processor() != "arm":
+            # only work for Mac M chips
+            return False
+        if "vision" not in llm_family.model_ability:
+            return False
+        return True
+
+    def _load_model(self, **kwargs):
+        try:
+            from mlx_vlm import load
+        except ImportError:
+            error_message = "Failed to import module 'mlx_vlm'"
+            installation_guide = [
+                "Please make sure 'mlx_vlm' is installed. ",
+                "You can install it by `pip install mlx_vlm`\n",
+            ]
+
+            raise ImportError(f"{error_message}\n\n{''.join(installation_guide)}")
+
+        return load(self.model_path)
+
+    def load(self):
+        kwargs = {}
+        kwargs["revision"] = self._model_config.get(
+            "revision", self.model_spec.model_revision
+        )
+        kwargs["trust_remote_code"] = self._model_config.get("trust_remote_code")
+        kwargs["cache_limit_gb"] = self._model_config.pop("cache_limit_gb", None)
+
+        self._model, self._processor = self._load_model(**kwargs)
+        self._tokenizer = self._processor.tokenizer
+
+    def _generate_stream_inner(self, **kwargs):
+        import mlx.core as mx
+        from mlx_lm.utils import GenerationResponse
+        from mlx_vlm.utils import generate_step
+
+        max_tokens = kwargs.pop("max_tokens")
+        inputs = kwargs["prompt_token_ids"]
+        input_ids, pixel_values, mask = inputs[:3]
+
+        kwargs = {
+            k: v
+            for k, v in zip(
+                [
+                    "image_grid_thw",
+                    "image_sizes",
+                    "aspect_ratio_ids",
+                    "aspect_ratio_mask",
+                    "cross_attention_mask",
+                ],
+                inputs[3:],
+            )
+        }
+
+        tokenizer = self._processor.tokenizer
+        detokenizer = self._processor.detokenizer
+
+        detokenizer.reset()
+        tic = time.perf_counter()
+        for (token, logprobs), n in zip(
+            generate_step(input_ids, self._model, pixel_values, mask, **kwargs),
+            range(max_tokens),
+        ):
+            if n == 0:
+                prompt_time = time.perf_counter() - tic
+                prompt_tps = len(input_ids) / prompt_time
+                tic = time.perf_counter()
+            if token == tokenizer.eos_token_id:
+                break
+            detokenizer.add_token(token)
+
+            # Yield the last segment if streaming
+            yield GenerationResponse(
+                text=detokenizer.last_segment,
+                token=token,
+                logprobs=logprobs,
+                prompt_tokens=len(input_ids),
+                prompt_tps=prompt_tps,
+                generation_tokens=n + 1,
+                generation_tps=(n + 1) / (time.perf_counter() - tic),
+                peak_memory=mx.metal.get_peak_memory() / 1e9,
+            )
+
+        detokenizer.finalize()
+        yield GenerationResponse(
+            text=detokenizer.last_segment,
+            token=token,
+            logprobs=logprobs,
+            prompt_tokens=len(input_ids),
+            prompt_tps=prompt_tps,
+            generation_tokens=n + 1,
+            generation_tps=(n + 1) / (time.perf_counter() - tic),
+            peak_memory=mx.metal.get_peak_memory() / 1e9,
+        )
+
+    def _prepare_inputs(
+        self, prompt: Union[str, Dict[str, Any]], kwargs
+    ) -> Tuple[Any, int]:
+        from mlx_vlm import prepare_inputs
+
+        prompt_str = prompt.get("prompt")  # type: ignore
+        images = prompt.get("multi_modal_data", {}).get("image")  # type: ignore
+        if images and not isinstance(images, list):
+            images = [images]
+        if hasattr(self._model.config, "image_token_index"):
+            image_token_index = self._model.config.image_token_index
+        else:
+            image_token_index = None
+
+        inputs = prepare_inputs(
+            None,
+            self._processor,
+            images,
+            prompt_str,
+            image_token_index,
+            kwargs.get("resize_shape"),
+        )
+        input_ids = inputs[0]
+        return inputs, len(input_ids)
+
+    def chat(
+        self,
+        messages: List[Dict],
+        generate_config: Optional[MLXGenerateConfig] = None,
+    ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
+        messages = self._transform_messages(messages)  # type: ignore
+        tools = generate_config.pop("tools", []) if generate_config else None
+
+        model_family = self.model_family.model_family or self.model_family.model_name
+
+        if "internvl2" not in model_family.lower():
+            from qwen_vl_utils import process_vision_info
+
+            full_context_kwargs = {}
+            if tools and model_family in QWEN_TOOL_CALL_FAMILY:
+                full_context_kwargs["tools"] = tools
+            assert self.model_family.chat_template is not None
+            prompt = self.get_full_context(
+                messages, self.model_family.chat_template, **full_context_kwargs
+            )
+            images, video_inputs = process_vision_info(messages)
+            if video_inputs:
+                raise ValueError("Not support video input now.")
+        else:
+            prompt, images = self.get_specific_prompt(model_family, messages)  # type: ignore
+
+        if not images:
+            inputs = {
+                "prompt": prompt,
+            }
+        elif len(images) == 1:
+            inputs = {
+                "prompt": prompt,
+                "multi_modal_data": {"image": images[-1]},  # type: ignore
+            }
+        else:
+            inputs = {
+                "prompt": prompt,
+                "multi_modal_data": {"image": images},  # type: ignore
+            }
+        generate_config = self._sanitize_generate_config(generate_config)
+
+        stream = generate_config.get("stream", False)
+        if stream:
+            it = self.generate(inputs, generate_config)
+            assert isinstance(it, Iterator)
+            return self._to_chat_completion_chunks(it)
+        else:
+            c = self.generate(inputs, generate_config)
+            assert not isinstance(c, Iterator)
+            if tools:
+                return self._tool_calls_completion(self.model_family, self.model_uid, c)
+            return self._to_chat_completion(c)

xinference/model/llm/sglang/core.py

@@ -75,6 +75,7 @@ SGLANG_SUPPORTED_CHAT_MODELS = [
     "llama-2-chat",
     "llama-3-instruct",
     "llama-3.1-instruct",
+    "llama-3.3-instruct",
     "qwen-chat",
     "qwen1.5-chat",
     "qwen2-instruct",