xinference 0.15.4__py3-none-any.whl → 0.16.1__py3-none-any.whl

xinference/model/image/stable_diffusion/mlx.py

@@ -0,0 +1,221 @@
+# Copyright 2022-2023 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import contextlib
+import gc
+import logging
+import re
+from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
+
+import numpy as np
+from PIL import Image
+from xoscar.utils import classproperty
+
+from ....types import LoRA
+from ..sdapi import SDAPIDiffusionModelMixin
+from ..utils import handle_image_result
+
+if TYPE_CHECKING:
+    from ....core.progress_tracker import Progressor
+    from ..core import ImageModelFamilyV1
+
+
+logger = logging.getLogger(__name__)
+
+
+def quantization_predicate(name: str, m) -> bool:
+    return hasattr(m, "to_quantized") and m.weight.shape[1] % 512 == 0
+
+
+def to_latent_size(image_size: Tuple[int, int]):
+    h, w = image_size
+    h = ((h + 15) // 16) * 16
+    w = ((w + 15) // 16) * 16
+
+    if (h, w) != image_size:
+        print(
+            "Warning: The image dimensions need to be divisible by 16px. "
+            f"Changing size to {h}x{w}."
+        )
+
+    return (h // 8, w // 8)
+
+
+class MLXDiffusionModel(SDAPIDiffusionModelMixin):
+    def __init__(
+        self,
+        model_uid: str,
+        model_path: Optional[str] = None,
+        device: Optional[str] = None,
+        lora_model: Optional[List[LoRA]] = None,
+        lora_load_kwargs: Optional[Dict] = None,
+        lora_fuse_kwargs: Optional[Dict] = None,
+        model_spec: Optional["ImageModelFamilyV1"] = None,
+        **kwargs,
+    ):
+        self._model_uid = model_uid
+        self._model_path = model_path
+        self._device = device
+        # model info when loading
+        self._model = None
+        self._lora_model = lora_model
+        self._lora_load_kwargs = lora_load_kwargs or {}
+        self._lora_fuse_kwargs = lora_fuse_kwargs or {}
+        # info
+        self._model_spec = model_spec
+        self._abilities = model_spec.model_ability or []  # type: ignore
+        self._kwargs = kwargs
+
+    @property
+    def model_ability(self):
+        return self._abilities
+
+    @classproperty
+    def supported_models(self):
+        return ["FLUX.1-schnell", "FLUX.1-dev"]
+
+    def load(self):
+        try:
+            import mlx.nn as nn
+        except ImportError:
+            error_message = "Failed to import module 'mlx'"
+            installation_guide = [
+                "Please make sure 'mlx' is installed. ",
+                "You can install it by `pip install mlx`\n",
+            ]
+
+            raise ImportError(f"{error_message}\n\n{''.join(installation_guide)}")
+
+        from ....thirdparty.mlx.flux import FluxPipeline
+
+        logger.debug(
+            "Loading model from %s, kwargs: %s", self._model_path, self._kwargs
+        )
+        flux = self._model = FluxPipeline(
+            "flux-" + self._model_spec.model_name.split("-")[1],
+            model_path=self._model_path,
+            t5_padding=self._kwargs.get("t5_padding", True),
+        )
+        self._apply_lora()
+
+        quantize = self._kwargs.get("quantize", True)
+        if quantize:
+            nn.quantize(flux.flow, class_predicate=quantization_predicate)
+            nn.quantize(flux.t5, class_predicate=quantization_predicate)
+            nn.quantize(flux.clip, class_predicate=quantization_predicate)
+
+    def _apply_lora(self):
+        if self._lora_model is not None:
+            import mlx.core as mx
+
+            for lora_model in self._lora_model:
+                weights, lora_config = mx.load(
+                    lora_model.local_path, return_metadata=True
+                )
+                rank = int(lora_config.get("lora_rank", 8))
+                num_blocks = int(lora_config.get("lora_blocks", -1))
+                flux = self._model
+                flux.linear_to_lora_layers(rank, num_blocks)
+                flux.flow.load_weights(list(weights.items()), strict=False)
+                flux.fuse_lora_layers()
+            logger.info(f"Successfully loaded the LoRA for model {self._model_uid}.")
+
+    @staticmethod
+    @contextlib.contextmanager
+    def _release_after():
+        import mlx.core as mx
+
+        try:
+            yield
+        finally:
+            gc.collect()
+            mx.metal.clear_cache()
+
+    def text_to_image(
+        self,
+        prompt: str,
+        n: int = 1,
+        size: str = "1024*1024",
+        response_format: str = "url",
+        **kwargs,
+    ):
+        import mlx.core as mx
+
+        flux = self._model
+        width, height = map(int, re.split(r"[^\d]+", size))
+
+        # Make the generator
+        latent_size = to_latent_size((height, width))
+        gen_latent_kwargs = {}
+        if (num_steps := kwargs.get("num_inference_steps")) is None:
+            num_steps = 50 if "dev" in self._model_spec.model_name else 2  # type: ignore
+        gen_latent_kwargs["num_steps"] = num_steps
+        if guidance := kwargs.get("guidance_scale"):
+            gen_latent_kwargs["guidance"] = guidance
+        if seed := kwargs.get("seed"):
+            gen_latent_kwargs["seed"] = seed
+
+        with self._release_after():
+            latents = flux.generate_latents(  # type: ignore
+                prompt, n_images=n, latent_size=latent_size, **gen_latent_kwargs
+            )
+
+            # First we get and eval the conditioning
+            conditioning = next(latents)
+            mx.eval(conditioning)
+            peak_mem_conditioning = mx.metal.get_peak_memory() / 1024**3
+            mx.metal.reset_peak_memory()
+
+            progressor: Progressor = kwargs.pop("progressor", None)
+            # Actual denoising loop
+            for i, x_t in enumerate(latents):
+                mx.eval(x_t)
+                progressor.set_progress((i + 1) / num_steps)
+
+            peak_mem_generation = mx.metal.get_peak_memory() / 1024**3
+            mx.metal.reset_peak_memory()
+
+            # Decode them into images
+            decoded = []
+            for i in range(n):
+                decoded.append(flux.decode(x_t[i : i + 1], latent_size))  # type: ignore
+                mx.eval(decoded[-1])
+            peak_mem_decoding = mx.metal.get_peak_memory() / 1024**3
+            peak_mem_overall = max(
+                peak_mem_conditioning, peak_mem_generation, peak_mem_decoding
+            )
+
+            images = []
+            x = mx.concatenate(decoded, axis=0)
+            x = (x * 255).astype(mx.uint8)
+            for i in range(len(x)):
+                im = Image.fromarray(np.array(x[i]))
+                images.append(im)
+
+        logger.debug(
+            f"Peak memory used for the text:       {peak_mem_conditioning:.3f}GB"
+        )
+        logger.debug(
+            f"Peak memory used for the generation: {peak_mem_generation:.3f}GB"
+        )
+        logger.debug(f"Peak memory used for the decoding:   {peak_mem_decoding:.3f}GB")
+        logger.debug(f"Peak memory used overall:            {peak_mem_overall:.3f}GB")
+
+        return handle_image_result(response_format, images)
+
+    def image_to_image(self, **kwargs):
+        raise NotImplementedError
+
+    def inpainting(self, **kwargs):
+        raise NotImplementedError

xinference/model/image/utils.py

@@ -11,16 +11,52 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from typing import Optional
+import base64
+import os
+import time
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+from functools import partial
+from io import BytesIO
+from typing import TYPE_CHECKING, Optional
 
-from .core import ImageModelFamilyV1
+from ...constants import XINFERENCE_IMAGE_DIR
+from ...types import Image, ImageList
+
+if TYPE_CHECKING:
+    from .core import ImageModelFamilyV1
 
 
 def get_model_version(
-    image_model: ImageModelFamilyV1, controlnet: Optional[ImageModelFamilyV1]
+    image_model: "ImageModelFamilyV1", controlnet: Optional["ImageModelFamilyV1"]
 ) -> str:
     return (
         image_model.model_name
         if controlnet is None
         else f"{image_model.model_name}--{controlnet.model_name}"
     )
+
+
+def handle_image_result(response_format: str, images) -> ImageList:
+    if response_format == "url":
+        os.makedirs(XINFERENCE_IMAGE_DIR, exist_ok=True)
+        image_list = []
+        with ThreadPoolExecutor() as executor:
+            for img in images:
+                path = os.path.join(XINFERENCE_IMAGE_DIR, uuid.uuid4().hex + ".jpg")
+                image_list.append(Image(url=path, b64_json=None))
+                executor.submit(img.save, path, "jpeg")
+        return ImageList(created=int(time.time()), data=image_list)
+    elif response_format == "b64_json":
+
+        def _gen_base64_image(_img):
+            buffered = BytesIO()
+            _img.save(buffered, format="jpeg")
+            return base64.b64encode(buffered.getvalue()).decode()
+
+        with ThreadPoolExecutor() as executor:
+            results = list(map(partial(executor.submit, _gen_base64_image), images))  # type: ignore
+            image_list = [Image(url=None, b64_json=s.result()) for s in results]  # type: ignore
+        return ImageList(created=int(time.time()), data=image_list)
+    else:
+        raise ValueError(f"Unsupported response format: {response_format}")

xinference/model/llm/__init__.py

@@ -146,6 +146,7 @@ def _install():
     from .transformers.internlm2 import Internlm2PytorchChatModel
     from .transformers.minicpmv25 import MiniCPMV25Model
     from .transformers.minicpmv26 import MiniCPMV26Model
+    from .transformers.opt import OptPytorchModel
     from .transformers.qwen2_audio import Qwen2AudioChatModel
     from .transformers.qwen2_vl import Qwen2VLChatModel
     from .transformers.qwen_vl import QwenVLChatModel
@@ -190,6 +191,7 @@ def _install():
             Glm4VModel,
             DeepSeekV2PytorchModel,
             DeepSeekV2PytorchChatModel,
+            OptPytorchModel,
         ]
     )
     if OmniLMMModel:  # type: ignore

xinference/model/llm/llm_family.json

@@ -206,7 +206,7 @@
           "none"
         ],
         "model_id": "THUDM/glm-4-9b-chat",
-        "model_revision": "f6e0743b285dd808084530f070ad08e504386750"
+        "model_revision": "eb55a443d66541f30869f6caac5ad0d2e95bcbaa"
       },
       {
         "model_format": "ggufv2",
@@ -7923,6 +7923,174 @@
             "00021-of-00021"
           ]
         }
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": "0_5",
+        "quantizations": [
+          "4-bit"
+        ],
+        "model_id": "mlx-community/Qwen2.5-0.5B-Instruct-4bit"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": "0_5",
+        "quantizations": [
+          "8-bit"
+        ],
+        "model_id": "mlx-community/Qwen2.5-0.5B-Instruct-8bit"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": "0_5",
+        "quantizations": [
+          "none"
+        ],
+        "model_id": "mlx-community/Qwen2.5-0.5B-Instruct-bf16"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": "1_5",
+        "quantizations": [
+          "4-bit"
+        ],
+        "model_id": "mlx-community/Qwen2.5-1.5B-Instruct-4bit"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": "1_5",
+        "quantizations": [
+          "8-bit"
+        ],
+        "model_id": "mlx-community/Qwen2.5-1.5B-Instruct-8bit"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": "1_5",
+        "quantizations": [
+          "none"
+        ],
+        "model_id": "mlx-community/Qwen2.5-1.5B-Instruct-bf16"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 3,
+        "quantizations": [
+          "4-bit"
+        ],
+        "model_id": "mlx-community/Qwen2.5-3B-Instruct-4bit"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 3,
+        "quantizations": [
+          "8-bit"
+        ],
+        "model_id": "mlx-community/Qwen2.5-3B-Instruct-8bit"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 3,
+        "quantizations": [
+          "none"
+        ],
+        "model_id": "mlx-community/Qwen2.5-3B-Instruct-bf16"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 7,
+        "quantizations": [
+          "4-bit"
+        ],
+        "model_id": "mlx-community/Qwen2.5-7B-Instruct-4bit"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 7,
+        "quantizations": [
+          "8-bit"
+        ],
+        "model_id": "mlx-community/Qwen2.5-7B-Instruct-8bit"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 7,
+        "quantizations": [
+          "none"
+        ],
+        "model_id": "mlx-community/Qwen2.5-7B-Instruct-bf16"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 14,
+        "quantizations": [
+          "4-bit"
+        ],
+        "model_id": "mlx-community/Qwen2.5-14B-Instruct-4bit"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 14,
+        "quantizations": [
+          "8-bit"
+        ],
+        "model_id": "mlx-community/Qwen2.5-14B-Instruct-8bit"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 14,
+        "quantizations": [
+          "none"
+        ],
+        "model_id": "mlx-community/Qwen2.5-14B-Instruct-bf16"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 32,
+        "quantizations": [
+          "4-bit"
+        ],
+        "model_id": "mlx-community/Qwen2.5-32B-Instruct-4bit"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 32,
+        "quantizations": [
+          "8-bit"
+        ],
+        "model_id": "mlx-community/Qwen2.5-32B-Instruct-8bit"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 32,
+        "quantizations": [
+          "none"
+        ],
+        "model_id": "mlx-community/Qwen2.5-32B-Instruct-bf16"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 72,
+        "quantizations": [
+          "4-bit"
+        ],
+        "model_id": "mlx-community/Qwen2.5-72B-Instruct-4bit"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 72,
+        "quantizations": [
+          "8-bit"
+        ],
+        "model_id": "mlx-community/Qwen2.5-72B-Instruct-8bit"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 72,
+        "quantizations": [
+          "none"
+        ],
+        "model_id": "mlx-community/Qwen2.5-72B-Instruct-bf16"
       }
     ],
     "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
@@ -8008,6 +8176,15 @@
         ],
         "model_id": "Qwen/Qwen2.5-Coder-7B-Instruct"
       },
+      {
+        "model_format": "gptq",
+        "model_size_in_billions": "7",
+        "quantizations": [
+            "Int4",
+            "Int8"
+        ],
+        "model_id": "Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-{quantization}"
+      },
       {
         "model_format": "ggufv2",
         "model_size_in_billions": "1_5",

xinference/model/llm/llm_family_modelscope.json

@@ -5681,6 +5681,114 @@
             "00021-of-00021"
           ]
         }
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 3,
+        "quantizations": [
+          "4-bit"
+        ],
+        "model_id": "okwinds/Qwen2.5-3B-Instruct-MLX-4bit",
+        "model_hub": "modelscope"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 3,
+        "quantizations": [
+          "8-bit"
+        ],
+        "model_id": "okwinds/Qwen2.5-3B-Instruct-MLX-8bit",
+        "model_hub": "modelscope"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 7,
+        "quantizations": [
+          "4-bit"
+        ],
+        "model_id": "okwinds/Qwen2.5-7B-Instruct-MLX-4bit",
+        "model_hub": "modelscope"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 7,
+        "quantizations": [
+          "8-bit"
+        ],
+        "model_id": "okwinds/Qwen2.5-7B-Instruct-MLX-8bit",
+        "model_hub": "modelscope"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 14,
+        "quantizations": [
+          "4-bit"
+        ],
+        "model_id": "okwinds/Qwen2.5-14B-Instruct-MLX-4bit",
+        "model_hub": "modelscope"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 14,
+        "quantizations": [
+          "8-bit"
+        ],
+        "model_id": "okwinds/Qwen2.5-14B-Instruct-MLX-8bit",
+        "model_hub": "modelscope"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 32,
+        "quantizations": [
+          "2-bit"
+        ],
+        "model_id": "okwinds/Qwen2.5-32B-Instruct-MLX-2bit",
+        "model_hub": "modelscope"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 32,
+        "quantizations": [
+          "4-bit"
+        ],
+        "model_id": "okwinds/Qwen2.5-32B-Instruct-MLX-4bit",
+        "model_hub": "modelscope"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 32,
+        "quantizations": [
+          "8-bit"
+        ],
+        "model_id": "okwinds/Qwen2.5-32B-Instruct-MLX-8bit",
+        "model_hub": "modelscope"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 72,
+        "quantizations": [
+          "2-bit"
+        ],
+        "model_id": "okwinds/Qwen2.5-32B-Instruct-MLX-2bit",
+        "model_hub": "modelscope"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 72,
+        "quantizations": [
+          "4-bit"
+        ],
+        "model_id": "okwinds/Qwen2.5-72B-Instruct-MLX-4bit",
+        "model_hub": "modelscope"
+      },
+      {
+        "model_format": "mlx",
+        "model_size_in_billions": 72,
+        "quantizations": [
+          "8-bit"
+        ],
+        "model_id": "okwinds/Qwen2.5-72B-Instruct-MLX-8bit",
+        "model_hub": "modelscope"
       }
     ],
     "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
@@ -5772,6 +5880,17 @@
         "model_revision": "master",
         "model_hub": "modelscope"
       },
+      {
+        "model_format": "gptq",
+        "model_size_in_billions": 7,
+        "quantizations": [
+          "Int4",
+          "Int8"
+        ],
+        "model_id": "qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-{quantization}",
+        "model_revision": "master",
+        "model_hub": "modelscope"
+      },
       {
         "model_format": "ggufv2",
         "model_size_in_billions": "1_5",