xinference 0.15.3__py3-none-any.whl → 0.16.0__py3-none-any.whl

xinference/model/llm/transformers/core.py

@@ -29,7 +29,6 @@ from ....device_utils import (
 from ....types import (
     ChatCompletion,
     ChatCompletionChunk,
-    Completion,
     CompletionChoice,
     CompletionChunk,
     CreateCompletionTorch,
@@ -40,15 +39,13 @@ from ....types import (
 from ...utils import select_device
 from ..core import LLM
 from ..llm_family import LLMFamilyV1, LLMSpecV1
-from ..utils import QWEN_TOOL_CALL_FAMILY, ChatModelMixin
+from ..utils import LLAMA3_TOOL_CALL_FAMILY, QWEN_TOOL_CALL_FAMILY, ChatModelMixin
 from .utils import get_context_length, get_max_src_len, pad_prefill_tokens
 
 logger = logging.getLogger(__name__)
 
 NON_DEFAULT_MODEL_LIST: List[str] = [
-    "chatglm3",
-    "chatglm3-32k",
-    "chatglm3-128k",
+    "opt",
     "glm4-chat",
     "glm4-chat-1m",
     "internlm2-chat",
@@ -345,69 +342,6 @@ class PytorchModel(LLM):
             return False
         return True
 
-    def generate(
-        self, prompt: str, generate_config: Optional[PytorchGenerateConfig] = None
-    ) -> Union[Completion, Iterator[CompletionChunk]]:
-        from .utils import generate_stream
-
-        def generator_wrapper(
-            prompt: str, generate_config: PytorchGenerateConfig
-        ) -> Iterator[CompletionChunk]:
-            for completion_chunk, completion_usage in generate_stream(
-                self.model_uid,
-                self._model,
-                self._tokenizer,
-                prompt,
-                self._device,
-                generate_config,
-            ):
-                completion_chunk["usage"] = completion_usage
-                yield completion_chunk
-
-        logger.debug(
-            "Enter generate, prompt: %s, generate config: %s", prompt, generate_config
-        )
-
-        generate_config = self._sanitize_generate_config(generate_config)
-
-        assert self._model is not None
-        assert self._tokenizer is not None
-
-        lora_model = generate_config.pop("lora_name")
-
-        if lora_model is not None and self._peft_model is not None:
-            for lora in self._peft_model:
-                if lora_model == lora.lora_name:
-                    self._model.set_adapter(lora_model)
-                    logger.info(f"Set lora model to {lora_model}")
-                    break
-            else:
-                self._model.disable_adapter()
-                logger.info(f"No lora model {lora_model} found, skip setting")
-
-        stream = generate_config.get("stream", False)
-        if not stream:
-            for completion_chunk, completion_usage in generate_stream(
-                self.model_uid,
-                self._model,
-                self._tokenizer,
-                prompt,
-                self._device,
-                generate_config,
-            ):
-                pass
-            completion = Completion(
-                id=completion_chunk["id"],
-                object=completion_chunk["object"],
-                created=completion_chunk["created"],
-                model=completion_chunk["model"],
-                choices=completion_chunk["choices"],
-                usage=completion_usage,
-            )
-            return completion
-        else:
-            return generator_wrapper(prompt, generate_config)
-
     def build_prefill_attention_mask(
         self, batch_size: int, seq_length: int, reqs: List[InferenceRequest]
     ):
@@ -730,10 +664,19 @@ class PytorchChatModel(PytorchModel, ChatModelMixin):
         messages: List[Dict],
         generate_config: Optional[PytorchGenerateConfig] = None,
     ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
-        tools = generate_config.pop("tools", []) if generate_config else None
+        raise NotImplementedError
+
+    def load(self):
+        super().load()
+
+    def _get_full_prompt(self, messages: List[Dict], tools):
         model_family = self.model_family.model_family or self.model_family.model_name
         full_context_kwargs = {}
-        if tools and model_family in QWEN_TOOL_CALL_FAMILY:
+        if (
+            tools
+            and model_family in QWEN_TOOL_CALL_FAMILY
+            or model_family in LLAMA3_TOOL_CALL_FAMILY
+        ):
             full_context_kwargs["tools"] = tools
         assert self.model_family.chat_template is not None
         full_prompt = self.get_full_context(
@@ -742,29 +685,6 @@ class PytorchChatModel(PytorchModel, ChatModelMixin):
             tokenizer=self._tokenizer,
             **full_context_kwargs,
         )
-
-        generate_config = self._sanitize_generate_config(generate_config)
-
-        stream = generate_config.get("stream", False)
-        if stream:
-            it = self.generate(full_prompt, generate_config)
-            assert isinstance(it, Iterator)
-            return self._to_chat_completion_chunks(it)
-        else:
-            c = self.generate(full_prompt, generate_config)
-            assert not isinstance(c, Iterator)
-            if tools:
-                return self._tool_calls_completion(self.model_family, self.model_uid, c)
-            return self._to_chat_completion(c)
-
-    def load(self):
-        super().load()
-
-    def _get_full_prompt(self, messages: List[Dict], tools):
-        assert self.model_family.chat_template is not None
-        full_prompt = self.get_full_context(
-            messages, self.model_family.chat_template, tokenizer=self._tokenizer
-        )
         return full_prompt
 
     def prepare_batch_inference(self, req_list: List[InferenceRequest]):
@@ -772,12 +692,39 @@ class PytorchChatModel(PytorchModel, ChatModelMixin):
         for r in req_list:
             try:
                 if not r.stopped and r.is_prefill:
-                    r.full_prompt = self._get_full_prompt(r.prompt, None)
+                    tools = r.generate_config.get("tools", None)
+                    r.full_prompt = self._get_full_prompt(r.prompt, tools)
+                    if tools:
+                        r.tools = tools
             except Exception as e:
                 logger.exception(f"prepare inference error with {e}")
                 r.stopped = True
                 r.error_msg = str(e)
 
+    def handle_chat_result_non_streaming(self, req: InferenceRequest):
+        if req.tools:
+            req.completion[0] = self._tool_calls_completion(
+                self.model_family, self.model_uid, req.completion[0]
+            )
+        else:
+            req.completion[0] = self._to_chat_completion(req.completion[0])
+
+    def handle_chat_result_streaming(self, req: InferenceRequest):
+        results = []
+        for i, c in enumerate(req.completion):
+            if c == "<bos_stream>":
+                results.append(
+                    self._get_first_chat_completion_chunk(req.completion[i + 1])
+                )
+            elif c == "<eos_stream>":
+                break
+            else:
+                results.append(self._to_chat_completion_chunk(c))
+
+        if req.stopped and req.include_usage:
+            results.append(self._get_final_chat_completion_chunk(req.completion[-1]))
+        req.completion = results
+
     def handle_batch_inference_results(self, req_list: List[InferenceRequest]):
         for req in req_list:
             if req.error_msg is None and req.completion:
@@ -796,23 +743,6 @@ class PytorchChatModel(PytorchModel, ChatModelMixin):
                     continue
 
                 if req.stream:
-                    results = []
-                    for i, c in enumerate(req.completion):
-                        if c == "<bos_stream>":
-                            results.append(
-                                self._get_first_chat_completion_chunk(
-                                    req.completion[i + 1]
-                                )
-                            )
-                        elif c == "<eos_stream>":
-                            break
-                        else:
-                            results.append(self._to_chat_completion_chunk(c))
-
-                    if req.stopped and req.include_usage:
-                        results.append(
-                            self._get_final_chat_completion_chunk(req.completion[-1])
-                        )
-                    req.completion = results
+                    self.handle_chat_result_streaming(req)
                 else:
-                    req.completion[0] = self._to_chat_completion(req.completion[0])
+                    self.handle_chat_result_non_streaming(req)

xinference/model/llm/transformers/deepseek_v2.py

@@ -12,24 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-import uuid
-from typing import Dict, Iterator, List, Optional, Union
 
 import torch
 
-from ....types import (
-    ChatCompletion,
-    ChatCompletionChunk,
-    Completion,
-    CompletionChunk,
-    PytorchGenerateConfig,
-)
 from ..llm_family import LLMFamilyV1, LLMSpecV1
-from ..utils import (
-    generate_chat_completion,
-    generate_completion,
-    generate_completion_chunk,
-)
 from .core import PytorchChatModel, PytorchModel
 
 logger = logging.getLogger(__name__)
@@ -80,95 +66,6 @@ class DeepSeekV2PytorchModel(PytorchModel):
             return False
         return True
 
-    def generate(
-        self, prompt: str, generate_config: Optional[PytorchGenerateConfig] = None
-    ) -> Union[Completion, Iterator[CompletionChunk]]:
-        input_tensor = self._tokenizer(prompt, return_tensors="pt")
-        generate_config = self._sanitize_generate_config(generate_config)
-        default_generate_config = self._model.generation_config
-        generate_kwargs = {
-            "input_ids": input_tensor["input_ids"].cuda(),
-            "attention_mask": input_tensor["attention_mask"].cuda(),
-            "temperature": float(
-                generate_config.get("temperature", default_generate_config.temperature)
-            ),
-            "repetition_penalty": float(generate_config.get("repetition_penalty", 1.0)),
-            "top_p": float(generate_config.get("top_p", default_generate_config.top_p)),
-            "top_k": int(generate_config.get("top_k", -1)),
-            "max_new_tokens": generate_config.get("max_tokens", 512),
-            "bos_token_id": default_generate_config.bos_token_id,
-            "do_sample": default_generate_config.do_sample,
-            "eos_token_id": default_generate_config.eos_token_id,
-        }
-
-        stream = generate_config.get("stream", False)
-        if stream:
-            return self._generate_stream(generate_kwargs, input_tensor)
-        else:
-            return self._generate(generate_kwargs, input_tensor)
-
-    def _generate(self, generate_kwargs, input_ids) -> Completion:
-        prompt_tokens = len(input_ids[0])
-        logger.info(f"generate_kwargs:{generate_kwargs}")
-        generation_output = self._model.generate(**generate_kwargs)
-        completion_tokens = len(generation_output[0])
-        response = self._tokenizer.decode(
-            generation_output[0], skip_special_tokens=True
-        )
-        return generate_completion(
-            self.model_uid,
-            response,
-            prompt_tokens=prompt_tokens,
-            completion_tokens=completion_tokens,
-            total_tokens=prompt_tokens + completion_tokens,
-        )
-
-    def _generate_stream(self, generate_kwargs, input_ids):
-        from threading import Thread
-
-        from transformers import TextIteratorStreamer
-
-        # Initialize the streamer
-        streamer = TextIteratorStreamer(
-            self._tokenizer, skip_prompt=True, skip_special_tokens=True, timeout=10
-        )
-        # Define the generation configuration
-        generate_kwargs["streamer"] = streamer
-        # Start the model chat in a separate thread
-        thread = Thread(
-            target=self._model.generate,
-            kwargs=generate_kwargs,
-        )
-        thread.start()
-
-        completion_id = str(uuid.uuid1())
-        prompt_tokens = len(input_ids[0])
-        total_tokens, completion_tokens = 0, 0
-        # Loop through the streamer to get the new text as it is generated
-        for i, new_text in enumerate(streamer):
-            completion_tokens = i
-            total_tokens = prompt_tokens + completion_tokens
-            yield generate_completion_chunk(
-                chunk_text=new_text,
-                finish_reason=None,
-                chunk_id=completion_id,
-                model_uid=self.model_uid,
-                prompt_tokens=prompt_tokens,
-                completion_tokens=completion_tokens,
-                total_tokens=total_tokens,
-            )
-        yield generate_completion_chunk(
-            chunk_text=None,
-            finish_reason="stop",
-            chunk_id=completion_id,
-            model_uid=self.model_uid,
-            prompt_tokens=prompt_tokens,
-            completion_tokens=completion_tokens,
-            total_tokens=total_tokens,
-            has_choice=True,
-            has_content=False,
-        )
-
 
 class DeepSeekV2PytorchChatModel(PytorchChatModel):
     def _load_model(self, **kwargs):
@@ -215,126 +112,3 @@ class DeepSeekV2PytorchChatModel(PytorchChatModel):
         if "chat" not in llm_family.model_ability:
             return False
         return True
-
-    def chat(
-        self,
-        messages: List[Dict],
-        generate_config: Optional[PytorchGenerateConfig] = None,
-    ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
-        assert self.model_family.chat_template is not None
-        full_prompt = self.get_full_context(
-            messages,
-            self.model_family.chat_template,
-            tokenizer=self._tokenizer,
-        )
-        input_tensor = self._tokenizer.encode(
-            full_prompt,
-            padding=False,
-            truncation=False,
-            max_length=None,
-            add_special_tokens=False,
-            return_tensors="pt",
-        )
-
-        generate_config = self._sanitize_generate_config(generate_config)
-        default_generate_config = self._model.generation_config
-        generate_kwargs = {
-            "input_ids": input_tensor.cuda(),
-            "temperature": float(
-                generate_config.get("temperature", default_generate_config.temperature)
-            ),
-            "repetition_penalty": float(generate_config.get("repetition_penalty", 1.0)),
-            "top_p": float(generate_config.get("top_p", default_generate_config.top_p)),
-            "top_k": int(generate_config.get("top_k", -1)),
-            "max_new_tokens": generate_config.get("max_tokens", 512),
-            "bos_token_id": default_generate_config.bos_token_id,
-            "do_sample": default_generate_config.do_sample,
-            "eos_token_id": default_generate_config.eos_token_id,
-        }
-
-        stream = generate_config.get("stream", False)
-        stream_options = generate_config.get("stream_options", None)
-        include_usage = (
-            stream_options["include_usage"]
-            if isinstance(stream_options, dict)
-            else False
-        )
-        if stream:
-            chunk = self._generate_stream(generate_kwargs, input_tensor, include_usage)
-            return self._to_chat_completion_chunks(chunk)
-        else:
-            return self._generate(generate_kwargs, input_tensor)
-
-    def _generate(self, generate_kwargs, input_ids) -> ChatCompletion:
-        prompt_tokens = len(input_ids[0])
-        generation_output = self._model.generate(**generate_kwargs)
-        completion_tokens = len(generation_output[0])
-        response = self._tokenizer.decode(
-            generation_output[0][input_ids.shape[1] :], skip_special_tokens=True
-        )
-        return generate_chat_completion(
-            self.model_uid,
-            response,
-            prompt_tokens=prompt_tokens,
-            completion_tokens=completion_tokens,
-            total_tokens=prompt_tokens + completion_tokens,
-        )
-
-    def _generate_stream(self, generate_kwargs, input_ids, include_usage):
-        from threading import Thread
-
-        from transformers import TextIteratorStreamer
-
-        # Initialize the streamer
-        streamer = TextIteratorStreamer(
-            self._tokenizer, skip_prompt=True, skip_special_tokens=True, timeout=10
-        )
-        # Define the generation configuration
-        generate_kwargs["streamer"] = streamer
-        # Start the model chat in a separate thread
-        thread = Thread(
-            target=self._model.generate,
-            kwargs=generate_kwargs,
-        )
-        thread.start()
-
-        completion_id = str(uuid.uuid1())
-        prompt_tokens = len(input_ids[0])
-        total_tokens, completion_tokens = 0, 0
-        # Loop through the streamer to get the new text as it is generated
-        for i, new_text in enumerate(streamer):
-            completion_tokens = max(completion_tokens, len(streamer.token_cache))
-            total_tokens = prompt_tokens + completion_tokens
-            yield generate_completion_chunk(
-                chunk_text=new_text,
-                finish_reason=None,
-                chunk_id=completion_id,
-                model_uid=self.model_uid,
-                prompt_tokens=prompt_tokens,
-                completion_tokens=completion_tokens,
-                total_tokens=total_tokens,
-            )
-        yield generate_completion_chunk(
-            chunk_text=None,
-            finish_reason="stop",
-            chunk_id=completion_id,
-            model_uid=self.model_uid,
-            prompt_tokens=prompt_tokens,
-            completion_tokens=completion_tokens,
-            total_tokens=total_tokens,
-            has_choice=True,
-            has_content=False,
-        )
-
-        if include_usage:
-            yield generate_completion_chunk(
-                chunk_text=None,
-                finish_reason=None,
-                chunk_id=completion_id,
-                model_uid=self.model_uid,
-                prompt_tokens=prompt_tokens,
-                completion_tokens=completion_tokens,
-                total_tokens=total_tokens,
-                has_choice=False,
-                has_content=False,
-            )

xinference/model/llm/transformers/deepseek_vl.py

@@ -28,6 +28,7 @@ from ....types import ChatCompletion, ChatCompletionChunk, CompletionChunk
 from ..llm_family import LLMFamilyV1, LLMSpecV1
 from ..utils import generate_chat_completion, generate_completion_chunk
 from .core import PytorchChatModel, PytorchGenerateConfig
+from .utils import cache_clean
 
 logger = logging.getLogger(__name__)
 
@@ -137,6 +138,7 @@ class DeepSeekVLChatModel(PytorchChatModel):
             return "".join(new_content), images
         return content, []
 
+    @cache_clean
     def chat(
         self,
         messages: List[Dict],

xinference/model/llm/transformers/glm4v.py

@@ -26,7 +26,7 @@ from ...utils import select_device
 from ..llm_family import LLMFamilyV1, LLMSpecV1
 from ..utils import _decode_image, generate_chat_completion, generate_completion_chunk
 from .core import PytorchChatModel, PytorchGenerateConfig
-from .utils import get_max_src_len
+from .utils import cache_clean, get_max_src_len
 
 logger = logging.getLogger(__name__)
 
@@ -129,6 +129,7 @@ class Glm4VModel(PytorchChatModel):
                     res.append({"role": role, "content": text})
         return res
 
+    @cache_clean
     def chat(
         self,
         messages: List[Dict],

xinference/model/llm/transformers/intern_vl.py

@@ -27,6 +27,7 @@ from ..utils import (
     parse_messages,
 )
 from .core import PytorchChatModel, PytorchGenerateConfig
+from .utils import cache_clean
 
 logger = logging.getLogger(__name__)
 
@@ -326,6 +327,7 @@ class InternVLChatModel(PytorchChatModel):
             use_fast=False,
         )
 
+    @cache_clean
     def chat(
         self,
         messages: List[Dict],

xinference/model/llm/transformers/internlm2.py

@@ -11,13 +11,12 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import uuid
-from typing import Any, Dict, Iterator, List, Optional, Union
+
+from typing import List, Optional
 
 from ....core.scheduler import InferenceRequest
-from ....types import ChatCompletion, ChatCompletionChunk, LoRA, PytorchGenerateConfig
+from ....types import LoRA
 from ..llm_family import LLMFamilyV1, LLMSpecV1
-from ..utils import generate_chat_completion, generate_completion_chunk, parse_messages
 from .core import PytorchChatModel, PytorchModelConfig
 
 
@@ -93,94 +92,3 @@ class Internlm2PytorchChatModel(PytorchChatModel):
         if top_p is None:
             raw_config["top_p"] = 0.8
         return raw_config
-
-    def chat(
-        self,
-        messages: List[Dict],
-        generate_config: Optional[PytorchGenerateConfig] = None,
-    ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
-        kwargs: Dict[str, Any] = {}
-        generate_config = generate_config or {}
-        temperature = generate_config.get("temperature")
-        if temperature is not None:
-            kwargs["temperature"] = float(temperature)
-        top_p = generate_config.get("top_p")
-        if top_p is not None:
-            kwargs["top_p"] = float(top_p)
-        max_new_tokens = generate_config.get("max_tokens")
-        if max_new_tokens is not None:
-            kwargs["max_length"] = int(max_new_tokens)
-
-        stream = generate_config.get("stream", False)
-        stream_options = generate_config.pop("stream_options", None)
-        include_usage = (
-            stream_options["include_usage"]
-            if isinstance(stream_options, dict)
-            else False
-        )
-
-        prompt, system_prompt, chat_history = parse_messages(messages)
-        if chat_history:
-            input_history = [
-                (chat_history[i]["content"], (chat_history[i + 1]["content"]))
-                for i in range(0, len(chat_history), 2)
-            ]
-        else:
-            input_history = []
-        if system_prompt:
-            kwargs["meta_instruction"] = system_prompt
-        if stream:
-
-            def _stream_generator():
-                last_chunk_text_length = 0
-                chunk_id = "chat-" + str(uuid.uuid1())
-                prompt_tokens, completion_tokens, total_tokens = 0, 0, 0
-                inputs = self._tokenizer([prompt], return_tensors="pt")
-                inputs = inputs.to(self._model.device)
-                prompt_tokens = len(inputs["input_ids"][0])
-                for chunk_text, _ in self._model.stream_chat(
-                    self._tokenizer, prompt, input_history, **kwargs
-                ):
-                    completion_tokens = completion_tokens + 1
-                    total_tokens = prompt_tokens + completion_tokens
-                    chunk_text = chunk_text[last_chunk_text_length:]
-                    last_chunk_text_length += len(chunk_text)
-
-                    yield generate_completion_chunk(
-                        chunk_text,
-                        finish_reason=None,
-                        chunk_id=chunk_id,
-                        model_uid=self.model_uid,
-                        prompt_tokens=prompt_tokens,
-                        completion_tokens=completion_tokens,
-                        total_tokens=total_tokens,
-                    )
-                yield generate_completion_chunk(
-                    None,
-                    finish_reason="stop",
-                    chunk_id=chunk_id,
-                    model_uid=self.model_uid,
-                    prompt_tokens=prompt_tokens,
-                    completion_tokens=completion_tokens,
-                    total_tokens=total_tokens,
-                    has_choice=True,
-                    has_content=False,
-                )
-                if include_usage:
-                    yield generate_completion_chunk(
-                        None,
-                        finish_reason=None,
-                        chunk_id=chunk_id,
-                        model_uid=self.model_uid,
-                        prompt_tokens=prompt_tokens,
-                        completion_tokens=completion_tokens,
-                        total_tokens=total_tokens,
-                        has_choice=False,
-                    )
-
-            return self._to_chat_completion_chunks(_stream_generator())
-        else:
-            response, _ = self._model.chat(
-                self._tokenizer, prompt, input_history, **kwargs
-            )
-            return generate_chat_completion(self.model_uid, response)

xinference/model/llm/transformers/minicpmv25.py

@@ -29,6 +29,7 @@ from ..utils import (
     parse_messages,
 )
 from .core import PytorchChatModel, PytorchGenerateConfig
+from .utils import cache_clean
 
 logger = logging.getLogger(__name__)
 
@@ -119,6 +120,7 @@ class MiniCPMV25Model(PytorchChatModel):
                 raise RuntimeError("Only one image per message is supported")
         return content, []
 
+    @cache_clean
     def chat(
         self,
         messages: List[Dict],

xinference/model/llm/transformers/minicpmv26.py

@@ -30,6 +30,7 @@ from ..utils import (
     parse_messages,
 )
 from .core import PytorchChatModel, PytorchGenerateConfig
+from .utils import cache_clean
 
 logger = logging.getLogger(__name__)
 
@@ -198,6 +199,7 @@ class MiniCPMV26Model(PytorchChatModel):
         msgs.append({"role": "user", "content": images_chat + [content]})
         return msgs, video_existed
 
+    @cache_clean
     def chat(
         self,
         messages: List[Dict],

xinference/model/llm/transformers/omnilmm.py

@@ -24,6 +24,7 @@ from ...utils import select_device
 from ..llm_family import LLMFamilyV1, LLMSpecV1
 from ..utils import generate_chat_completion, parse_messages
 from .core import PytorchChatModel, PytorchGenerateConfig
+from .utils import cache_clean
 
 logger = logging.getLogger(__name__)
 
@@ -87,6 +88,7 @@ class OmniLMMModel(PytorchChatModel):
             return images, other_content
         return [], [{"type": "text", "text": content}]
 
+    @cache_clean
     def chat(
         self,
         messages: List[Dict],