xinference 0.12.1__py3-none-any.whl → 0.12.2.post1__py3-none-any.whl

xinference/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2024-06-14T17:17:50+0800",
+ "date": "2024-06-22T23:28:43+0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "34a57df449f0890415c424802d3596f3c8758412",
- "version": "0.12.1"
+ "full-revisionid": "7705d4ae1eb68523e87c4f2abf84026dae18b694",
+ "version": "0.12.2.post1"
 }
 '''  # END VERSION_JSON
 

xinference/api/restful_api.py

@@ -109,6 +109,7 @@ class RerankRequest(BaseModel):
     documents: List[str]
     top_n: Optional[int] = None
     return_documents: Optional[bool] = False
+    return_len: Optional[bool] = False
     max_chunks_per_doc: Optional[int] = None
 
 
@@ -981,7 +982,8 @@ class RESTfulAPI:
         return JSONResponse(content=self._supervisor_address)
 
     async def create_completion(self, request: Request) -> Response:
-        body = CreateCompletionRequest.parse_obj(await request.json())
+        raw_body = await request.json()
+        body = CreateCompletionRequest.parse_obj(raw_body)
         exclude = {
             "prompt",
             "model",
@@ -991,6 +993,7 @@ class RESTfulAPI:
             "logit_bias_type",
             "user",
         }
+        raw_kwargs = {k: v for k, v in raw_body.items() if k not in exclude}
         kwargs = body.dict(exclude_unset=True, exclude=exclude)
 
         # TODO: Decide if this default value override is necessary #1061
@@ -1020,7 +1023,9 @@ class RESTfulAPI:
                 iterator = None
                 try:
                     try:
-                        iterator = await model.generate(body.prompt, kwargs)
+                        iterator = await model.generate(
+                            body.prompt, kwargs, raw_params=raw_kwargs
+                        )
                     except RuntimeError as re:
                         self.handle_request_limit_error(re)
                     async for item in iterator:
@@ -1040,7 +1045,7 @@ class RESTfulAPI:
             return EventSourceResponse(stream_results())
         else:
             try:
-                data = await model.generate(body.prompt, kwargs)
+                data = await model.generate(body.prompt, kwargs, raw_params=raw_kwargs)
                 return Response(data, media_type="application/json")
             except Exception as e:
                 logger.error(e, exc_info=True)
@@ -1112,6 +1117,7 @@ class RESTfulAPI:
                 top_n=body.top_n,
                 max_chunks_per_doc=body.max_chunks_per_doc,
                 return_documents=body.return_documents,
+                return_len=body.return_len,
                 **kwargs,
             )
             return Response(scores, media_type="application/json")
@@ -1341,7 +1347,8 @@ class RESTfulAPI:
             raise HTTPException(status_code=500, detail=str(e))
 
     async def create_chat_completion(self, request: Request) -> Response:
-        body = CreateChatCompletion.parse_obj(await request.json())
+        raw_body = await request.json()
+        body = CreateChatCompletion.parse_obj(raw_body)
         exclude = {
             "prompt",
             "model",
@@ -1351,6 +1358,7 @@ class RESTfulAPI:
             "logit_bias_type",
             "user",
         }
+        raw_kwargs = {k: v for k, v in raw_body.items() if k not in exclude}
         kwargs = body.dict(exclude_unset=True, exclude=exclude)
 
         # TODO: Decide if this default value override is necessary #1061
@@ -1425,7 +1433,9 @@ class RESTfulAPI:
             "gorilla-openfunctions-v1",
             "qwen-chat",
             "qwen1.5-chat",
+            "qwen1.5-moe-chat",
             "qwen2-instruct",
+            "qwen2-moe-instruct",
         ]
 
         is_qwen = desc.get("model_format") == "ggmlv3" and "qwen-chat" == model_family
@@ -1451,7 +1461,9 @@ class RESTfulAPI:
             if not is_vllm or model_family not in [
                 "qwen-chat",
                 "qwen1.5-chat",
+                "qwen1.5-moe-chat",
                 "qwen2-instruct",
+                "qwen2-moe-instruct",
             ]:
                 raise HTTPException(
                     status_code=400,
@@ -1465,10 +1477,16 @@ class RESTfulAPI:
                 try:
                     try:
                         if is_qwen:
-                            iterator = await model.chat(prompt, chat_history, kwargs)
+                            iterator = await model.chat(
+                                prompt, chat_history, kwargs, raw_params=raw_kwargs
+                            )
                         else:
                             iterator = await model.chat(
-                                prompt, system_prompt, chat_history, kwargs
+                                prompt,
+                                system_prompt,
+                                chat_history,
+                                kwargs,
+                                raw_params=raw_kwargs,
                             )
                     except RuntimeError as re:
                         await self._report_error_event(model_uid, str(re))
@@ -1498,9 +1516,17 @@ class RESTfulAPI:
         else:
             try:
                 if is_qwen:
-                    data = await model.chat(prompt, chat_history, kwargs)
+                    data = await model.chat(
+                        prompt, chat_history, kwargs, raw_params=raw_kwargs
+                    )
                 else:
-                    data = await model.chat(prompt, system_prompt, chat_history, kwargs)
+                    data = await model.chat(
+                        prompt,
+                        system_prompt,
+                        chat_history,
+                        kwargs,
+                        raw_params=raw_kwargs,
+                    )
                 return Response(content=data, media_type="application/json")
             except Exception as e:
                 logger.error(e, exc_info=True)

xinference/client/restful/restful_client.py

@@ -135,6 +135,7 @@ class RESTfulRerankModelHandle(RESTfulModelHandle):
         top_n: Optional[int] = None,
         max_chunks_per_doc: Optional[int] = None,
         return_documents: Optional[bool] = None,
+        return_len: Optional[bool] = None,
         **kwargs,
     ):
         """
@@ -152,6 +153,8 @@ class RESTfulRerankModelHandle(RESTfulModelHandle):
             The maximum number of chunks derived from a document
         return_documents: bool
             if return documents
+        return_len: bool
+            if return tokens len
         Returns
         -------
         Scores
@@ -170,6 +173,7 @@ class RESTfulRerankModelHandle(RESTfulModelHandle):
             "top_n": top_n,
             "max_chunks_per_doc": max_chunks_per_doc,
             "return_documents": return_documents,
+            "return_len": return_len,
         }
         request_body.update(kwargs)
         response = requests.post(url, json=request_body, headers=self.auth_headers)

xinference/core/event.py

@@ -12,8 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import queue
-from collections import defaultdict
+from collections import defaultdict, deque
 from enum import Enum
 from typing import Dict, List, TypedDict
 
@@ -37,8 +36,8 @@ class Event(TypedDict):
 class EventCollectorActor(xo.StatelessActor):
     def __init__(self):
         super().__init__()
-        self._model_uid_to_events: Dict[str, queue.Queue] = defaultdict(  # type: ignore
-            lambda: queue.Queue(maxsize=MAX_EVENT_COUNT_PER_MODEL)
+        self._model_uid_to_events: Dict[str, deque] = defaultdict(  # type: ignore
+            lambda: deque(maxlen=MAX_EVENT_COUNT_PER_MODEL)
         )
 
     @classmethod
@@ -50,7 +49,7 @@ class EventCollectorActor(xo.StatelessActor):
         if event_queue is None:
             return []
         else:
-            return [dict(e, event_type=e["event_type"].name) for e in event_queue.queue]
+            return [dict(e, event_type=e["event_type"].name) for e in iter(event_queue)]
 
     def report_event(self, model_uid: str, event: Event):
-        self._model_uid_to_events[model_uid].put(event)
+        self._model_uid_to_events[model_uid].append(event)

xinference/core/model.py

@@ -264,13 +264,14 @@ class ModelActor(xo.StatelessActor):
         return isinstance(self._model, VLLMModel)
 
     def allow_batching(self) -> bool:
-        from ..model.llm.pytorch.core import PytorchChatModel, PytorchModel
+        from ..model.llm.pytorch.core import PytorchModel
+
+        model_ability = self._model_description.get("model_ability", [])
 
         return (
             XINFERENCE_TRANSFORMERS_ENABLE_BATCHING
             and isinstance(self._model, PytorchModel)
-            and self._model.__class__.__name__
-            in (PytorchChatModel.__name__, PytorchModel.__name__)
+            and "vision" not in model_ability
         )
 
     async def load(self):
@@ -399,6 +400,7 @@ class ModelActor(xo.StatelessActor):
                 prompt, "generate", *args, **kwargs
             )
         else:
+            kwargs.pop("raw_params", None)
             if hasattr(self._model, "generate"):
                 return await self._call_wrapper(
                     self._model.generate, prompt, *args, **kwargs
@@ -481,6 +483,7 @@ class ModelActor(xo.StatelessActor):
                     prompt, "chat", *args, **kwargs
                 )
             else:
+                kwargs.pop("raw_params", None)
                 if hasattr(self._model, "chat"):
                     response = await self._call_wrapper(
                         self._model.chat, prompt, *args, **kwargs
@@ -540,6 +543,7 @@ class ModelActor(xo.StatelessActor):
         top_n: Optional[int],
         max_chunks_per_doc: Optional[int],
         return_documents: Optional[bool],
+        return_len: Optional[bool],
         *args,
         **kwargs,
     ):
@@ -551,6 +555,7 @@ class ModelActor(xo.StatelessActor):
                 top_n,
                 max_chunks_per_doc,
                 return_documents,
+                return_len,
                 *args,
                 **kwargs,
             )

xinference/core/scheduler.py

@@ -18,7 +18,7 @@ import logging
 import uuid
 from collections import deque
 from enum import Enum
-from typing import List, Optional, Set
+from typing import List, Optional, Set, Tuple
 
 import xoscar as xo
 
@@ -53,7 +53,8 @@ class InferenceRequest:
         self._kv_cache = None
         # use passed args from upstream interface
         self._inference_args = args
-        # use passed kwargs from upstream interface, basically not used for now
+        # use passed kwargs from upstream interface, currently for getting raw generate config from upstream,
+        # which is useful for some special models
         self._inference_kwargs = kwargs
         # should this request be stopped
         self._stopped = False
@@ -66,6 +67,8 @@ class InferenceRequest:
         self._sanitized_generate_config = None
         # Chunk id for results. In stream mode, all the chunk ids should be same.
         self._stream_chunk_id = str(uuid.uuid4())
+        # For calculate attention mask if needed
+        self.padding_len = 0
         # Use in stream mode
         self.last_output_length = 0
         # inference results,
@@ -172,6 +175,10 @@ class InferenceRequest:
     def sanitized_generate_config(self, value: dict):
         self._sanitized_generate_config = value
 
+    @property
+    def inference_kwargs(self):
+        return self._inference_kwargs
+
     @property
     def stopped(self):
         return self._stopped
@@ -231,7 +238,9 @@ class InferenceRequest:
         )
 
     @functools.lru_cache
-    def get_generate_configs(self, eos_token_id: int):
+    def get_generate_configs(
+        self, eos_token_id: int, builtin_stop_token_ids: Optional[Tuple[int]] = None
+    ):
         from ..types import max_tokens_field
 
         max_new_tokens = int(
@@ -245,6 +254,7 @@ class InferenceRequest:
         )
         stop_token_ids = set(stop_token_ids)
         stop_token_ids.add(eos_token_id)
+        stop_token_ids.update(builtin_stop_token_ids or [])
         temperature = float(self.sanitized_generate_config.get("temperature", 1.0))
         repetition_penalty = float(
             self.sanitized_generate_config.get("repetition_penalty", 1.0)

xinference/model/llm/llm_family.json

@@ -2290,7 +2290,8 @@
       "zh"
     ],
     "model_ability": [
-      "chat"
+      "chat",
+      "tools"
     ],
     "model_description": "Qwen1.5-MoE is a transformer-based MoE decoder-only language model pretrained on a large amount of data.",
     "model_specs": [
@@ -2595,7 +2596,8 @@
       "zh"
     ],
     "model_ability": [
-      "chat"
+      "chat",
+      "tools"
     ],
     "model_description": "Qwen2 is the new series of Qwen large language models. ",
     "model_specs": [
@@ -5675,9 +5677,11 @@
       ],
       "intra_message_sep": "<|im_end|>",
       "stop_token_ids": [
+        2,
         92542
       ],
       "stop": [
+        "</s>",
         "<|im_end|>"
       ]
     }

xinference/model/llm/llm_family_modelscope.json

@@ -2644,7 +2644,8 @@
       "zh"
     ],
     "model_ability": [
-      "chat"
+      "chat",
+      "tools"
     ],
     "model_description": "Qwen1.5-MoE is a transformer-based MoE decoder-only language model pretrained on a large amount of data.",
     "model_specs": [
@@ -2968,7 +2969,8 @@
       "zh"
     ],
     "model_ability": [
-      "chat"
+      "chat",
+      "tools"
     ],
     "model_description": "Qwen2 is the new series of Qwen large language models. ",
     "model_specs": [
@@ -3350,9 +3352,11 @@
       ],
       "intra_message_sep": "<|im_end|>",
       "stop_token_ids": [
+        2,
         92542
       ],
       "stop": [
+        "</s>",
         "<|im_end|>"
       ]
     }

xinference/model/llm/pytorch/chatglm.py

@@ -15,6 +15,7 @@ import time
 import uuid
 from typing import Any, Dict, Iterator, List, Optional, Union
 
+from ....core.scheduler import InferenceRequest
 from ....types import (
     SPECIAL_TOOL_PROMPT,
     ChatCompletion,
@@ -244,3 +245,25 @@ class ChatglmPytorchChatModel(PytorchChatModel):
                         prompt_tokens=-1, completion_tokens=-1, total_tokens=-1
                     ),
                 )
+
+    @staticmethod
+    def require_attention_mask():
+        """
+        GLM4 needs to use attention mask and position ids during inference.
+        Otherwise, the inference result would be not available.
+        """
+        return True
+
+    def prepare_sanitize_generate_config(self, req: InferenceRequest):
+        """
+        Set temperature and top_p to 0.8 by default
+        """
+        raw_config = req.inference_kwargs.get("raw_params", {})
+        temperature = raw_config.get("temperature", None)
+        if temperature is None:
+            raw_config["temperature"] = 0.8
+        top_p = raw_config.get("top_p", None)
+        if top_p is None:
+            raw_config["top_p"] = 0.8
+
+        return raw_config

xinference/model/llm/pytorch/core.py

@@ -16,7 +16,7 @@ import json
 import logging
 import os
 from functools import lru_cache
-from typing import Iterable, Iterator, List, Optional, Union
+from typing import Iterable, Iterator, List, Optional, Tuple, Union
 
 from ....core.scheduler import InferenceRequest
 from ....device_utils import (
@@ -283,35 +283,21 @@ class PytorchModel(LLM):
     def generate(
         self, prompt: str, generate_config: Optional[PytorchGenerateConfig] = None
     ) -> Union[Completion, Iterator[CompletionChunk]]:
-        from .utils import generate_stream, generate_stream_falcon
-
-        model_family_name = self.model_family.model_name.lower()
+        from .utils import generate_stream
 
         def generator_wrapper(
             prompt: str, generate_config: PytorchGenerateConfig
         ) -> Iterator[CompletionChunk]:
-            if "falcon" in model_family_name:
-                for completion_chunk, completion_usage in generate_stream_falcon(
-                    self.model_uid,
-                    self._model,
-                    self._tokenizer,
-                    prompt,
-                    self._device,
-                    generate_config,
-                ):
-                    completion_chunk["usage"] = completion_usage
-                    yield completion_chunk
-            else:
-                for completion_chunk, completion_usage in generate_stream(
-                    self.model_uid,
-                    self._model,
-                    self._tokenizer,
-                    prompt,
-                    self._device,
-                    generate_config,
-                ):
-                    completion_chunk["usage"] = completion_usage
-                    yield completion_chunk
+            for completion_chunk, completion_usage in generate_stream(
+                self.model_uid,
+                self._model,
+                self._tokenizer,
+                prompt,
+                self._device,
+                generate_config,
+            ):
+                completion_chunk["usage"] = completion_usage
+                yield completion_chunk
 
         logger.debug(
             "Enter generate, prompt: %s, generate config: %s", prompt, generate_config
@@ -336,26 +322,15 @@ class PytorchModel(LLM):
 
         stream = generate_config.get("stream", False)
         if not stream:
-            if "falcon" in model_family_name:
-                for completion_chunk, completion_usage in generate_stream_falcon(
-                    self.model_uid,
-                    self._model,
-                    self._tokenizer,
-                    prompt,
-                    self._device,
-                    generate_config,
-                ):
-                    pass
-            else:
-                for completion_chunk, completion_usage in generate_stream(
-                    self.model_uid,
-                    self._model,
-                    self._tokenizer,
-                    prompt,
-                    self._device,
-                    generate_config,
-                ):
-                    pass
+            for completion_chunk, completion_usage in generate_stream(
+                self.model_uid,
+                self._model,
+                self._tokenizer,
+                prompt,
+                self._device,
+                generate_config,
+            ):
+                pass
             completion = Completion(
                 id=completion_chunk["id"],
                 object=completion_chunk["object"],
@@ -368,6 +343,10 @@ class PytorchModel(LLM):
         else:
             return generator_wrapper(prompt, generate_config)
 
+    @staticmethod
+    def require_attention_mask():
+        return False
+
     @lru_cache
     def get_context_len(self):
         return get_context_length(self._model.config)
@@ -375,13 +354,14 @@ class PytorchModel(LLM):
     def get_max_num_seqs(self) -> int:
         return self._pytorch_model_config.get("max_num_seqs")  # type: ignore
 
+    def prepare_sanitize_generate_config(self, req: InferenceRequest):
+        return self._sanitize_generate_config(req.generate_config)
+
     def prepare_batch_inference(self, req_list: List[InferenceRequest]):
         # check some parameters
         for r in req_list:
             if r.sanitized_generate_config is None:
-                r.sanitized_generate_config = self._sanitize_generate_config(
-                    r.generate_config
-                )
+                r.sanitized_generate_config = self.prepare_sanitize_generate_config(r)
             if r.is_prefill:
                 # check some generate params
                 max_src_len = get_max_src_len(self.get_context_len(), r)  # type: ignore
@@ -401,6 +381,14 @@ class PytorchModel(LLM):
                     r.error_msg = "Invalid `stop` field type"
                     continue
 
+    def _get_builtin_stop_token_ids(self) -> Tuple:
+        return (
+            tuple(self.model_family.prompt_style.stop_token_ids)
+            if self.model_family.prompt_style
+            and self.model_family.prompt_style.stop_token_ids
+            else tuple()
+        )
+
     def handle_batch_inference_results(self, req_list: List[InferenceRequest]):
         for req in req_list:
             if req.error_msg is None:
@@ -449,6 +437,8 @@ class PytorchModel(LLM):
             self._tokenizer,
             self._device,
             context_len,
+            self._get_builtin_stop_token_ids(),
+            require_attention_mask=self.require_attention_mask(),
         )
         self.handle_batch_inference_results(req_list)
 

xinference/model/llm/pytorch/glm4v.py

@@ -64,6 +64,8 @@ class Glm4VModel(PytorchChatModel):
 
         kwargs = {"device_map": self._device}
         quantization = self.quantization
+
+        # referenced from PytorchModel.load
         if quantization != "none":
             if self._device == "cuda" and self._is_linux():
                 kwargs["device_map"] = "auto"
@@ -72,6 +74,15 @@ class Glm4VModel(PytorchChatModel):
                     kwargs["load_in_4bit"] = True
                 elif quantization == "8-bit":
                     kwargs["load_in_8bit"] = True
+                else:
+                    raise ValueError(
+                        f"Quantization {quantization} is not supported in temporary"
+                    )
+            else:
+                if quantization != "8-bit":
+                    raise ValueError(
+                        f"Only 8-bit quantization is supported if it is not linux system or cuda device"
+                    )
 
         model = AutoModelForCausalLM.from_pretrained(
             self.model_path,

xinference/model/llm/pytorch/internlm2.py

@@ -15,6 +15,7 @@ import time
 import uuid
 from typing import Any, Dict, Iterator, List, Optional, Union
 
+from ....core.scheduler import InferenceRequest
 from ....types import (
     ChatCompletion,
     ChatCompletionChoice,
@@ -88,6 +89,20 @@ class Internlm2PytorchChatModel(PytorchChatModel):
             return False
         return True
 
+    def prepare_sanitize_generate_config(self, req: InferenceRequest):
+        """
+        Overwrite this func for this special model.
+        Cannot use the default configuration, which works poorly on this model.
+        """
+        raw_config = req.inference_kwargs.get("raw_params", {})
+        temperature = raw_config.get("temperature", None)
+        if temperature is None:
+            raw_config["temperature"] = 0.8
+        top_p = raw_config.get("top_p", None)
+        if top_p is None:
+            raw_config["top_p"] = 0.8
+        return raw_config
+
     def chat(
         self,
         prompt: str,