xinference 1.3.0.post1__py3-none-any.whl → 1.3.1__py3-none-any.whl

xinference/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-02-22T00:10:55+0800",
+ "date": "2025-03-09T12:06:50+0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "b2004d49ddeda17dc6404473b1f25f8769911e18",
- "version": "1.3.0.post1"
+ "full-revisionid": "5d6ec937ce2aca2511e9e0debc4c2ab06ca41f09",
+ "version": "1.3.1"
 }
 '''  # END VERSION_JSON
 

xinference/api/restful_api.py

@@ -1330,6 +1330,7 @@ class RESTfulAPI(CancelMixin):
             raise HTTPException(status_code=500, detail=str(e))
 
         try:
+            kwargs["model_uid"] = model_uid
             embedding = await model.create_embedding(body.input, **kwargs)
             return Response(embedding, media_type="application/json")
         except Exception as e:

xinference/conftest.py

@@ -304,3 +304,10 @@ def setup_with_auth():
             os.remove(auth_file)
         except:
             pass
+
+
+@pytest.fixture
+def set_use_xllamacpp():
+    os.environ["USE_XLLAMACPP"] = "1"
+    yield
+    del os.environ["USE_XLLAMACPP"]

xinference/core/model.py

@@ -231,6 +231,7 @@ class ModelActor(xo.StatelessActor, CancelMixin):
         driver_info: Optional[dict] = None,  # for model across workers
     ):
         super().__init__()
+        from ..model.llm.llama_cpp.core import XllamaCppModel
         from ..model.llm.lmdeploy.core import LMDeployModel
         from ..model.llm.sglang.core import SGLANGModel
         from ..model.llm.transformers.core import PytorchModel
@@ -251,7 +252,8 @@ class ModelActor(xo.StatelessActor, CancelMixin):
         self._lock = (
             None
             if isinstance(
-                self._model, (PytorchModel, VLLMModel, SGLANGModel, LMDeployModel)
+                self._model,
+                (PytorchModel, VLLMModel, SGLANGModel, LMDeployModel, XllamaCppModel),
             )
             else asyncio.locks.Lock()
         )

xinference/core/scheduler.py

@@ -97,6 +97,9 @@ class InferenceRequest:
         # check the integrity of args passed upstream
         self._check_args()
 
+        # for reasoning_content using
+        self.previous_texts = [""]
+
     def _check_args(self):
         assert len(self._inference_args) == 1
         # generate config

xinference/core/worker.py

@@ -1002,7 +1002,7 @@ class WorkerActor(xo.StatelessActor):
             )
         try:
             subpool_address = self._model_uid_to_addr[model_uid]
-            await self._main_pool.remove_sub_pool(subpool_address)
+            await self._main_pool.remove_sub_pool(subpool_address, force=True)
         except Exception as e:
             logger.debug(
                 "Remove sub pool failed, model uid: %s, error: %s", model_uid, e

xinference/deploy/cmdline.py

@@ -821,12 +821,6 @@ def remove_cache(
     type=bool,
     help="Whether or not to allow for custom models defined on the Hub in their own modeling files.",
 )
-@click.option(
-    "--reasoning-content",
-    default=False,
-    type=bool,
-    help="Whether or not to enable reasoning content in model responses.",
-)
 @click.option(
     "--api-key",
     "-ak",
@@ -855,7 +849,6 @@ def model_launch(
     worker_ip: Optional[str],
     gpu_idx: Optional[str],
     trust_remote_code: bool,
-    reasoning_content: bool,
     api_key: Optional[str],
     model_path: Optional[str],
 ):
@@ -948,7 +941,6 @@ def model_launch(
         gpu_idx=_gpu_idx,
         trust_remote_code=trust_remote_code,
         model_path=model_path,
-        reasoning_content=reasoning_content,
         **kwargs,
     )
 

xinference/model/embedding/core.py

@@ -268,7 +268,7 @@ class EmbeddingModel:
         **kwargs,
     ):
         sentences = self._fix_langchain_openai_inputs(sentences)
-
+        model_uid = kwargs.pop("model_uid", None)
         from sentence_transformers import SentenceTransformer
 
         kwargs.setdefault("normalize_embeddings", True)
@@ -546,8 +546,14 @@ class EmbeddingModel:
                 # when batching, the attention mask 1 means there is a token
                 # thus we just sum up it to get the total number of tokens
                 if "clip" in self._model_spec.model_name.lower():
-                    all_token_nums += features["input_ids"].numel()
-                    all_token_nums += features["pixel_values"].numel()
+                    if "input_ids" in features and hasattr(
+                        features["input_ids"], "numel"
+                    ):
+                        all_token_nums += features["input_ids"].numel()
+                    if "pixel_values" in features and hasattr(
+                        features["pixel_values"], "numel"
+                    ):
+                        all_token_nums += features["pixel_values"].numel()
                 else:
                     all_token_nums += features["attention_mask"].sum().item()
 
@@ -657,7 +663,7 @@ class EmbeddingModel:
                 self._model,
                 objs,
                 convert_to_numpy=False,
-                **self._kwargs,
+                **kwargs,
             )
         else:
             all_embeddings, all_token_nums = encode(
@@ -693,7 +699,8 @@ class EmbeddingModel:
                 if not is_bge_m3_flag_model and not kwargs.get("return_sparse")
                 else "dict"
             ),
-            model=self._model_uid,
+            model=model_uid,  # type: ignore
+            model_replica=self._model_uid,
             data=embedding_list,
             usage=usage,
         )

xinference/model/llm/__init__.py

@@ -129,7 +129,7 @@ def register_custom_model():
 
 
 def _install():
-    from .llama_cpp.core import LlamaCppChatModel, LlamaCppModel
+    from .llama_cpp.core import LlamaCppChatModel, LlamaCppModel, XllamaCppModel
     from .lmdeploy.core import LMDeployChatModel, LMDeployModel
     from .mlx.core import MLXChatModel, MLXModel, MLXVisionModel
     from .sglang.core import SGLANGChatModel, SGLANGModel
@@ -169,6 +169,7 @@ def _install():
         [
             LlamaCppChatModel,
             LlamaCppModel,
+            XllamaCppModel,
         ]
     )
     SGLANG_CLASSES.extend([SGLANGModel, SGLANGChatModel])

xinference/model/llm/core.py

@@ -25,6 +25,8 @@ from typing import TYPE_CHECKING, Dict, List, Literal, Optional, Tuple, Union
 from ...core.utils import parse_replica_model_uid
 from ...types import PeftModelConfig
 from ..core import ModelDescription
+from .reasoning_parsers import deepseek_r1_reasoning_parser  # noqa: F401
+from .reasoning_parsers.abs_reasoning_parsers import ReasoningParserManager
 
 if TYPE_CHECKING:
     from .llm_family import LLMFamilyV1, LLMSpecV1
@@ -57,6 +59,7 @@ class LLM(abc.ABC):
         self.model_spec = model_spec
         self.quantization = quantization
         self.model_path = model_path
+        self.reasoning_parser = None
         if args:
             raise ValueError(f"Unrecognized positional arguments: {args}")
         if kwargs:
@@ -117,6 +120,16 @@ class LLM(abc.ABC):
     ) -> bool:
         raise NotImplementedError
 
+    def prepare_parse_reasoning_content(self, reasoning_content):
+        # Initialize reasoning parser if model has reasoning ability
+        if "reasoning" in self.model_family.model_ability and reasoning_content:
+            module_name = self.model_family.model_family or self.model_family.model_name
+            self.reasoning_parser = ReasoningParserManager.get_parser(module_name)
+            self.reasoning_parser = self.reasoning_parser(
+                self.model_family.reasoning_start_tag,
+                self.model_family.reasoning_end_tag,
+            )
+
 
 class LLMDescription(ModelDescription):
     def __init__(

xinference/model/llm/llama_cpp/core.py

@@ -11,11 +11,15 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import concurrent.futures
 import logging
 import os
+import queue
 import time
 from typing import Dict, Iterator, List, Optional, Union
 
+import orjson
+
 from ....types import (
     ChatCompletion,
     ChatCompletionChunk,
@@ -32,6 +36,248 @@ from ..utils import DEEPSEEK_TOOL_CALL_FAMILY, QWEN_TOOL_CALL_FAMILY, ChatModelM
 
 logger = logging.getLogger(__name__)
 
+USE_XLLAMACPP = bool(int(os.environ.get("USE_XLLAMACPP", 0)))
+
+
+class _Sentinel:
+    pass
+
+
+class XllamaCppModel(LLM):
+    def __init__(
+        self,
+        model_uid: str,
+        model_family: "LLMFamilyV1",
+        model_spec: "LLMSpecV1",
+        quantization: str,
+        model_path: str,
+        llamacpp_model_config: Optional[LlamaCppModelConfig] = None,
+    ):
+        super().__init__(model_uid, model_family, model_spec, quantization, model_path)
+
+        self._llamacpp_model_config: LlamaCppModelConfig = self._sanitize_model_config(
+            llamacpp_model_config
+        )
+        self._llm = None
+        self._executor: Optional[concurrent.futures.ThreadPoolExecutor] = None
+
+    def _sanitize_model_config(
+        self, llamacpp_model_config: Optional[LlamaCppModelConfig]
+    ) -> LlamaCppModelConfig:
+        if llamacpp_model_config is None:
+            llamacpp_model_config = LlamaCppModelConfig()
+
+        if self.model_family.context_length:
+            llamacpp_model_config.setdefault("n_ctx", self.model_family.context_length)
+        llamacpp_model_config.setdefault("use_mmap", False)
+        llamacpp_model_config.setdefault("use_mlock", True)
+
+        if (
+            "llama-2" in self.model_family.model_name
+            and self.model_spec.model_size_in_billions == 70
+        ):
+            llamacpp_model_config["use_mlock"] = False
+            llamacpp_model_config["n_gqa"] = 8
+
+        if self._is_darwin_and_apple_silicon():
+            llamacpp_model_config.setdefault("n_gpu_layers", -1)
+        elif self._is_linux():
+            llamacpp_model_config.setdefault("n_gpu_layers", -1)
+
+        return llamacpp_model_config
+
+    def _sanitize_generate_config(
+        self, generate_config: Optional[LlamaCppGenerateConfig]
+    ) -> LlamaCppGenerateConfig:
+        if generate_config is None:
+            generate_config = LlamaCppGenerateConfig(
+                **CreateCompletionLlamaCpp().dict()
+            )
+        else:
+            from llama_cpp import LlamaGrammar
+
+            grammar = generate_config.get("grammar")
+            if grammar is not None and not isinstance(grammar, LlamaGrammar):
+                generate_config["grammar"] = LlamaGrammar.from_string(
+                    generate_config["grammar"]
+                )
+            # Validate generate_config and fill default values to the generate config.
+            generate_config = LlamaCppGenerateConfig(
+                **CreateCompletionLlamaCpp(**generate_config).dict()
+            )
+        # Currently, llama.cpp does not support lora
+        generate_config.pop("lora_name", None)  # type: ignore
+        return generate_config
+
+    @classmethod
+    def match(
+        cls, llm_family: LLMFamilyV1, llm_spec: LLMSpecV1, quantization: str
+    ) -> bool:
+        if llm_spec.model_format not in ["ggufv2"]:
+            return False
+        if (
+            "chat" not in llm_family.model_ability
+            and "generate" not in llm_family.model_ability
+        ):
+            return False
+        return True
+
+    def load(self):
+        try:
+            from xllamacpp import CommonParams, Server
+        except ImportError:
+            error_message = "Failed to import module 'xllamacpp'"
+            installation_guide = ["Please make sure 'xllamacpp' is installed. "]
+
+            raise ImportError(f"{error_message}\n\n{''.join(installation_guide)}")
+
+        if os.path.isfile(self.model_path):
+            # mostly passed from --model_path
+            model_path = os.path.realpath(self.model_path)
+        else:
+            # handle legacy cache.
+            model_path = os.path.realpath(
+                os.path.join(
+                    self.model_path,
+                    self.model_spec.model_file_name_template.format(
+                        quantization=self.quantization
+                    ),
+                )
+            )
+            legacy_model_file_path = os.path.join(self.model_path, "model.bin")
+            if os.path.exists(legacy_model_file_path):
+                model_path = legacy_model_file_path
+
+        try:
+            params = CommonParams()
+            params.model = model_path
+            if self.model_family.chat_template:
+                params.chat_template = self.model_family.chat_template
+            # This is the default value, could be overwritten by _llamacpp_model_config
+            params.n_parallel = os.cpu_count()
+            for k, v in self._llamacpp_model_config.items():
+                try:
+                    setattr(params, k, v)
+                except Exception as e:
+                    logger.error("Failed to set the param %s = %s, error: %s", k, v, e)
+            n_threads = self._llamacpp_model_config.get("n_threads", os.cpu_count())
+            params.cpuparams.n_threads = n_threads
+            params.cpuparams_batch.n_threads = n_threads
+            if params.n_gpu_layers == -1:
+                # Number of layers to offload to GPU (-ngl). If -1, all layers are offloaded.
+                # 0x7FFFFFFF is INT32 max, will be auto set to all layers
+                params.n_gpu_layers = 0x7FFFFFFF
+            self._llm = Server(params)
+            self._executor = concurrent.futures.ThreadPoolExecutor(
+                max_workers=max(10, n_threads)
+            )
+        except AssertionError:
+            raise RuntimeError(f"Load model {self.model_family.model_name} failed")
+
+    def generate(
+        self, prompt: str, generate_config: Optional[LlamaCppGenerateConfig] = None
+    ) -> Union[Completion, Iterator[CompletionChunk]]:
+        generate_config = self._sanitize_generate_config(generate_config)
+        stream = generate_config.get("stream", False)
+        q: queue.Queue = queue.Queue()
+
+        def _handle_completion():
+            # TODO(fyrestone): Replace the LlamaCppGenerateConfig with OpenAI params.
+            data = generate_config
+            data.pop("stopping_criteria", None)
+            data.pop("logits_processor", None)
+            data.pop("suffix", None)
+            data.pop("best_of", None)
+            data.update(
+                {
+                    "prompt": prompt,
+                    "stream": stream,
+                }
+            )
+            prompt_json = orjson.dumps(data)
+
+            def _res_callback(ok):
+                try:
+                    res = orjson.loads(ok)
+                    res["model"] = self.model_uid
+                    q.put(res)
+                except Exception as e:
+                    logger.exception("handle_completions callback failed: %s", e)
+
+            try:
+                self._llm.handle_completions(prompt_json, _res_callback, _res_callback)
+            except Exception as ex:
+                logger.exception("handle_completions failed: %s", ex)
+            q.put(_Sentinel)
+
+        assert self._executor
+        self._executor.submit(_handle_completion)
+
+        if stream:
+
+            def _to_iterator():
+                while (r := q.get()) is not _Sentinel:
+                    yield r
+
+            return _to_iterator()
+        else:
+            return q.get()
+
+    def chat(
+        self,
+        messages: List[Dict],
+        generate_config: Optional[LlamaCppGenerateConfig] = None,
+    ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
+        generate_config = self._sanitize_generate_config(generate_config)
+        stream = generate_config.get("stream", False)
+        tools = generate_config.pop("tools", []) if generate_config else None
+        q: queue.Queue = queue.Queue()
+
+        def _handle_chat_completion():
+            # TODO(fyrestone): Replace the LlamaCppGenerateConfig with OpenAI params.
+            data = generate_config
+            data.pop("stopping_criteria", None)
+            data.pop("logits_processor", None)
+            data.pop("suffix", None)
+            data.pop("best_of", None)
+            data.update(
+                {
+                    "messages": messages,
+                    "stream": stream,
+                    "tools": tools,
+                }
+            )
+            prompt_json = orjson.dumps(data)
+
+            def _res_callback(ok):
+                try:
+                    res = orjson.loads(ok)
+                    res["model"] = self.model_uid
+                    q.put(res)
+                except Exception as e:
+                    logger.exception("handle_chat_completions callback failed: %s", e)
+
+            try:
+                self._llm.handle_chat_completions(
+                    prompt_json, _res_callback, _res_callback
+                )
+            except Exception as ex:
+                logger.exception("handle_chat_completions failed: %s", ex)
+            q.put(_Sentinel)
+
+        assert self._executor
+        self._executor.submit(_handle_chat_completion)
+
+        if stream:
+
+            def _to_iterator():
+                while (r := q.get()) is not _Sentinel:
+                    yield r
+
+            return _to_iterator()
+        else:
+            return q.get()
+
 
 class LlamaCppModel(LLM):
     def __init__(
@@ -76,6 +322,7 @@ class LlamaCppModel(LLM):
             llamacpp_model_config.setdefault("n_gpu_layers", -1)
         elif self._is_linux() and self._can_apply_cublas():
             llamacpp_model_config.setdefault("n_gpu_layers", -1)
+        llamacpp_model_config.setdefault("reasoning_content", False)
 
         return llamacpp_model_config
 
@@ -123,6 +370,9 @@ class LlamaCppModel(LLM):
 
             raise ImportError(f"{error_message}\n\n{''.join(installation_guide)}")
 
+        reasoning_content = self._llamacpp_model_config.pop("reasoning_content")
+        self.prepare_parse_reasoning_content(reasoning_content)
+
         if os.path.isfile(self.model_path):
             # mostly passed from --model_path
             model_path = os.path.realpath(self.model_path)
@@ -292,10 +542,17 @@ class LlamaCppChatModel(LlamaCppModel, ChatModelMixin):
         if stream:
             it = self.generate(full_prompt, generate_config)
             assert isinstance(it, Iterator)
-            return self._to_chat_completion_chunks(it)
+            return self._to_chat_completion_chunks(it, self.reasoning_parser)
         else:
             c = self.generate(full_prompt, generate_config)
             assert not isinstance(c, Iterator)
             if tools:
-                return self._tool_calls_completion(self.model_family, self.model_uid, c)
-            return self._to_chat_completion(c)
+                return self._post_process_completion(
+                    self.model_family, self.model_uid, c, self.reasoning_parser
+                )
+            return self._to_chat_completion(c, self.reasoning_parser)
+
+
+if USE_XLLAMACPP:
+    LlamaCppModel = XllamaCppModel  # type: ignore  # noqa: F811
+    LlamaCppChatModel = XllamaCppModel  # type: ignore  # noqa: F811