xinference 1.7.0__py3-none-any.whl → 1.7.1__py3-none-any.whl

xinference/model/flexible/launchers/transformers_launcher.py

@@ -18,7 +18,7 @@ from ..core import FlexibleModel, FlexibleModelSpec
 
 
 class MockModel(FlexibleModel):
-    def infer(self, **kwargs):
+    def infer(self, *args, **kwargs):
         return kwargs
 
 
@@ -27,8 +27,8 @@ class AutoModel(FlexibleModel):
         config = self.config or {}
         self._pipeline = pipeline(model=self.model_path, device=self.device, **config)
 
-    def infer(self, **kwargs):
-        return self._pipeline(**kwargs)
+    def infer(self, *args, **kwargs):
+        return self._pipeline(*args, **kwargs)
 
 
 class TransformersTextClassificationModel(FlexibleModel):
@@ -37,8 +37,8 @@ class TransformersTextClassificationModel(FlexibleModel):
 
         self._pipeline = pipeline(model=self._model_path, device=self._device, **config)
 
-    def infer(self, **kwargs):
-        return self._pipeline(**kwargs)
+    def infer(self, *args, **kwargs):
+        return self._pipeline(*args, **kwargs)
 
 
 def launcher(model_uid: str, model_spec: FlexibleModelSpec, **kwargs) -> FlexibleModel:

xinference/model/flexible/launchers/yolo_launcher.py

@@ -0,0 +1,62 @@
+# Copyright 2022-2025 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base64
+import inspect
+import io
+import json
+
+import PIL.Image
+
+from ..core import FlexibleModel, FlexibleModelSpec
+
+
+class UltralyticsModel(FlexibleModel):
+    def load(self):
+        from ultralytics import YOLO
+
+        config = dict(self.config or {})
+        if self._device:
+            config["device"] = self._device
+
+        self._model = YOLO(model=self._model_path, **config)
+
+    def infer(self, *args, **kwargs):
+        predict_func = self._model.predict
+
+        sig = inspect.signature(predict_func)
+        bound_args = sig.bind_partial(*args, **kwargs)  # 或 bind() 视场景选择
+        bound_args.apply_defaults()
+
+        if "source" in bound_args.arguments:
+            source = bound_args.arguments["source"]
+            decoded = base64.b64decode(source)
+            img = PIL.Image.open(io.BytesIO(decoded))
+
+            bound_args.arguments["source"] = img
+
+        results = predict_func(*bound_args.args, **bound_args.kwargs)
+        return [json.loads(r.to_json()) for r in results]
+
+
+def launcher(model_uid: str, model_spec: FlexibleModelSpec, **kwargs) -> FlexibleModel:
+    device = kwargs.get("device")
+
+    model_path = model_spec.model_uri
+    if model_path is None:
+        raise ValueError("model_path required")
+
+    return UltralyticsModel(
+        model_uid=model_uid, model_path=model_path, device=device, config=kwargs
+    )

xinference/model/llm/__init__.py

@@ -152,6 +152,13 @@ def load_model_family_from_json(json_filename, target_families):
                     "stop_token_ids": model_spec.stop_token_ids,
                     "stop": model_spec.stop,
                 }
+                if model_spec.reasoning_start_tag and model_spec.reasoning_end_tag:
+                    BUILTIN_LLM_PROMPT_STYLE[model_spec.model_name][
+                        "reasoning_start_tag"
+                    ] = model_spec.reasoning_start_tag
+                    BUILTIN_LLM_PROMPT_STYLE[model_spec.model_name][
+                        "reasoning_end_tag"
+                    ] = model_spec.reasoning_end_tag
 
         # register model family
         if "chat" in model_spec.model_ability:

xinference/model/llm/core.py

@@ -20,6 +20,7 @@ import platform
 import warnings
 from abc import abstractmethod
 from collections import defaultdict
+from contextvars import ContextVar
 from functools import lru_cache
 from typing import TYPE_CHECKING, Dict, List, Literal, Optional, Tuple, Union
 
@@ -105,9 +106,14 @@ class LLM(abc.ABC):
     @staticmethod
     @lru_cache
     def _get_cuda_count():
+        from ...device_utils import get_available_device_env_name
         from ...utils import cuda_count
 
-        cuda_visible_devices = os.getenv("CUDA_VISIBLE_DEVICES", None)
+        env_name = get_available_device_env_name()
+        if env_name is None:
+            return cuda_count()
+
+        cuda_visible_devices = os.getenv(env_name, None)
         if cuda_visible_devices is None:
             return cuda_count()
 
@@ -152,6 +158,17 @@ class LLM(abc.ABC):
         )
 
 
+# Context variable for passing per-request chat context (e.g., chat_template_kwargs).
+# This variable should be set at the beginning of each chat or stream_chat call.
+# It allows downstream components (e.g., reasoning_parser) to access request-specific
+# settings like 'enable_thinking', without requiring those values to be passed explicitly
+# through every function layer.
+#
+# The context is automatically isolated per thread or coroutine, so concurrent requests
+# will not interfere with each other.
+chat_context_var: ContextVar[dict] = ContextVar("chat_context_var", default={})
+
+
 class LLMDescription(ModelDescription):
     def __init__(
         self,

xinference/model/llm/llama_cpp/core.py

@@ -162,7 +162,7 @@ class XllamaCppModel(LLM, ChatModelMixin):
             if self.model_family.chat_template:
                 params.chat_template = self.model_family.chat_template
             # This is the default value, could be overwritten by _llamacpp_model_config
-            params.n_parallel = os.cpu_count()
+            params.n_parallel = min(8, os.cpu_count() or 1)
             for k, v in self._llamacpp_model_config.items():
                 try:
                     if "." in k:

xinference/model/llm/llm_family.json

@@ -6160,7 +6160,7 @@
         "quantizations": [
           "none"
         ],
-        "model_id": "openbmb/MiniCPM4-0.5B"
+        "model_id": "JunHowie/MiniCPM4-0.5B"
       },
       {
         "model_format": "pytorch",
@@ -6168,7 +6168,7 @@
         "quantizations": [
           "none"
         ],
-        "model_id": "openbmb/MiniCPM4-8B"
+        "model_id": "JunHowie/MiniCPM4-8B"
       },
       {
         "model_format": "mlx",
@@ -10518,8 +10518,48 @@
       "packages": [
         "transformers>=4.51.0",
         "mlx-lm>=0.24.0 ; sys_platform=='darwin'",
-        "numpy==1.26.4"
+        "#system_numpy#"
       ]
     }
+  },
+  {
+    "version": 1,
+    "context_length": 32768,
+    "model_name": "qwenLong-l1",
+    "model_lang": [
+      "en",
+      "zh"
+    ],
+    "model_ability": [
+      "chat"
+    ],
+    "model_description": "QwenLong-L1: Towards Long-Context Large Reasoning Models with Reinforcement Learning",
+    "model_specs": [
+      {
+        "model_format": "pytorch",
+        "model_size_in_billions": 32,
+        "quantizations": [
+          "none"
+        ],
+        "model_id": "Tongyi-Zhiwen/QwenLong-L1-32B"
+      },
+      {
+        "model_format": "awq",
+        "model_size_in_billions": 32,
+        "quantizations": [
+          "Int4"
+        ],
+        "model_id": "Tongyi-Zhiwen/QwenLong-L1-32B-AWQ"
+      }
+    ],
+    "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='') %}{%- for message in messages %}{%- if message['role'] == 'system' %}{% set ns.system_prompt = message['content'] %}{%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{'<｜User｜>' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is none %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls']%}{%- if not ns.is_first %}{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{%- set ns.is_first = true -%}{%- else %}{{'\\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}{%- endif %}{%- endfor %}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is not none %}{%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}{%- set ns.is_tool = false -%}{%- else %}{% set content = message['content'] %}{% if '</think>' in content %}{% set content = content.split('</think>')[-1] %}{% endif %}{{'<｜Assistant｜>' + content + '<｜end▁of▁sentence｜>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- set ns.is_output_first = false %}{%- else %}{{'\\n<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<｜Assistant｜><think>\\n'}}{% endif %}",
+    "stop_token_ids": [
+      151643
+    ],
+    "stop": [
+      "<｜end▁of▁sentence｜>"
+    ],
+    "reasoning_start_tag": "<think>",
+    "reasoning_end_tag": "</think>"
   }
 ]

xinference/model/llm/llm_family.py

@@ -232,6 +232,12 @@ class CustomLLMFamilyV1(LLMFamilyV1):
                 "stop_token_ids"
             ]
             llm_spec.stop = BUILTIN_LLM_PROMPT_STYLE[llm_spec.chat_template]["stop"]
+            llm_spec.reasoning_start_tag = BUILTIN_LLM_PROMPT_STYLE[
+                llm_spec.chat_template
+            ].get("reasoning_start_tag")
+            llm_spec.reasoning_end_tag = BUILTIN_LLM_PROMPT_STYLE[
+                llm_spec.chat_template
+            ].get("reasoning_end_tag")
             llm_spec.chat_template = BUILTIN_LLM_PROMPT_STYLE[llm_spec.chat_template][
                 "chat_template"
             ]

xinference/model/llm/llm_family_modelscope.json

@@ -4295,7 +4295,7 @@
         "quantizations": [
           "none"
         ],
-        "model_id": "OpenBMB/MiniCPM4-0.5B",
+        "model_id": "JunHowie/MiniCPM4-0.5B",
         "model_hub": "modelscope"
       },
       {
@@ -4304,7 +4304,7 @@
         "quantizations": [
           "none"
         ],
-        "model_id": "OpenBMB/MiniCPM4-8B",
+        "model_id": "JunHowie/MiniCPM4-8B",
         "model_hub": "modelscope"
       },
       {
@@ -8651,8 +8651,50 @@
       "packages": [
         "transformers>=4.51.0",
         "mlx-lm>=0.24.0 ; sys_platform=='darwin'",
-        "numpy==1.26.4"
+        "#system_numpy#"
       ]
     }
+  },
+  {
+    "version": 1,
+    "context_length": 32768,
+    "model_name": "qwenLong-l1",
+    "model_lang": [
+      "en",
+      "zh"
+    ],
+    "model_ability": [
+      "chat"
+    ],
+    "model_description": "QwenLong-L1: Towards Long-Context Large Reasoning Models with Reinforcement Learning",
+    "model_specs": [
+      {
+        "model_format": "pytorch",
+        "model_size_in_billions": 32,
+        "quantizations": [
+          "none"
+        ],
+        "model_id": "iic/QwenLong-L1-32B",
+        "model_hub": "modelscope"
+      },
+      {
+        "model_format": "awq",
+        "model_size_in_billions": 32,
+        "quantizations": [
+          "Int4"
+        ],
+        "model_id": "iic/QwenLong-L1-32B-AWQ",
+        "model_hub": "modelscope"
+      }
+    ],
+    "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='') %}{%- for message in messages %}{%- if message['role'] == 'system' %}{% set ns.system_prompt = message['content'] %}{%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{'<｜User｜>' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is none %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls']%}{%- if not ns.is_first %}{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{%- set ns.is_first = true -%}{%- else %}{{'\\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}{%- endif %}{%- endfor %}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is not none %}{%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}{%- set ns.is_tool = false -%}{%- else %}{% set content = message['content'] %}{% if '</think>' in content %}{% set content = content.split('</think>')[-1] %}{% endif %}{{'<｜Assistant｜>' + content + '<｜end▁of▁sentence｜>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- set ns.is_output_first = false %}{%- else %}{{'\\n<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<｜Assistant｜><think>\\n'}}{% endif %}",
+    "stop_token_ids": [
+      151643
+    ],
+    "stop": [
+      "<｜end▁of▁sentence｜>"
+    ],
+    "reasoning_start_tag": "<think>",
+    "reasoning_end_tag": "</think>"
   }
 ]