xinference 1.7.0__py3-none-any.whl → 1.7.1__py3-none-any.whl

xinference/client/restful/restful_client.py

@@ -12,18 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import json
-import typing
 from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Union
 
 import requests
 
-from ..common import streaming_response_iterator
+from ..common import convert_float_to_int_or_str, streaming_response_iterator
 
 if TYPE_CHECKING:
     from ...types import (
         ChatCompletion,
         ChatCompletionChunk,
-        ChatCompletionMessage,
         Completion,
         CompletionChunk,
         Embedding,
@@ -33,17 +31,6 @@ if TYPE_CHECKING:
     )
 
 
-def convert_float_to_int_or_str(model_size: float) -> Union[int, str]:
-    """convert float to int or string
-
-    if float can be presented as int, convert it to int, otherwise convert it to string
-    """
-    if int(model_size) == model_size:
-        return int(model_size)
-    else:
-        return str(model_size)
-
-
 def _get_error_string(response: requests.Response) -> str:
     try:
         if response.content:
@@ -57,25 +44,6 @@ def _get_error_string(response: requests.Response) -> str:
     return "Unknown error"
 
 
-@typing.no_type_check
-def handle_system_prompts(
-    chat_history: List["ChatCompletionMessage"], system_prompt: Optional[str]
-) -> List["ChatCompletionMessage"]:
-    history_system_prompts = [
-        ch["content"] for ch in chat_history if ch["role"] == "system"
-    ]
-    if system_prompt is not None:
-        history_system_prompts.append(system_prompt)
-
-    # remove all the system prompt in the chat_history
-    chat_history = list(filter(lambda x: x["role"] != "system", chat_history))
-    # insert all system prompts at the beginning
-    chat_history.insert(
-        0, {"role": "system", "content": ". ".join(history_system_prompts)}
-    )
-    return chat_history
-
-
 class RESTfulModelHandle:
     """
     A sync model interface (for RESTful client) which provides type hints that makes it much easier to use xinference
@@ -86,6 +54,19 @@ class RESTfulModelHandle:
         self._model_uid = model_uid
         self._base_url = base_url
         self.auth_headers = auth_headers
+        self.session = requests.Session()
+
+    def close(self):
+        """
+        Close the session.
+        """
+        if self.session:
+            self.session.close()
+            self.session = None
+
+    def __del__(self):
+        if self.session:
+            self.close()
 
 
 class RESTfulEmbeddingModelHandle(RESTfulModelHandle):
@@ -116,7 +97,7 @@ class RESTfulEmbeddingModelHandle(RESTfulModelHandle):
             "input": input,
         }
         request_body.update(kwargs)
-        response = requests.post(url, json=request_body, headers=self.auth_headers)
+        response = self.session.post(url, json=request_body, headers=self.auth_headers)
         if response.status_code != 200:
             raise RuntimeError(
                 f"Failed to create the embeddings, detail: {_get_error_string(response)}"
@@ -154,7 +135,7 @@ class RESTfulEmbeddingModelHandle(RESTfulModelHandle):
             "input": input,
         }
         request_body.update(kwargs)
-        response = requests.post(url, json=request_body, headers=self.auth_headers)
+        response = self.session.post(url, json=request_body, headers=self.auth_headers)
         if response.status_code != 200:
             raise RuntimeError(
                 f"Failed to decode token ids, detail: {_get_error_string(response)}"
@@ -213,7 +194,7 @@ class RESTfulRerankModelHandle(RESTfulModelHandle):
             "kwargs": json.dumps(kwargs),
         }
         request_body.update(kwargs)
-        response = requests.post(url, json=request_body, headers=self.auth_headers)
+        response = self.session.post(url, json=request_body, headers=self.auth_headers)
         if response.status_code != 200:
             raise RuntimeError(
                 f"Failed to rerank documents, detail: {response.json()['detail']}"
@@ -258,7 +239,7 @@ class RESTfulImageModelHandle(RESTfulModelHandle):
             "response_format": response_format,
             "kwargs": json.dumps(kwargs),
         }
-        response = requests.post(url, json=request_body, headers=self.auth_headers)
+        response = self.session.post(url, json=request_body, headers=self.auth_headers)
         if response.status_code != 200:
             raise RuntimeError(
                 f"Failed to create the images, detail: {_get_error_string(response)}"
@@ -322,7 +303,7 @@ class RESTfulImageModelHandle(RESTfulModelHandle):
         for key, value in params.items():
             files.append((key, (None, value)))
         files.append(("image", ("image", image, "application/octet-stream")))
-        response = requests.post(url, files=files, headers=self.auth_headers)
+        response = self.session.post(url, files=files, headers=self.auth_headers)
         if response.status_code != 200:
             raise RuntimeError(
                 f"Failed to variants the images, detail: {_get_error_string(response)}"
@@ -397,7 +378,7 @@ class RESTfulImageModelHandle(RESTfulModelHandle):
         files.append(
             ("mask_image", ("mask_image", mask_image, "application/octet-stream"))
         )
-        response = requests.post(url, files=files, headers=self.auth_headers)
+        response = self.session.post(url, files=files, headers=self.auth_headers)
         if response.status_code != 200:
             raise RuntimeError(
                 f"Failed to inpaint the images, detail: {_get_error_string(response)}"
@@ -416,7 +397,7 @@ class RESTfulImageModelHandle(RESTfulModelHandle):
         for key, value in params.items():
             files.append((key, (None, value)))
         files.append(("image", ("image", image, "application/octet-stream")))
-        response = requests.post(url, files=files, headers=self.auth_headers)
+        response = self.session.post(url, files=files, headers=self.auth_headers)
         if response.status_code != 200:
             raise RuntimeError(
                 f"Failed to ocr the images, detail: {_get_error_string(response)}"
@@ -454,7 +435,7 @@ class RESTfulVideoModelHandle(RESTfulModelHandle):
             "n": n,
             "kwargs": json.dumps(kwargs),
         }
-        response = requests.post(url, json=request_body, headers=self.auth_headers)
+        response = self.session.post(url, json=request_body, headers=self.auth_headers)
         if response.status_code != 200:
             raise RuntimeError(
                 f"Failed to create the video, detail: {_get_error_string(response)}"
@@ -501,7 +482,7 @@ class RESTfulVideoModelHandle(RESTfulModelHandle):
         for key, value in params.items():
             files.append((key, (None, value)))
         files.append(("image", ("image", image, "application/octet-stream")))
-        response = requests.post(url, files=files, headers=self.auth_headers)
+        response = self.session.post(url, files=files, headers=self.auth_headers)
         if response.status_code != 200:
             raise RuntimeError(
                 f"Failed to create the video from image, detail: {_get_error_string(response)}"
@@ -554,7 +535,7 @@ class RESTfulVideoModelHandle(RESTfulModelHandle):
             ("first_frame", ("image", first_frame, "application/octet-stream"))
         )
         files.append(("last_frame", ("image", last_frame, "application/octet-stream")))
-        response = requests.post(url, files=files, headers=self.auth_headers)
+        response = self.session.post(url, files=files, headers=self.auth_headers)
         if response.status_code != 200:
             raise RuntimeError(
                 f"Failed to create the video from image, detail: {_get_error_string(response)}"
@@ -604,7 +585,7 @@ class RESTfulGenerateModelHandle(RESTfulModelHandle):
 
         stream = bool(generate_config and generate_config.get("stream"))
 
-        response = requests.post(
+        response = self.session.post(
             url, json=request_body, stream=stream, headers=self.auth_headers
         )
         if response.status_code != 200:
@@ -665,7 +646,7 @@ class RESTfulChatModelHandle(RESTfulGenerateModelHandle):
                 request_body[key] = value
 
         stream = bool(generate_config and generate_config.get("stream"))
-        response = requests.post(
+        response = self.session.post(
             url, json=request_body, stream=stream, headers=self.auth_headers
         )
 
@@ -736,7 +717,7 @@ class RESTfulAudioModelHandle(RESTfulModelHandle):
         }
         files: List[Any] = []
         files.append(("file", ("file", audio, "application/octet-stream")))
-        response = requests.post(
+        response = self.session.post(
             url, data=params, files=files, headers=self.auth_headers
         )
         if response.status_code != 200:
@@ -799,7 +780,7 @@ class RESTfulAudioModelHandle(RESTfulModelHandle):
         }
         files: List[Any] = []
         files.append(("file", ("file", audio, "application/octet-stream")))
-        response = requests.post(
+        response = self.session.post(
             url, data=params, files=files, headers=self.auth_headers
         )
         if response.status_code != 200:
@@ -873,11 +854,11 @@ class RESTfulAudioModelHandle(RESTfulModelHandle):
                 )
             )
         if files:
-            response = requests.post(
+            response = self.session.post(
                 url, data=params, files=files, headers=self.auth_headers, stream=stream
             )
         else:
-            response = requests.post(
+            response = self.session.post(
                 url, json=params, headers=self.auth_headers, stream=stream
             )
         if response.status_code != 200:
@@ -894,6 +875,7 @@ class RESTfulAudioModelHandle(RESTfulModelHandle):
 class RESTfulFlexibleModelHandle(RESTfulModelHandle):
     def infer(
         self,
+        *args,
         **kwargs,
     ):
         """
@@ -914,16 +896,17 @@ class RESTfulFlexibleModelHandle(RESTfulModelHandle):
         url = f"{self._base_url}/v1/flexible/infers"
         params = {
             "model": self._model_uid,
+            "args": args,
         }
         params.update(kwargs)
 
-        response = requests.post(url, json=params, headers=self.auth_headers)
+        response = self.session.post(url, json=params, headers=self.auth_headers)
         if response.status_code != 200:
             raise RuntimeError(
                 f"Failed to predict, detail: {_get_error_string(response)}"
             )
 
-        return response.content
+        return response.json()
 
 
 class Client:
@@ -931,10 +914,23 @@ class Client:
         self.base_url = base_url
         self._headers: Dict[str, str] = {}
         self._cluster_authed = False
+        self.session = requests.Session()
         self._check_cluster_authenticated()
         if api_key is not None and self._cluster_authed:
             self._headers["Authorization"] = f"Bearer {api_key}"
 
+    def close(self):
+        """
+        Close the session.
+        """
+        if self.session:
+            self.session.close()
+            self.session = None
+
+    def __del__(self):
+        if self.session:
+            self.close()
+
     def _set_token(self, token: Optional[str]):
         if not self._cluster_authed or token is None:
             return
@@ -949,7 +945,7 @@ class Client:
 
     def _check_cluster_authenticated(self):
         url = f"{self.base_url}/v1/cluster/auth"
-        response = requests.get(url)
+        response = self.session.get(url)
         # compatible with old version of xinference
         if response.status_code == 404:
             self._cluster_authed = False
@@ -963,7 +959,7 @@ class Client:
 
     def vllm_models(self) -> Dict[str, Any]:
         url = f"{self.base_url}/v1/models/vllm-supported"
-        response = requests.get(url, headers=self._headers)
+        response = self.session.get(url, headers=self._headers)
         if response.status_code != 200:
             raise RuntimeError(
                 f"Failed to fetch VLLM models. detail: {response.json()['detail']}"
@@ -981,7 +977,7 @@ class Client:
 
         payload = {"username": username, "password": password}
 
-        response = requests.post(url, json=payload)
+        response = self.session.post(url, json=payload)
         if response.status_code != 200:
             raise RuntimeError(f"Failed to login, detail: {response.json()['detail']}")
 
@@ -1003,7 +999,7 @@ class Client:
 
         url = f"{self.base_url}/v1/models"
 
-        response = requests.get(url, headers=self._headers)
+        response = self.session.get(url, headers=self._headers)
         if response.status_code != 200:
             raise RuntimeError(
                 f"Failed to list model, detail: {_get_error_string(response)}"
@@ -1111,9 +1107,9 @@ class Client:
             payload[str(key)] = value
 
         if wait_ready:
-            response = requests.post(url, json=payload, headers=self._headers)
+            response = self.session.post(url, json=payload, headers=self._headers)
         else:
-            response = requests.post(
+            response = self.session.post(
                 url, json=payload, headers=self._headers, params={"wait_ready": False}
             )
         if response.status_code != 200:
@@ -1142,7 +1138,7 @@ class Client:
 
         url = f"{self.base_url}/v1/models/{model_uid}"
 
-        response = requests.delete(url, headers=self._headers)
+        response = self.session.delete(url, headers=self._headers)
         if response.status_code != 200:
             raise RuntimeError(
                 f"Failed to terminate model, detail: {_get_error_string(response)}"
@@ -1169,7 +1165,7 @@ class Client:
         """
         url = f"{self.base_url}/v1/models/{model_uid}/progress"
 
-        response = requests.get(url, headers=self._headers)
+        response = self.session.get(url, headers=self._headers)
         if response.status_code != 200:
             raise RuntimeError(
                 f"Fail to get model launching progress, detail: {_get_error_string(response)}"
@@ -1192,7 +1188,7 @@ class Client:
         """
         url = f"{self.base_url}/v1/models/{model_uid}/cancel"
 
-        response = requests.post(url, headers=self._headers)
+        response = self.session.post(url, headers=self._headers)
         if response.status_code != 200:
             raise RuntimeError(
                 f"Fail to cancel launching model, detail: {_get_error_string(response)}"
@@ -1200,7 +1196,7 @@ class Client:
 
     def get_instance_info(self, model_name: str, model_uid: str):
         url = f"{self.base_url}/v1/models/instances"
-        response = requests.get(
+        response = self.session.get(
             url,
             headers=self._headers,
             params={"model_name": model_name, "model_uid": model_uid},
@@ -1212,9 +1208,9 @@ class Client:
 
     def _get_supervisor_internal_address(self):
         url = f"{self.base_url}/v1/address"
-        response = requests.get(url, headers=self._headers)
+        response = self.session.get(url, headers=self._headers)
         if response.status_code != 200:
-            raise RuntimeError(f"Failed to get supervisor internal address")
+            raise RuntimeError("Failed to get supervisor internal address")
         response_data = response.json()
         return response_data
 
@@ -1243,7 +1239,7 @@ class Client:
         """
 
         url = f"{self.base_url}/v1/models/{model_uid}"
-        response = requests.get(url, headers=self._headers)
+        response = self.session.get(url, headers=self._headers)
         if response.status_code != 200:
             raise RuntimeError(
                 f"Failed to get the model description, detail: {_get_error_string(response)}"
@@ -1331,7 +1327,7 @@ class Client:
         """
 
         url = f"{self.base_url}/v1/models/{model_uid}"
-        response = requests.get(url, headers=self._headers)
+        response = self.session.get(url, headers=self._headers)
         if response.status_code != 200:
             raise RuntimeError(
                 f"Failed to get the model description, detail: {_get_error_string(response)}"
@@ -1366,7 +1362,7 @@ class Client:
         """
         url = f"{self.base_url}/v1/model_registrations/{model_type}"
         request_body = {"model": model, "worker_ip": worker_ip, "persist": persist}
-        response = requests.post(url, json=request_body, headers=self._headers)
+        response = self.session.post(url, json=request_body, headers=self._headers)
         if response.status_code != 200:
             raise RuntimeError(
                 f"Failed to register model, detail: {_get_error_string(response)}"
@@ -1392,7 +1388,7 @@ class Client:
             Report failure to unregister the custom model. Provide details of failure through error message.
         """
         url = f"{self.base_url}/v1/model_registrations/{model_type}/{model_name}"
-        response = requests.delete(url, headers=self._headers)
+        response = self.session.delete(url, headers=self._headers)
         if response.status_code != 200:
             raise RuntimeError(
                 f"Failed to register model, detail: {_get_error_string(response)}"
@@ -1422,7 +1418,7 @@ class Client:
 
         """
         url = f"{self.base_url}/v1/model_registrations/{model_type}"
-        response = requests.get(url, headers=self._headers)
+        response = self.session.get(url, headers=self._headers)
         if response.status_code != 200:
             raise RuntimeError(
                 f"Failed to list model registration, detail: {_get_error_string(response)}"
@@ -1459,7 +1455,7 @@ class Client:
             "model_name": model_name,
             "worker_ip": worker_ip,
         }
-        response = requests.get(url, headers=self._headers, params=params)
+        response = self.session.get(url, headers=self._headers, params=params)
         if response.status_code != 200:
             raise RuntimeError(
                 f"Failed to list cached model, detail: {_get_error_string(response)}"
@@ -1490,7 +1486,7 @@ class Client:
             "model_version": model_version,
             "worker_ip": worker_ip,
         }
-        response = requests.get(url, headers=self._headers, params=params)
+        response = self.session.get(url, headers=self._headers, params=params)
         if response.status_code != 200:
             raise RuntimeError(
                 f"Failed to get paths by model name, detail: {_get_error_string(response)}"
@@ -1520,7 +1516,7 @@ class Client:
             "model_version": model_version,
             "worker_ip": worker_ip,
         }
-        response = requests.delete(url, headers=self._headers, params=params)
+        response = self.session.delete(url, headers=self._headers, params=params)
         if response.status_code != 200:
             raise RuntimeError(
                 f"Failed to remove cached models, detail: {_get_error_string(response)}"
@@ -1548,7 +1544,7 @@ class Client:
             The collection of registered models on the server.
         """
         url = f"{self.base_url}/v1/model_registrations/{model_type}/{model_name}"
-        response = requests.get(url, headers=self._headers)
+        response = self.session.get(url, headers=self._headers)
         if response.status_code != 200:
             raise RuntimeError(
                 f"Failed to list model registration, detail: {_get_error_string(response)}"
@@ -1578,7 +1574,7 @@ class Client:
             url = f"{self.base_url}/v1/engines/{model_name}"
         else:
             url = f"{self.base_url}/v1/engines/{model_type}/{model_name}"
-        response = requests.get(url, headers=self._headers)
+        response = self.session.get(url, headers=self._headers)
         if response.status_code != 200:
             raise RuntimeError(
                 f"Failed to query engine parameters by model name, detail: {_get_error_string(response)}"
@@ -1608,7 +1604,7 @@ class Client:
             Return empty dict.
         """
         url = f"{self.base_url}/v1/models/{model_uid}/requests/{request_id}/abort"
-        response = requests.post(
+        response = self.session.post(
             url, headers=self._headers, json={"block_duration": block_duration}
         )
         if response.status_code != 200:
@@ -1621,7 +1617,7 @@ class Client:
 
     def get_workers_info(self):
         url = f"{self.base_url}/v1/workers"
-        response = requests.get(url, headers=self._headers)
+        response = self.session.get(url, headers=self._headers)
         if response.status_code != 200:
             raise RuntimeError(
                 f"Failed to get workers info, detail: {_get_error_string(response)}"
@@ -1631,7 +1627,7 @@ class Client:
 
     def get_supervisor_info(self):
         url = f"{self.base_url}/v1/supervisor"
-        response = requests.get(url, headers=self._headers)
+        response = self.session.get(url, headers=self._headers)
         if response.status_code != 200:
             raise RuntimeError(
                 f"Failed to get supervisor info, detail: {_get_error_string(response)}"
@@ -1641,7 +1637,7 @@ class Client:
 
     def get_progress(self, request_id: str):
         url = f"{self.base_url}/v1/requests/{request_id}/progress"
-        response = requests.get(url, headers=self._headers)
+        response = self.session.get(url, headers=self._headers)
         if response.status_code != 200:
             raise RuntimeError(
                 f"Failed to get progress, detail: {_get_error_string(response)}"
@@ -1651,7 +1647,7 @@ class Client:
 
     def abort_cluster(self):
         url = f"{self.base_url}/v1/clusters"
-        response = requests.delete(url, headers=self._headers)
+        response = self.session.delete(url, headers=self._headers)
         if response.status_code != 200:
             raise RuntimeError(
                 f"Failed to abort cluster, detail: {_get_error_string(response)}"

xinference/core/media_interface.py

@@ -16,6 +16,7 @@ import base64
 import io
 import logging
 import os
+import tempfile
 import threading
 import time
 import uuid
@@ -784,7 +785,8 @@ class MediaInterface:
             )
 
             # Write to a temp .mp3 file and return its path
-            audio_path = f"/tmp/{uuid.uuid4()}.mp3"
+            temp_dir = tempfile.gettempdir()
+            audio_path = os.path.join(temp_dir, f"{uuid.uuid4()}.mp3")
             with open(audio_path, "wb") as f:
                 f.write(response)
 

xinference/core/model.py

@@ -160,10 +160,6 @@ def oom_check(fn):
 class ModelActor(xo.StatelessActor, CancelMixin):
     _replica_model_uid: Optional[str]
 
-    @classmethod
-    def gen_uid(cls, model: "LLM"):
-        return f"{model.__class__}-model-actor"
-
     async def __pre_destroy__(self):
         from ..model.embedding.core import EmbeddingModel
         from ..model.llm.sglang.core import SGLANGModel
@@ -318,6 +314,9 @@ class ModelActor(xo.StatelessActor, CancelMixin):
     def __repr__(self) -> str:
         return f"ModelActor({self._replica_model_uid})"
 
+    def __getattr__(self, attr: str):
+        return getattr(self._model, attr)
+
     def decrease_serve_count(self):
         self._serve_count -= 1
 
@@ -1223,12 +1222,14 @@ class ModelActor(xo.StatelessActor, CancelMixin):
     @log_async(logger=logger, ignore_kwargs=["image"])
     async def infer(
         self,
+        *args,
         **kwargs,
     ):
         kwargs.pop("request_id", None)
         if hasattr(self._model, "infer"):
             return await self._call_wrapper_json(
                 self._model.infer,
+                *args,
                 **kwargs,
             )
         raise AttributeError(

xinference/core/supervisor.py

@@ -348,15 +348,20 @@ class SupervisorActor(xo.StatelessActor):
             BUILTIN_LLM_MODEL_TOOL_CALL_FAMILIES,
         )
 
+        to_filter_abilities = ["vision", "reasoning", "audio", "omni", "hybrid"]
+        ability_to_names: Dict[str, List[str]] = {
+            ability: [] for ability in to_filter_abilities
+        }
+        for family in BUILTIN_LLM_FAMILIES:
+            for ability in to_filter_abilities:
+                if ability in family.model_ability:
+                    ability_to_names[ability].append(family.model_name)
+
         return {
             "chat": list(BUILTIN_LLM_MODEL_CHAT_FAMILIES),
             "generate": list(BUILTIN_LLM_MODEL_GENERATE_FAMILIES),
             "tools": list(BUILTIN_LLM_MODEL_TOOL_CALL_FAMILIES),
-            "vision": [
-                family.model_name
-                for family in BUILTIN_LLM_FAMILIES
-                if "vision" in family.model_ability
-            ],
+            **ability_to_names,
         }
 
     async def get_devices_count(self) -> int:

xinference/core/worker.py

@@ -15,10 +15,12 @@
 import asyncio
 import logging
 import os
+import pathlib
 import platform
 import queue
 import shutil
 import signal
+import sys
 import threading
 import time
 from collections import defaultdict
@@ -809,7 +811,13 @@ class WorkerActor(xo.StatelessActor):
             virtual_env_name or "uv", env_path
         )
         # create env
-        virtual_env_manager.create_env()
+        python_path = None
+        if not hasattr(sys, "_MEIPASS"):
+            # not in pyinstaller
+            # we specify python_path explicitly
+            # sometimes uv would find other versions.
+            python_path = pathlib.Path(sys.executable)
+        virtual_env_manager.create_env(python_path=python_path)
         return virtual_env_manager
 
     @classmethod
@@ -829,25 +837,18 @@ class WorkerActor(xo.StatelessActor):
                 if hasattr(settings, k) and not getattr(settings, k):
                     setattr(settings, k, v)
 
+        conf = dict(settings)
         packages = settings.packages
-        index_url = settings.index_url
-        extra_index_url = settings.extra_index_url
-        find_links = settings.find_links
-        trusted_host = settings.trusted_host
+        conf.pop("packages", None)
+        conf.pop("inherit_pip_config", None)
 
         logger.info(
-            "Installing packages %s in virtual env %s, with settings(index_url=%s)",
+            "Installing packages %s in virtual env %s, with settings(%s)",
             packages,
             virtual_env_manager.env_path,
-            index_url,
-        )
-        virtual_env_manager.install_packages(
-            packages,
-            index_url=index_url,
-            extra_index_url=extra_index_url,
-            find_links=find_links,
-            trusted_host=trusted_host,
+            ", ".join([f"{k}={v}" for k, v in conf.items() if v]),
         )
+        virtual_env_manager.install_packages(packages, **conf)
 
     async def _get_progressor(self, request_id: str):
         from .progress_tracker import Progressor, ProgressTrackerActor