xinference-client 3.0.0__tar.gz → 3.2.0__tar.gz

{xinference_client-3.0.0 → xinference_client-3.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xinference-client
-Version: 3.0.0
+Version: 3.2.0
 Summary: Client for Xinference
 Home-page: https://github.com/xorbitsai/inference-client
 Author: Qin Xuye

{xinference_client-3.0.0 → xinference_client-3.2.0}/xinference_client/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2026-07-19T01:13:11+0800",
+ "date": "2026-08-15T14:17:59+0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "a9a54d21006dab4780ddaf26c51057541351f950",
- "version": "3.0.0"
+ "full-revisionid": "3ed6a1c10c0b3793ebe05c29c6cc6ba331af318b",
+ "version": "3.2.0"
 }
 '''  # END VERSION_JSON
 

{xinference_client-3.0.0 → xinference_client-3.2.0}/xinference_client/client/restful/async_restful_client.py

@@ -22,6 +22,7 @@ from ..common import async_streaming_response_iterator, convert_float_to_int_or_
 
 if TYPE_CHECKING:
     from ...types import (
+        AudioEmbedding,
         ChatCompletion,
         ChatCompletionChunk,
         Completion,
@@ -601,16 +602,15 @@ class AsyncRESTfulImageModelHandle(AsyncRESTfulModelHandle):
 
     async def ocr(self, image: Union[str, bytes], **kwargs):
         url = f"{self._base_url}/v1/images/ocr"
-        params = {
-            "model": self._model_uid,
-            "kwargs": json.dumps(kwargs),
-        }
-        params = _filter_params(params)
-        files: List[Any] = []
-        for key, value in params.items():
-            files.append((key, (None, value)))
-        files.append(("image", ("image", image, "application/octet-stream")))
-        response = await self.session.post(url, data=files, headers=self.auth_headers)
+        # aiohttp does not understand requests-style (key, (filename, value))
+        # tuples; build a proper multipart form instead.
+        form = aiohttp.FormData()
+        form.add_field("model", self._model_uid)
+        form.add_field("kwargs", json.dumps(kwargs))
+        form.add_field(
+            "image", image, filename="image", content_type="application/octet-stream"
+        )
+        response = await self.session.post(url, data=form, headers=self.auth_headers)
         if response.status != 200:
             raise RuntimeError(
                 f"Failed to ocr the images, detail: {await _get_error_string(response)}"
@@ -667,6 +667,7 @@ class AsyncRESTfulVideoModelHandle(AsyncRESTfulModelHandle):
         prompt: str,
         negative_prompt: Optional[str] = None,
         n: int = 1,
+        video: Optional[Union[str, bytes]] = None,
         **kwargs,
     ) -> "VideoList":
         """
@@ -682,6 +683,8 @@ class AsyncRESTfulVideoModelHandle(AsyncRESTfulModelHandle):
             The prompt or prompts not to guide the image generation.
         n: `int`, defaults to 1
             The number of videos to generate per prompt. Must be between 1 and 10.
+        video: `Union[str, bytes]`, optional
+            The driving video for character animation models.
         Returns
         -------
         VideoList
@@ -696,11 +699,25 @@ class AsyncRESTfulVideoModelHandle(AsyncRESTfulModelHandle):
             "kwargs": json.dumps(kwargs),
         }
         params = _filter_params(params)
-        files: List[Any] = []
+        data = aiohttp.FormData()
         for key, value in params.items():
-            files.append((key, (None, value)))
-        files.append(("image", ("image", image, "application/octet-stream")))
-        response = await self.session.post(url, data=files, headers=self.auth_headers)
+            data.add_field(key, str(value))
+        data.add_field(
+            "image", image, filename="image", content_type="application/octet-stream"
+        )
+        if video is not None:
+            if isinstance(video, str):
+                with open(video, "rb") as f:
+                    video_data = f.read()
+            else:
+                video_data = video
+            data.add_field(
+                "video",
+                video_data,
+                filename="video",
+                content_type="application/octet-stream",
+            )
+        response = await self.session.post(url, data=data, headers=self.auth_headers)
         if response.status != 200:
             raise RuntimeError(
                 f"Failed to create the video from image, detail: {await _get_error_string(response)}"
@@ -916,6 +933,24 @@ class AsyncRESTfulChatModelHandle(AsyncRESTfulGenerateModelHandle):
 
 
 class AsyncRESTfulAudioModelHandle(AsyncRESTfulModelHandle):
+    async def create_embedding(self, audio: bytes) -> "AudioEmbedding":
+        """Create a speaker embedding from encoded audio bytes."""
+        url = f"{self._base_url}/v1/audio/embeddings"
+        data = aiohttp.FormData()
+        data.add_field("model", self._model_uid)
+        data.add_field(
+            "file", audio, filename="file", content_type="application/octet-stream"
+        )
+        response = await self.session.post(url, data=data, headers=self.auth_headers)
+        if response.status != 200:
+            raise RuntimeError(
+                "Failed to create the audio embedding, "
+                f"detail: {await _get_error_string(response)}"
+            )
+        response_data = await response.json()
+        await _release_response(response)
+        return response_data
+
     async def transcriptions(
         self,
         audio: bytes,
@@ -1307,6 +1342,7 @@ class AsyncClient:
         request_limits: Optional[int] = None,
         worker_ip: Optional[str] = None,
         gpu_idx: Optional[Union[int, List[int]]] = None,
+        replica_config: Optional[List[Dict]] = None,
         model_path: Optional[str] = None,
         enable_thinking: Optional[bool] = None,
         **kwargs,
@@ -1348,6 +1384,13 @@ class AsyncClient:
             Specify the worker ip where the model is located in a distributed scenario.
         gpu_idx: Optional[Union[int, List[int]]]
             Specify the GPU index where the model is located.
+        replica_config: Optional[List[Dict]]
+            Per-replica placement spec. Each item is ``{"replica_uid": str|None,
+            "devices": [{"worker_ip": "ip:port", "n_gpu": int|"auto",
+            "gpu_idx": [int]|None}]}``. When set, each replica is pinned to the
+            given worker/GPU. Do not combine it with worker_ip/n_gpu/gpu_idx.
+            An omitted replica_uid defaults to ``{model_uid}-{replica_index}``.
+            ``devices`` length must be 1 (no cross-worker sharding per replica).
         model_path: Optional[str]
             Model path, if gguf format, should be the file path, otherwise, should be directory of the model.
         enable_thinking: Optional[bool]
@@ -1384,6 +1427,7 @@ class AsyncClient:
             "request_limits": request_limits,
             "worker_ip": worker_ip,
             "gpu_idx": gpu_idx,
+            "replica_config": replica_config,
             "model_path": model_path,
             "enable_thinking": enable_thinking,
         }
@@ -1477,6 +1521,47 @@ class AsyncClient:
             )
         await _release_response(response)
 
+    async def add_model_replica(
+        self,
+        model_uid: str,
+        replica_config: Optional[dict] = None,
+    ) -> dict:
+        """Add a new replica to a running model (scale-up).
+
+        Parameters
+        ----------
+        model_uid : str
+            The UID of the running model to extend.
+        replica_config : Optional[dict]
+            Optional single-device placement config, e.g.::
+
+                {
+                  "replica_uid": "my-replica-label",
+                  "devices": [
+                    {"worker_ip": "192.168.1.100:9999", "gpu_idx": [0, 1]}
+                  ]
+                }
+
+            Omit to let the supervisor auto-select a worker and GPU.
+
+        Returns
+        -------
+        dict
+            ``{"replica_id": int, "replica_model_uid": str, "worker_address": str}``
+        """
+        url = f"{self.base_url}/v1/models/{model_uid}/replicas"
+        payload: Dict[str, Any] = {}
+        if replica_config is not None:
+            payload["replica_config"] = replica_config
+        response = await self.session.post(url, json=payload, headers=self._headers)
+        if response.status != 200:
+            raise RuntimeError(
+                f"Failed to add model replica, detail: {await _get_error_string(response)}"
+            )
+        result = await response.json()
+        await _release_response(response)
+        return result
+
     async def terminate_model_replica(self, model_uid: str, replica_id: int) -> int:
         """Terminate a specific replica of a running model."""
 

{xinference_client-3.0.0 → xinference_client-3.2.0}/xinference_client/client/restful/restful_client.py

@@ -21,6 +21,7 @@ from ..common import convert_float_to_int_or_str, streaming_response_iterator
 
 if TYPE_CHECKING:
     from ...types import (
+        AudioEmbedding,
         ChatCompletion,
         ChatCompletionChunk,
         Completion,
@@ -599,6 +600,7 @@ class RESTfulVideoModelHandle(RESTfulModelHandle):
         prompt: str,
         negative_prompt: Optional[str] = None,
         n: int = 1,
+        video: Optional[Union[str, bytes]] = None,
         **kwargs,
     ) -> "VideoList":
         """
@@ -614,6 +616,8 @@ class RESTfulVideoModelHandle(RESTfulModelHandle):
             The prompt or prompts not to guide the image generation.
         n: `int`, defaults to 1
             The number of videos to generate per prompt. Must be between 1 and 10.
+        video: `Union[str, bytes]`, optional
+            The driving video for character animation models.
         Returns
         -------
         VideoList
@@ -631,6 +635,18 @@ class RESTfulVideoModelHandle(RESTfulModelHandle):
         for key, value in params.items():
             files.append((key, (None, value)))
         files.append(("image", ("image", image, "application/octet-stream")))
+        if video is not None:
+            if isinstance(video, str):
+                with open(video, "rb") as f:
+                    video_data = f.read()
+            else:
+                video_data = video
+            files.append(
+                (
+                    "video",
+                    ("video", video_data, "application/octet-stream"),
+                )
+            )
         response = self.session.post(url, files=files, headers=self.auth_headers)
         if response.status_code != 200:
             raise RuntimeError(
@@ -831,6 +847,23 @@ class RESTfulChatModelHandle(RESTfulGenerateModelHandle):
 
 
 class RESTfulAudioModelHandle(RESTfulModelHandle):
+    def create_embedding(self, audio: bytes) -> "AudioEmbedding":
+        """Create a speaker embedding from encoded audio bytes."""
+        url = f"{self._base_url}/v1/audio/embeddings"
+        files = [("file", ("file", audio, "application/octet-stream"))]
+        response = self.session.post(
+            url,
+            data={"model": self._model_uid},
+            files=files,
+            headers=self.auth_headers,
+        )
+        if response.status_code != 200:
+            raise RuntimeError(
+                "Failed to create the audio embedding, "
+                f"detail: {_get_error_string(response)}"
+            )
+        return response.json()
+
     def transcriptions(
         self,
         audio: bytes,
@@ -1193,6 +1226,7 @@ class Client:
         request_limits: Optional[int] = None,
         worker_ip: Optional[str] = None,
         gpu_idx: Optional[Union[int, List[int]]] = None,
+        replica_config: Optional[List[Dict]] = None,
         model_path: Optional[str] = None,
         enable_thinking: Optional[bool] = None,
         enable_virtual_env: Optional[bool] = None,
@@ -1237,6 +1271,13 @@ class Client:
             Specify the worker ip where the model is located in a distributed scenario.
         gpu_idx: Optional[Union[int, List[int]]]
             Specify the GPU index where the model is located.
+        replica_config: Optional[List[Dict]]
+            Per-replica placement spec. Each item is ``{"replica_uid": str|None,
+            "devices": [{"worker_ip": "ip:port", "n_gpu": int|"auto",
+            "gpu_idx": [int]|None}]}``. When set, each replica is pinned to the
+            given worker/GPU. Do not combine it with worker_ip/n_gpu/gpu_idx.
+            An omitted replica_uid defaults to ``{model_uid}-{replica_index}``.
+            ``devices`` length must be 1 (no cross-worker sharding per replica).
         model_path: Optional[str]
             Model path, if gguf format, should be the file path, otherwise, should be directory of the model.
         enable_thinking: Optional[bool]
@@ -1280,6 +1321,7 @@ class Client:
             "request_limits": request_limits,
             "worker_ip": worker_ip,
             "gpu_idx": gpu_idx,
+            "replica_config": replica_config,
             "model_path": model_path,
             "enable_thinking": enable_thinking,
             "enable_virtual_env": enable_virtual_env,
@@ -1366,6 +1408,45 @@ class Client:
                 f"Failed to terminate model, detail: {_get_error_string(response)}"
             )
 
+    def add_model_replica(
+        self,
+        model_uid: str,
+        replica_config: Optional[dict] = None,
+    ) -> dict:
+        """Add a new replica to a running model (scale-up).
+
+        Parameters
+        ----------
+        model_uid : str
+            The UID of the running model to extend.
+        replica_config : Optional[dict]
+            Optional single-device placement config, e.g.::
+
+                {
+                  "replica_uid": "my-replica-label",
+                  "devices": [
+                    {"worker_ip": "192.168.1.100:9999", "gpu_idx": [0, 1]}
+                  ]
+                }
+
+            Omit to let the supervisor auto-select a worker and GPU.
+
+        Returns
+        -------
+        dict
+            ``{"replica_id": int, "replica_model_uid": str, "worker_address": str}``
+        """
+        url = f"{self.base_url}/v1/models/{model_uid}/replicas"
+        payload: Dict[str, Any] = {}
+        if replica_config is not None:
+            payload["replica_config"] = replica_config
+        response = self.session.post(url, json=payload, headers=self._headers)
+        if response.status_code != 200:
+            raise RuntimeError(
+                f"Failed to add model replica, detail: {_get_error_string(response)}"
+            )
+        return response.json()
+
     def terminate_model_replica(self, model_uid: str, replica_id: int) -> int:
         """Terminate a specific replica of a running model."""
 

{xinference_client-3.0.0 → xinference_client-3.2.0}/xinference_client/types.py

@@ -92,6 +92,13 @@ class Embedding(TypedDict):
     usage: EmbeddingUsage
 
 
+class AudioEmbedding(TypedDict):
+    object: Literal["embedding"]
+    model: str
+    dimensions: int
+    embedding: List[float]
+
+
 class Document(TypedDict):
     text: str
 
@@ -142,6 +149,47 @@ class CompletionLogprobs(TypedDict):
     top_logprobs: List[Optional[Dict[str, float]]]
 
 
+class ChatCompletionTopLogprob(TypedDict):
+    """A single alternative token in a chat ``logprobs.content[].top_logprobs`` entry.
+
+    Distinct from the legacy parallel-list :class:`CompletionLogprobs`: chat
+    completions expose per-token ``token`` / ``bytes`` / ``logprob`` objects so
+    ``openai-python`` parses ``choice.logprobs.content`` instead of dropping the
+    legacy fields as extras.
+    """
+
+    token: str
+    bytes: Optional[List[int]]
+    logprob: float
+
+
+class ChatCompletionLogprob(TypedDict):
+    """A token entry in chat completion ``logprobs.content[]``.
+
+    ``logprob`` is required (non-optional): the OpenAI chat-completions schema
+    mandates a ``float`` on every ``content[]`` entry, and ``openai-python``
+    rejects ``null``. Tokens whose logprob is unknown (``None`` in the legacy
+    ``CompletionLogprobs.token_logprobs`` shape) are omitted from ``content[]``
+    by the chat builder rather than emitted with a null or fabricated logprob.
+    """
+
+    token: str
+    bytes: Optional[List[int]]
+    logprob: float
+    top_logprobs: List[ChatCompletionTopLogprob]
+
+
+class ChatCompletionLogprobs(TypedDict):
+    """Chat Completions logprobs (``content`` list shape).
+
+    vLLM emits logprobs in the legacy parallel-list :class:`CompletionLogprobs`
+    shape; chat clients expect ``logprobs.content[]``. The chat builder converts
+    at the boundary so the legacy shape never reaches a chat choice.
+    """
+
+    content: Optional[List[ChatCompletionLogprob]]
+
+
 class ToolCallFunction(TypedDict):
     name: str
     arguments: str
@@ -204,6 +252,7 @@ class ChatCompletionMessage(TypedDict):
 class ChatCompletionChoice(TypedDict):
     index: int
     message: ChatCompletionMessage
+    logprobs: NotRequired[Optional[ChatCompletionLogprobs]]
     finish_reason: Optional[str]
 
 
@@ -226,6 +275,7 @@ class ChatCompletionChunkDelta(TypedDict):
 class ChatCompletionChunkChoice(TypedDict):
     index: int
     delta: ChatCompletionChunkDelta
+    logprobs: NotRequired[Optional[ChatCompletionLogprobs]]
     finish_reason: Optional[str]
 
 

{xinference_client-3.0.0 → xinference_client-3.2.0}/xinference_client.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xinference-client
-Version: 3.0.0
+Version: 3.2.0
 Summary: Client for Xinference
 Home-page: https://github.com/xorbitsai/inference-client
 Author: Qin Xuye