xinference-client 3.2.0__tar.gz → 3.4.0__tar.gz

{xinference_client-3.2.0 → xinference_client-3.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xinference-client
-Version: 3.2.0
+Version: 3.4.0
 Summary: Client for Xinference
 Home-page: https://github.com/xorbitsai/inference-client
 Author: Qin Xuye
@@ -16,6 +16,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Topic :: Software Development :: Libraries
 Description-Content-Type: text/markdown

{xinference_client-3.2.0 → xinference_client-3.4.0}/setup.cfg

@@ -17,6 +17,7 @@ classifier =
 	Programming Language :: Python :: 3.11
 	Programming Language :: Python :: 3.12
 	Programming Language :: Python :: 3.13
+	Programming Language :: Python :: 3.14
 	Programming Language :: Python :: Implementation :: CPython
 	Topic :: Software Development :: Libraries
 

xinference_client-3.4.0/tests/test_async_client.py

@@ -0,0 +1,37 @@
+import pytest
+from aiohttp import web
+
+from xinference_client import AsyncRESTfulClient
+
+
+@pytest.mark.asyncio
+async def test_list_models_and_explicit_close(monkeypatch):
+    # Authentication discovery is synchronous; isolate it from the async server.
+    monkeypatch.setattr(
+        AsyncRESTfulClient, "_check_cluster_authenticated", lambda self: None
+    )
+    model = {"id": "test-model", "model_type": "LLM"}
+
+    async def list_models(request):
+        return web.json_response({"data": [model]})
+
+    app = web.Application()
+    app.router.add_get("/v1/models", list_models)
+    runner = web.AppRunner(app)
+    await runner.setup()
+    client = None
+    try:
+        site = web.TCPSite(runner, "127.0.0.1", 0)
+        await site.start()
+        port = runner.addresses[0][1]
+        client = AsyncRESTfulClient(f"http://127.0.0.1:{port}")
+        session = client.session
+        assert await client.list_models() == {"test-model": model}
+        await client.close()
+        assert session.closed
+        assert client.session is None
+        await client.close()
+    finally:
+        if client is not None:
+            await client.close()
+        await runner.cleanup()

xinference_client-3.4.0/tests/test_async_client_cleanup.py

@@ -0,0 +1,126 @@
+# Copyright 2022-2026 Xinference Holdings Pte. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import asyncio
+import gc
+import threading
+
+import pytest
+
+from xinference_client.client.restful import async_restful_client as client_module
+
+
+@pytest.mark.parametrize(
+    "client_class",
+    [client_module.AsyncRESTfulModelHandle, client_module.AsyncClient],
+)
+def test_destructor_without_initialized_owner_loop(client_class, monkeypatch):
+    def fail_get_event_loop():
+        raise AssertionError("destructor must not request an implicit event loop")
+
+    monkeypatch.setattr(client_module.asyncio, "get_event_loop", fail_get_event_loop)
+    client = client_class.__new__(client_class)
+    client.session = object()
+
+    client.__del__()
+
+    client.session = None
+
+
+@pytest.mark.parametrize(
+    "client_class",
+    [client_module.AsyncRESTfulModelHandle, client_module.AsyncClient],
+)
+def test_destructor_after_owner_loop_closed(client_class):
+    client = client_class.__new__(client_class)
+    client.session = object()
+    client._session_loop = asyncio.new_event_loop()
+    client._session_loop.close()
+    client.__del__()
+    client.session = None
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "client_class",
+    [client_module.AsyncRESTfulModelHandle, client_module.AsyncClient],
+)
+async def test_destructor_schedules_close_on_running_loop(client_class):
+    client = client_class.__new__(client_class)
+    client._session_loop = asyncio.get_running_loop()
+    client.session = object()
+    closed = asyncio.Event()
+
+    async def close():
+        client.session = None
+        closed.set()
+
+    client.close = close
+
+    client.__del__()
+
+    await asyncio.wait_for(closed.wait(), timeout=1)
+    assert client.session is None
+
+
+@pytest.mark.parametrize(
+    "client_class",
+    [client_module.AsyncRESTfulModelHandle, client_module.AsyncClient],
+)
+@pytest.mark.parametrize("other_thread", [False, True])
+def test_real_session_cleanup_on_paused_owner_loop(
+    client_class, other_thread, monkeypatch
+):
+    monkeypatch.setattr(
+        client_module.AsyncClient, "_check_cluster_authenticated", lambda self: None
+    )
+    loop = asyncio.new_event_loop()
+    loop.set_debug(True)
+
+    async def create():
+        if client_class is client_module.AsyncClient:
+            return client_class("http://localhost:9997")
+        return client_class("model", "http://localhost:9997", {})
+
+    clients = [loop.run_until_complete(create())]
+    session = clients[0].session
+    closed_on = []
+    original_close = session.close
+
+    async def close():
+        closed_on.append(asyncio.get_running_loop())
+        await original_close()
+
+    monkeypatch.setattr(session, "close", close)
+
+    async def drop():
+        clients.clear()
+        gc.collect()
+        await asyncio.sleep(0)
+
+    try:
+        if other_thread:
+            thread = threading.Thread(target=lambda: asyncio.run(drop()))
+            thread.start()
+            thread.join()
+        else:
+            clients.clear()
+            gc.collect()
+        assert not session.closed
+        loop.run_until_complete(asyncio.sleep(0))
+        assert session.closed
+        assert closed_on == [loop]
+    finally:
+        loop.run_until_complete(original_close())
+        loop.close()

{xinference_client-3.2.0 → xinference_client-3.4.0}/xinference_client/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2026-08-15T14:17:59+0800",
+ "date": "2026-09-11T20:38:48+0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "3ed6a1c10c0b3793ebe05c29c6cc6ba331af318b",
- "version": "3.2.0"
+ "full-revisionid": "9ed05c51a3b9121ab6c7dac6f161f50fd4e28755",
+ "version": "3.4.0"
 }
 '''  # END VERSION_JSON
 

{xinference_client-3.2.0 → xinference_client-3.4.0}/xinference_client/client/common.py

@@ -12,10 +12,26 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import base64
 import json
+import os
 from typing import Any, AsyncIterator, Iterator, Union
 
 
+def encode_world_reference(reference: Union[str, bytes], media_type: str) -> str:
+    if isinstance(reference, str) and not os.path.isfile(reference):
+        return reference
+    if isinstance(reference, str):
+        with open(reference, "rb") as reference_file:
+            data = reference_file.read()
+    elif isinstance(reference, bytes):
+        data = reference
+    else:
+        raise TypeError("World generation references must be strings or bytes")
+    encoded = base64.b64encode(data).decode("ascii")
+    return f"data:{media_type};base64,{encoded}"
+
+
 def convert_float_to_int_or_str(model_size: float) -> Union[int, str]:
     """convert float to int or string
 
@@ -64,7 +80,7 @@ def streaming_response_iterator(
 
 
 async def async_streaming_response_iterator(
-    response_lines: AsyncIterator[bytes],
+    response_or_lines: Any,
 ) -> AsyncIterator[Any]:
     """
     Create an AsyncIterator to handle the streaming type of generation.
@@ -76,8 +92,8 @@ async def async_streaming_response_iterator(
 
     Parameters
     ----------
-    response_lines: AsyncIterator[bytes]
-        Generated lines by the Model Generator.
+    response_or_lines: aiohttp.ClientResponse or AsyncIterator[bytes]
+        The owning HTTP response, or generated lines for compatibility.
 
     Returns
     -------
@@ -86,14 +102,26 @@ async def async_streaming_response_iterator(
 
     """
 
-    async for line in response_lines:
-        line = line.strip()
-        if line.startswith(b"data:"):
-            json_str = line[len(b"data:") :].strip()
-            if json_str == b"[DONE]":
-                continue
-            data = json.loads(json_str.decode("utf-8"))
-            error = data.get("error", None)
-            if error is not None:
-                raise Exception(str(error))
-            yield data
+    response = (
+        response_or_lines
+        if hasattr(response_or_lines, "content")
+        and hasattr(response_or_lines, "release")
+        else None
+    )
+    response_lines = response.content if response is not None else response_or_lines
+    try:
+        async for line in response_lines:
+            line = line.strip()
+            if line.startswith(b"data:"):
+                json_str = line[len(b"data:") :].strip()
+                if json_str == b"[DONE]":
+                    continue
+                data = json.loads(json_str.decode("utf-8"))
+                error = data.get("error", None)
+                if error is not None:
+                    raise Exception(str(error))
+                yield data
+    finally:
+        if response is not None:
+            response.release()
+            await response.wait_for_close()

{xinference_client-3.2.0 → xinference_client-3.4.0}/xinference_client/client/restful/async_restful_client.py

@@ -18,7 +18,11 @@ from urllib.parse import quote
 
 import aiohttp
 
-from ..common import async_streaming_response_iterator, convert_float_to_int_or_str
+from ..common import (
+    async_streaming_response_iterator,
+    convert_float_to_int_or_str,
+    encode_world_reference,
+)
 
 if TYPE_CHECKING:
     from ...types import (
@@ -66,6 +70,25 @@ async def _release_response(response: aiohttp.ClientResponse):
     await response.wait_for_close()
 
 
+def _schedule_session_close(client: Any) -> None:
+    if not getattr(client, "session", None):
+        return
+    loop = getattr(client, "_session_loop", None)
+    if loop is None or loop.is_closed():
+        return
+
+    def close_on_owner_loop():
+        loop.create_task(client.close())
+
+    try:
+        # Also queues cleanup while a manually driven loop is paused, and when
+        # the last reference is released by another thread.
+        loop.call_soon_threadsafe(close_on_owner_loop)
+    except RuntimeError:
+        # The owning loop may have closed since the check above.
+        pass
+
+
 class AsyncRESTfulModelHandle:
     """
     A sync model interface (for RESTful client) which provides type hints that makes it much easier to use xinference
@@ -77,6 +100,7 @@ class AsyncRESTfulModelHandle:
         self._base_url = base_url
         self.auth_headers = auth_headers
         self.timeout = aiohttp.ClientTimeout(total=1800)
+        self._session_loop = asyncio.get_running_loop()
         self.session = aiohttp.ClientSession(
             connector=aiohttp.TCPConnector(force_close=True)
         )
@@ -88,23 +112,32 @@ class AsyncRESTfulModelHandle:
             self.session = None
 
     def __del__(self):
-        if self.session:
-            loop = asyncio.get_event_loop()
-            loop.create_task(self.close())
+        _schedule_session_close(self)
 
 
 class AsyncRESTfulEmbeddingModelHandle(AsyncRESTfulModelHandle):
     async def create_embedding(
-        self, input: Union[str, List[str]], **kwargs
+        self,
+        input: Union[
+            str,
+            List[str],
+            List[int],
+            List[List[int]],
+            Dict[str, Any],
+            List[Dict[str, Any]],
+            List[Union[str, Dict[str, Any]]],
+        ],
+        **kwargs,
     ) -> "Embedding":
         """
         Create an Embedding from user input via RESTful APIs.
 
         Parameters
         ----------
-        input: Union[str, List[str]]
-            Input text to embed, encoded as a string or array of tokens.
-            To embed multiple inputs in a single request, pass an array of strings or array of token arrays.
+        input: Union[str, List[str], List[int], List[List[int]], Dict[str, Any],
+                     List[Dict[str, Any]], List[Union[str, Dict[str, Any]]]]
+            Text, token IDs, or multimodal input. Multimodal dictionaries may
+            contain text, image, video, or interleaved role/content messages.
 
         Returns
         -------
@@ -179,8 +212,8 @@ class AsyncRESTfulEmbeddingModelHandle(AsyncRESTfulModelHandle):
 class AsyncRESTfulRerankModelHandle(AsyncRESTfulModelHandle):
     async def rerank(
         self,
-        documents: List[str],
-        query: str,
+        documents: List[Union[str, Dict[str, Any]]],
+        query: Union[str, Dict[str, Any]],
         top_n: Optional[int] = None,
         max_chunks_per_doc: Optional[int] = None,
         return_documents: Optional[bool] = None,
@@ -192,10 +225,10 @@ class AsyncRESTfulRerankModelHandle(AsyncRESTfulModelHandle):
 
         Parameters
         ----------
-        query: str
-            The search query
-        documents: List[str]
-            The documents to rerank
+        query: Union[str, Dict[str, Any]]
+            Text or multimodal query.
+        documents: List[Union[str, Dict[str, Any]]]
+            Text or multimodal documents to rerank.
         top_n: int
             The number of results to return, defaults to returning all results
         max_chunks_per_doc: int
@@ -795,6 +828,52 @@ class AsyncRESTfulVideoModelHandle(AsyncRESTfulModelHandle):
         return response_data
 
 
+class AsyncRESTfulWorldModelHandle(AsyncRESTfulModelHandle):
+    async def generate(
+        self,
+        prompt: str,
+        image: Optional[Union[str, bytes]] = None,
+        video: Optional[Union[str, bytes]] = None,
+        generation_config: Optional[Dict[str, Any]] = None,
+        **model_kwargs,
+    ) -> "VideoList":
+        """Generate a world video from text and an optional image or video."""
+        if image is not None and video is not None:
+            raise ValueError("Only one of image and video may be provided")
+        encoded_image = (
+            await asyncio.to_thread(encode_world_reference, image, "image/png")
+            if image is not None
+            else None
+        )
+        encoded_video = (
+            await asyncio.to_thread(encode_world_reference, video, "video/mp4")
+            if video is not None
+            else None
+        )
+        request_body = {
+            "model": self._model_uid,
+            "prompt": prompt,
+            "image": encoded_image,
+            "video": encoded_video,
+            "generation_config": generation_config or {},
+            "extra_body": model_kwargs,
+        }
+        response = await self.session.post(
+            f"{self._base_url}/v1/worlds/generations",
+            json=request_body,
+            headers=self.auth_headers,
+            timeout=self.timeout,
+        )
+        if response.status != 200:
+            raise RuntimeError(
+                "Failed to generate the world, detail: "
+                f"{await _get_error_string(response)}"
+            )
+        response_data = await response.json()
+        await _release_response(response)
+        return response_data
+
+
 class AsyncRESTfulGenerateModelHandle(AsyncRESTfulModelHandle):
     async def generate(
         self,
@@ -844,7 +923,7 @@ class AsyncRESTfulGenerateModelHandle(AsyncRESTfulModelHandle):
             )
 
         if stream:
-            return async_streaming_response_iterator(response.content)
+            return async_streaming_response_iterator(response)
         response_data = await response.json()
         await _release_response(response)
         return response_data
@@ -925,7 +1004,7 @@ class AsyncRESTfulChatModelHandle(AsyncRESTfulGenerateModelHandle):
             )
 
         if stream:
-            return async_streaming_response_iterator(response.content)
+            return async_streaming_response_iterator(response)
 
         response_data = await response.json()
         await _release_response(response)
@@ -1216,6 +1295,7 @@ class AsyncClient:
         self._headers: Dict[str, str] = {}
         self._cluster_authed = False
         self.timeout = aiohttp.ClientTimeout(total=1800)
+        self._session_loop = asyncio.get_running_loop()
         self.session = aiohttp.ClientSession(
             connector=aiohttp.TCPConnector(force_close=True), timeout=self.timeout
         )
@@ -1230,9 +1310,7 @@ class AsyncClient:
             self.session = None
 
     def __del__(self):
-        if self.session:
-            loop = asyncio.get_event_loop()
-            loop.create_task(self.close())
+        _schedule_session_close(self)
 
     def _set_token(self, token: Optional[str]):
         if not self._cluster_authed or token is None:
@@ -1345,6 +1423,10 @@ class AsyncClient:
         replica_config: Optional[List[Dict]] = None,
         model_path: Optional[str] = None,
         enable_thinking: Optional[bool] = None,
+        enable_virtual_env: Optional[bool] = None,
+        virtual_env_packages: Optional[List[str]] = None,
+        envs: Optional[Dict[str, str]] = None,
+        virtual_env_find_links: Optional[List[str]] = None,
         **kwargs,
     ) -> str:
         """
@@ -1357,7 +1439,7 @@ class AsyncClient:
         model_type: str
             type of model.
         model_engine: Optional[str]
-            Specify the inference engine of the model when launching LLM.
+            Specify the inference engine to use when launching the model.
         model_uid: str
             UID of model, auto generate a UUID if is None.
         model_size_in_billions: Optional[Union[int, str, float]]
@@ -1396,6 +1478,14 @@ class AsyncClient:
         enable_thinking: Optional[bool]
             Enable or disable thinking mode for hybrid reasoning LLMs (e.g., Qwen3). None uses
             the model default.
+        enable_virtual_env: Optional[bool]
+            If enable virtual env.
+        virtual_env_packages: Optional[List[str]]
+            Packages to specify in virtual env, can be used to override builtin packages in virtual env.
+        virtual_env_find_links: Optional[List[str]]
+            Worker-local wheel directories to use when installing virtual env packages.
+        envs: Optional[Dict[str, str]]
+            Environment variables to pass when launching model.
         **kwargs:
             Any other parameters been specified. e.g. multimodal_projector for multimodal inference with the llama.cpp backend.
 
@@ -1430,6 +1520,10 @@ class AsyncClient:
             "replica_config": replica_config,
             "model_path": model_path,
             "enable_thinking": enable_thinking,
+            "enable_virtual_env": enable_virtual_env,
+            "virtual_env_packages": virtual_env_packages,
+            "virtual_env_find_links": virtual_env_find_links,
+            "envs": envs,
         }
 
         wait_ready = kwargs.pop("wait_ready", True)
@@ -1524,15 +1618,18 @@ class AsyncClient:
     async def add_model_replica(
         self,
         model_uid: str,
-        replica_config: Optional[dict] = None,
+        replica_config: Optional[Union[dict, List[dict]]] = None,
+        replica: int = 1,
+        model_engine: Optional[str] = None,
+        n_gpu: Optional[Union[int, str]] = None,
     ) -> dict:
-        """Add a new replica to a running model (scale-up).
+        """Add one or more replicas to a running model (scale-up).
 
         Parameters
         ----------
         model_uid : str
             The UID of the running model to extend.
-        replica_config : Optional[dict]
+        replica_config : Optional[Union[dict, List[dict]]]
             Optional single-device placement config, e.g.::
 
                 {
@@ -1543,16 +1640,29 @@ class AsyncClient:
                 }
 
             Omit to let the supervisor auto-select a worker and GPU.
+        replica : int
+            Number of replicas to add, default is 1.
+        model_engine : Optional[str]
+            Override the model engine for the new replicas only.
+        n_gpu : Optional[Union[int, str]]
+            Override GPU usage for the new replicas. Use 0 for CPU or ``"auto"``.
 
         Returns
         -------
         dict
-            ``{"replica_id": int, "replica_model_uid": str, "worker_address": str}``
+            A single-replica result, or ``{"replica": int, "replicas": list}``
+            when more than one replica is requested.
         """
         url = f"{self.base_url}/v1/models/{model_uid}/replicas"
         payload: Dict[str, Any] = {}
         if replica_config is not None:
             payload["replica_config"] = replica_config
+        if replica != 1:
+            payload["replica"] = replica
+        if model_engine is not None:
+            payload["model_engine"] = model_engine
+        if n_gpu is not None:
+            payload["n_gpu"] = n_gpu
         response = await self.session.post(url, json=payload, headers=self._headers)
         if response.status != 200:
             raise RuntimeError(
@@ -1715,6 +1825,10 @@ class AsyncClient:
             return AsyncRESTfulVideoModelHandle(
                 model_uid, self.base_url, auth_headers=self._headers
             )
+        elif desc["model_type"] == "world":
+            return AsyncRESTfulWorldModelHandle(
+                model_uid, self.base_url, auth_headers=self._headers
+            )
         elif desc["model_type"] == "flexible":
             return AsyncRESTfulFlexibleModelHandle(
                 model_uid, self.base_url, auth_headers=self._headers

{xinference_client-3.2.0 → xinference_client-3.4.0}/xinference_client/client/restful/restful_client.py

@@ -17,7 +17,11 @@ from urllib.parse import quote
 
 import requests
 
-from ..common import convert_float_to_int_or_str, streaming_response_iterator
+from ..common import (
+    convert_float_to_int_or_str,
+    encode_world_reference,
+    streaming_response_iterator,
+)
 
 if TYPE_CHECKING:
     from ...types import (
@@ -72,15 +76,28 @@ class RESTfulModelHandle:
 
 
 class RESTfulEmbeddingModelHandle(RESTfulModelHandle):
-    def create_embedding(self, input: Union[str, List[str]], **kwargs) -> "Embedding":
+    def create_embedding(
+        self,
+        input: Union[
+            str,
+            List[str],
+            List[int],
+            List[List[int]],
+            Dict[str, Any],
+            List[Dict[str, Any]],
+            List[Union[str, Dict[str, Any]]],
+        ],
+        **kwargs,
+    ) -> "Embedding":
         """
         Create an Embedding from user input via RESTful APIs.
 
         Parameters
         ----------
-        input: Union[str, List[str]]
-            Input text to embed, encoded as a string or array of tokens.
-            To embed multiple inputs in a single request, pass an array of strings or array of token arrays.
+        input: Union[str, List[str], List[int], List[List[int]], Dict[str, Any],
+                     List[Dict[str, Any]], List[Union[str, Dict[str, Any]]]]
+            Text, token IDs, or multimodal input. Multimodal dictionaries may
+            contain text, image, video, or interleaved role/content messages.
 
         Returns
         -------
@@ -149,8 +166,8 @@ class RESTfulEmbeddingModelHandle(RESTfulModelHandle):
 class RESTfulRerankModelHandle(RESTfulModelHandle):
     def rerank(
         self,
-        documents: List[str],
-        query: str,
+        documents: List[Union[str, Dict[str, Any]]],
+        query: Union[str, Dict[str, Any]],
         top_n: Optional[int] = None,
         max_chunks_per_doc: Optional[int] = None,
         return_documents: Optional[bool] = None,
@@ -162,10 +179,10 @@ class RESTfulRerankModelHandle(RESTfulModelHandle):
 
         Parameters
         ----------
-        query: str
-            The search query
-        documents: List[str]
-            The documents to rerank
+        query: Union[str, Dict[str, Any]]
+            Text or multimodal query.
+        documents: List[Union[str, Dict[str, Any]]]
+            Text or multimodal documents to rerank.
         top_n: int
             The number of results to return, defaults to returning all results
         max_chunks_per_doc: int
@@ -710,6 +727,47 @@ class RESTfulVideoModelHandle(RESTfulModelHandle):
         return response_data
 
 
+class RESTfulWorldModelHandle(RESTfulModelHandle):
+    def generate(
+        self,
+        prompt: str,
+        image: Optional[Union[str, bytes]] = None,
+        video: Optional[Union[str, bytes]] = None,
+        generation_config: Optional[Dict[str, Any]] = None,
+        **model_kwargs,
+    ) -> "VideoList":
+        """Generate a world video from text and an optional image or video."""
+        if image is not None and video is not None:
+            raise ValueError("Only one of image and video may be provided")
+        request_body = {
+            "model": self._model_uid,
+            "prompt": prompt,
+            "image": (
+                encode_world_reference(image, "image/png")
+                if image is not None
+                else None
+            ),
+            "video": (
+                encode_world_reference(video, "video/mp4")
+                if video is not None
+                else None
+            ),
+            "generation_config": generation_config or {},
+            "extra_body": model_kwargs,
+        }
+        response = self.session.post(
+            f"{self._base_url}/v1/worlds/generations",
+            json=request_body,
+            headers=self.auth_headers,
+        )
+        if response.status_code != 200:
+            raise RuntimeError(
+                "Failed to generate the world, detail: "
+                f"{_get_error_string(response)}"
+            )
+        return response.json()
+
+
 class RESTfulGenerateModelHandle(RESTfulModelHandle):
     def generate(
         self,
@@ -1232,6 +1290,7 @@ class Client:
         enable_virtual_env: Optional[bool] = None,
         virtual_env_packages: Optional[List[str]] = None,
         envs: Optional[Dict[str, str]] = None,
+        virtual_env_find_links: Optional[List[str]] = None,
         **kwargs,
     ) -> str:
         """
@@ -1244,7 +1303,7 @@ class Client:
         model_type: str
             type of model.
         model_engine: Optional[str]
-            Specify the inference engine of the model when launching LLM.
+            Specify the inference engine to use when launching the model.
         model_uid: str
             UID of model, auto generate a UUID if is None.
         model_size_in_billions: Optional[Union[int, str, float]]
@@ -1287,6 +1346,8 @@ class Client:
             If enable virtual env.
         virtual_env_packages: Optional[List[str]]
             Packages to specify in virtual env, can be used to override builtin packages in virtual env.
+        virtual_env_find_links: Optional[List[str]]
+            Worker-local wheel directories to use when installing virtual env packages.
         envs: Optional[Dict[str, str]]
             Environment variables to pass when launching model.
 
@@ -1326,6 +1387,7 @@ class Client:
             "enable_thinking": enable_thinking,
             "enable_virtual_env": enable_virtual_env,
             "virtual_env_packages": virtual_env_packages,
+            "virtual_env_find_links": virtual_env_find_links,
             "envs": envs,
         }
 
@@ -1411,15 +1473,18 @@ class Client:
     def add_model_replica(
         self,
         model_uid: str,
-        replica_config: Optional[dict] = None,
+        replica_config: Optional[Union[dict, List[dict]]] = None,
+        replica: int = 1,
+        model_engine: Optional[str] = None,
+        n_gpu: Optional[Union[int, str]] = None,
     ) -> dict:
-        """Add a new replica to a running model (scale-up).
+        """Add one or more replicas to a running model (scale-up).
 
         Parameters
         ----------
         model_uid : str
             The UID of the running model to extend.
-        replica_config : Optional[dict]
+        replica_config : Optional[Union[dict, List[dict]]]
             Optional single-device placement config, e.g.::
 
                 {
@@ -1430,16 +1495,29 @@ class Client:
                 }
 
             Omit to let the supervisor auto-select a worker and GPU.
+        replica : int
+            Number of replicas to add, default is 1.
+        model_engine : Optional[str]
+            Override the model engine for the new replicas only.
+        n_gpu : Optional[Union[int, str]]
+            Override GPU usage for the new replicas. Use 0 for CPU or ``"auto"``.
 
         Returns
         -------
         dict
-            ``{"replica_id": int, "replica_model_uid": str, "worker_address": str}``
+            A single-replica result, or ``{"replica": int, "replicas": list}``
+            when more than one replica is requested.
         """
         url = f"{self.base_url}/v1/models/{model_uid}/replicas"
         payload: Dict[str, Any] = {}
         if replica_config is not None:
             payload["replica_config"] = replica_config
+        if replica != 1:
+            payload["replica"] = replica
+        if model_engine is not None:
+            payload["model_engine"] = model_engine
+        if n_gpu is not None:
+            payload["n_gpu"] = n_gpu
         response = self.session.post(url, json=payload, headers=self._headers)
         if response.status_code != 200:
             raise RuntimeError(
@@ -1594,6 +1672,10 @@ class Client:
             return RESTfulVideoModelHandle(
                 model_uid, self.base_url, auth_headers=self._headers
             )
+        elif desc["model_type"] == "world":
+            return RESTfulWorldModelHandle(
+                model_uid, self.base_url, auth_headers=self._headers
+            )
         elif desc["model_type"] == "flexible":
             return RESTfulFlexibleModelHandle(
                 model_uid, self.base_url, auth_headers=self._headers

{xinference_client-3.2.0 → xinference_client-3.4.0}/xinference_client/types.py

@@ -201,6 +201,18 @@ class ToolCalls(TypedDict):
     function: ToolCallFunction
 
 
+class ToolCallDeltaFunction(TypedDict, total=False):
+    name: str
+    arguments: str
+
+
+class ToolCallDelta(TypedDict):
+    index: int
+    id: NotRequired[str]
+    type: NotRequired[Literal["function"]]
+    function: NotRequired[ToolCallDeltaFunction]
+
+
 class CompletionChoice(TypedDict):
     text: NotRequired[str]
     index: int
@@ -209,10 +221,15 @@ class CompletionChoice(TypedDict):
     tool_calls: NotRequired[List[ToolCalls]]
 
 
+class PromptTokensDetails(TypedDict):
+    cached_tokens: int
+
+
 class CompletionUsage(TypedDict):
     prompt_tokens: int
     completion_tokens: int
     total_tokens: int
+    prompt_tokens_details: NotRequired[PromptTokensDetails]
 
 
 class CompletionChunk(TypedDict):
@@ -269,7 +286,7 @@ class ChatCompletionChunkDelta(TypedDict):
     role: NotRequired[str]
     reasoning_content: NotRequired[Union[str, None]]
     content: NotRequired[Union[str, None]]
-    tool_calls: NotRequired[List[ToolCalls]]
+    tool_calls: NotRequired[List[ToolCallDelta]]
 
 
 class ChatCompletionChunkChoice(TypedDict):
@@ -472,12 +489,14 @@ class CreateChatCompletion(  # type: ignore
 
 
 class LoRA:
-    def __init__(self, lora_name: str, local_path: str):
+    def __init__(self, lora_name: str, local_path: str, lora_scale: float = 1.0):
+        self.lora_scale = lora_scale
         self.lora_name = lora_name
         self.local_path = local_path
 
     def to_dict(self):
         return {
+            **({"lora_scale": self.lora_scale} if self.lora_scale != 1.0 else {}),
             "lora_name": self.lora_name,
             "local_path": self.local_path,
         }
@@ -485,6 +504,7 @@ class LoRA:
     @classmethod
     def from_dict(cls, data: Dict):
         return cls(
+            lora_scale=data.get("lora_scale", 1.0),
             lora_name=data["lora_name"],
             local_path=data["local_path"],
         )

{xinference_client-3.2.0 → xinference_client-3.4.0}/xinference_client.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xinference-client
-Version: 3.2.0
+Version: 3.4.0
 Summary: Client for Xinference
 Home-page: https://github.com/xorbitsai/inference-client
 Author: Qin Xuye
@@ -16,6 +16,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Topic :: Software Development :: Libraries
 Description-Content-Type: text/markdown

{xinference_client-3.2.0 → xinference_client-3.4.0}/xinference_client.egg-info/SOURCES.txt

@@ -5,6 +5,8 @@ pyproject.toml
 setup.cfg
 setup.py
 versioneer.py
+tests/test_async_client.py
+tests/test_async_client_cleanup.py
 third_party/inference/xinference/thirdparty/matcha/utils/monotonic_align/core.pyx
 xinference_client/__init__.py
 xinference_client/_compat.py