xinference 1.7.0__py3-none-any.whl → 1.7.1__py3-none-any.whl

xinference/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-06-13T18:51:07+0800",
+ "date": "2025-06-27T20:03:38+0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "a362dba7334ef08c758bbc4a3d4904fe53cefe78",
- "version": "1.7.0"
+ "full-revisionid": "cf64a869f69ace1f575eca74e1ae86bc53356acf",
+ "version": "1.7.1"
 }
 '''  # END VERSION_JSON
 

xinference/api/restful_api.py

@@ -1994,10 +1994,9 @@ class RESTfulAPI(CancelMixin):
         payload = await request.json()
 
         model_uid = payload.get("model")
+        args = payload.get("args")
 
-        exclude = {
-            "model",
-        }
+        exclude = {"model", "args"}
         kwargs = {key: value for key, value in payload.items() if key not in exclude}
 
         try:
@@ -2012,7 +2011,7 @@ class RESTfulAPI(CancelMixin):
             raise HTTPException(status_code=500, detail=str(e))
 
         try:
-            result = await model.infer(**kwargs)
+            result = await model.infer(*args, **kwargs)
             return Response(result, media_type="application/json")
         except Exception as e:
             e = await self._get_model_last_error(model.uid, e)

xinference/client/__init__.py

@@ -12,7 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from .restful.async_restful_client import AsyncClient
 from .restful.restful_client import Client
 
 # For compatibility
 RESTfulClient = Client
+AsyncRESTfulClient = AsyncClient

xinference/client/common.py

@@ -13,7 +13,18 @@
 # limitations under the License.
 
 import json
-from typing import Any, Iterator
+from typing import Any, AsyncIterator, Iterator, Union
+
+
+def convert_float_to_int_or_str(model_size: float) -> Union[int, str]:
+    """convert float to int or string
+
+    if float can be presented as int, convert it to int, otherwise convert it to string
+    """
+    if int(model_size) == model_size:
+        return int(model_size)
+    else:
+        return str(model_size)
 
 
 def streaming_response_iterator(
@@ -46,7 +57,43 @@ def streaming_response_iterator(
             if json_str == b"[DONE]":
                 continue
             data = json.loads(json_str.decode("utf-8"))
-            error = data.get("error")
+            error = data.get("error", None)
+            if error is not None:
+                raise Exception(str(error))
+            yield data
+
+
+async def async_streaming_response_iterator(
+    response_lines: AsyncIterator[bytes],
+) -> AsyncIterator[Any]:
+    """
+    Create an AsyncIterator to handle the streaming type of generation.
+
+    Note
+    ----------
+    This method is for compatible with openai. Please refer to:
+    https://github.com/openai/openai-python/blob/v0.28.1/openai/api_requestor.py#L99
+
+    Parameters
+    ----------
+    response_lines: AsyncIterator[bytes]
+        Generated lines by the Model Generator.
+
+    Returns
+    -------
+    AsyncIterator["CompletionChunk"]
+        AsyncIterator of CompletionChunks generated by models.
+
+    """
+
+    async for line in response_lines:
+        line = line.strip()
+        if line.startswith(b"data:"):
+            json_str = line[len(b"data:") :].strip()
+            if json_str == b"[DONE]":
+                continue
+            data = json.loads(json_str.decode("utf-8"))
+            error = data.get("error", None)
             if error is not None:
                 raise Exception(str(error))
             yield data

xinference/client/handlers.py

@@ -1,3 +1,21 @@
+from .restful.async_restful_client import (  # noqa: F401
+    AsyncRESTfulAudioModelHandle as AsyncAudioModelHandle,
+)
+from .restful.async_restful_client import (  # noqa: F401
+    AsyncRESTfulChatModelHandle as AsyncChatModelHandle,
+)
+from .restful.async_restful_client import (  # noqa: F401
+    AsyncRESTfulEmbeddingModelHandle as AsyncEmbeddingModelHandle,
+)
+from .restful.async_restful_client import (  # noqa: F401
+    AsyncRESTfulGenerateModelHandle as AsyncGenerateModelHandle,
+)
+from .restful.async_restful_client import (  # noqa: F401
+    AsyncRESTfulImageModelHandle as AsyncImageModelHandle,
+)
+from .restful.async_restful_client import (  # noqa: F401
+    AsyncRESTfulVideoModelHandle as AsyncVideoModelHandle,
+)
 from .restful.restful_client import (  # noqa: F401
     RESTfulAudioModelHandle as AudioModelHandle,
 )