xinference 1.7.1.post1__py3-none-any.whl → 1.8.0__py3-none-any.whl

xinference/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-06-30T19:23:32+0800",
+ "date": "2025-07-20T15:28:25+0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "84f10dcf287df295396d5988016452ad8db98dfa",
- "version": "1.7.1.post1"
+ "full-revisionid": "abc42ca3105e3e0fd6f7861fb155ff807c1777a5",
+ "version": "1.8.0"
 }
 '''  # END VERSION_JSON
 

xinference/client/restful/async_restful_client.py

@@ -86,12 +86,8 @@ class AsyncRESTfulModelHandle:
 
     def __del__(self):
         if self.session:
-            try:
-                loop = asyncio.get_event_loop()
-            except RuntimeError:
-                loop = asyncio.new_event_loop()
-                asyncio.set_event_loop(loop)
-            loop.run_until_complete(self.close())
+            loop = asyncio.get_event_loop()
+            loop.create_task(self.close())
 
 
 class AsyncRESTfulEmbeddingModelHandle(AsyncRESTfulModelHandle):
@@ -418,7 +414,10 @@ class AsyncRESTfulImageModelHandle(AsyncRESTfulModelHandle):
             files.append((key, (None, value)))
         files.append(("image", ("image", image, "application/octet-stream")))
         files.append(
-            ("mask_image", ("mask_image", mask_image, "application/octet-stream"))
+            (
+                "mask_image",
+                ("mask_image", mask_image, "application/octet-stream"),
+            )
         )
         response = await self.session.post(url, files=files, headers=self.auth_headers)
         if response.status != 200:
@@ -986,12 +985,8 @@ class AsyncClient:
 
     def __del__(self):
         if self.session:
-            try:
-                loop = asyncio.get_event_loop()
-            except RuntimeError:
-                loop = asyncio.new_event_loop()
-                asyncio.set_event_loop(loop)
-            loop.run_until_complete(self.close())
+            loop = asyncio.get_event_loop()
+            loop.create_task(self.close())
 
     def _set_token(self, token: Optional[str]):
         if not self._cluster_authed or token is None:

xinference/client/restful/restful_client.py

@@ -1397,7 +1397,9 @@ class Client:
         response_data = response.json()
         return response_data
 
-    def list_model_registrations(self, model_type: str) -> List[Dict[str, Any]]:
+    def list_model_registrations(
+        self, model_type: str, detailed: bool = False
+    ) -> List[Dict[str, Any]]:
         """
         List models registered on the server.
 
@@ -1405,6 +1407,8 @@ class Client:
         ----------
         model_type: str
             The type of the model.
+        detailed: bool
+            Whether to display detailed information.
 
         Returns
         -------
@@ -1417,7 +1421,7 @@ class Client:
             Report failure to list model registration. Provide details of failure through error message.
 
         """
-        url = f"{self.base_url}/v1/model_registrations/{model_type}"
+        url = f"{self.base_url}/v1/model_registrations/{model_type}?detailed={'true' if detailed else 'false'}"
         response = self.session.get(url, headers=self._headers)
         if response.status_code != 200:
             raise RuntimeError(

xinference/core/chat_interface.py

@@ -292,9 +292,11 @@ class GradioInterface:
                         max_tokens = gr.Slider(
                             minimum=1,
                             maximum=self.context_length,
-                            value=512
-                            if "reasoning" not in self.model_ability
-                            else self.context_length // 2,
+                            value=(
+                                512
+                                if "reasoning" not in self.model_ability
+                                else self.context_length // 2
+                            ),
                             step=1,
                             label="Max Tokens",
                         )
@@ -357,7 +359,7 @@ class GradioInterface:
                     if "content" not in delta:
                         continue
                     else:
-                        response_content += delta["content"]
+                        response_content += html.escape(delta["content"])
                         bot[-1][1] = response_content
                         yield history, bot
                 history.append(

xinference/core/media_interface.py

@@ -221,6 +221,7 @@ class MediaInterface:
             n: int,
             size_width: int,
             size_height: int,
+            guidance_scale: int,
             num_inference_steps: int,
             padding_image_to_multiple: int,
             sampler_name: Optional[str] = None,
@@ -237,6 +238,7 @@ class MediaInterface:
                 size = f"{int(size_width)}*{int(size_height)}"
             else:
                 size = None
+            guidance_scale = None if guidance_scale == -1 else guidance_scale  # type: ignore
             num_inference_steps = (
                 None if num_inference_steps == -1 else num_inference_steps  # type: ignore
             )
@@ -262,6 +264,7 @@ class MediaInterface:
                         size=size,
                         response_format="b64_json",
                         num_inference_steps=num_inference_steps,
+                        guidance_scale=guidance_scale,
                         padding_image_to_multiple=padding_image_to_multiple,
                         sampler_name=sampler_name,
                     )
@@ -314,6 +317,7 @@ class MediaInterface:
                     size_height = gr.Number(label="Height", value=-1)
 
                 with gr.Row():
+                    guidance_scale = gr.Number(label="Guidance scale", value=-1)
                     num_inference_steps = gr.Number(
                         label="Inference Step Number", value=-1
                     )
@@ -341,6 +345,7 @@ class MediaInterface:
                     n,
                     size_width,
                     size_height,
+                    guidance_scale,
                     num_inference_steps,
                     padding_image_to_multiple,
                     sampler_name,

xinference/core/model.py

@@ -51,7 +51,6 @@ if TYPE_CHECKING:
     from .progress_tracker import ProgressTrackerActor
     from .worker import WorkerActor
     from ..model.llm.core import LLM
-    from ..model.core import ModelDescription
     import PIL
 
 import logging
@@ -225,7 +224,6 @@ class ModelActor(xo.StatelessActor, CancelMixin):
         worker_address: str,
         model: "LLM",
         replica_model_uid: str,
-        model_description: Optional["ModelDescription"] = None,
         request_limits: Optional[int] = None,
         xavier_config: Optional[Dict] = None,
         n_worker: Optional[int] = 1,
@@ -244,9 +242,7 @@ class ModelActor(xo.StatelessActor, CancelMixin):
         self._worker_address = worker_address
         self._replica_model_uid = replica_model_uid
         self._model = model
-        self._model_description = (
-            model_description.to_dict() if model_description else {}
-        )
+        self._model_description = self._model.model_family.to_description()
         self._request_limits = (
             float("inf") if request_limits is None else request_limits
         )

xinference/core/supervisor.py

@@ -62,13 +62,13 @@ from .utils import (
 )
 
 if TYPE_CHECKING:
-    from ..model.audio import AudioModelFamilyV1
-    from ..model.embedding import EmbeddingModelSpec
+    from ..model.audio import AudioModelFamilyV2
+    from ..model.embedding import EmbeddingModelFamilyV2
     from ..model.flexible import FlexibleModelSpec
-    from ..model.image import ImageModelFamilyV1
-    from ..model.llm import LLMFamilyV1
-    from ..model.rerank import RerankModelSpec
-    from ..model.video import VideoModelFamilyV1
+    from ..model.image import ImageModelFamilyV2
+    from ..model.llm import LLMFamilyV2
+    from ..model.rerank import RerankModelFamilyV2
+    from ..model.video import VideoModelFamilyV2
     from .worker import WorkerActor
 
 
@@ -94,9 +94,9 @@ class WorkerStatus:
 class ReplicaInfo:
     replica: int
     scheduler: Iterator
-    replica_to_worker_refs: DefaultDict[
-        int, List[xo.ActorRefType["WorkerActor"]]
-    ] = field(default_factory=lambda: defaultdict(list))
+    replica_to_worker_refs: DefaultDict[int, List[xo.ActorRefType["WorkerActor"]]] = (
+        field(default_factory=lambda: defaultdict(list))
+    )
 
 
 class SupervisorActor(xo.StatelessActor):
@@ -144,10 +144,12 @@ class SupervisorActor(xo.StatelessActor):
         from .progress_tracker import ProgressTrackerActor
         from .status_guard import StatusGuardActor
 
-        self._status_guard_ref: xo.ActorRefType[  # type: ignore
-            "StatusGuardActor"
-        ] = await xo.create_actor(
-            StatusGuardActor, address=self.address, uid=StatusGuardActor.default_uid()
+        self._status_guard_ref: xo.ActorRefType["StatusGuardActor"] = (  # type: ignore
+            await xo.create_actor(
+                StatusGuardActor,
+                address=self.address,
+                uid=StatusGuardActor.default_uid(),
+            )
         )
         self._cache_tracker_ref: xo.ActorRefType[  # type: ignore
             "CacheTrackerActor"
@@ -173,14 +175,14 @@ class SupervisorActor(xo.StatelessActor):
         )
 
         from ..model.audio import (
-            CustomAudioModelFamilyV1,
+            CustomAudioModelFamilyV2,
             generate_audio_description,
             get_audio_model_descriptions,
             register_audio,
             unregister_audio,
         )
         from ..model.embedding import (
-            CustomEmbeddingModelSpec,
+            CustomEmbeddingModelFamilyV2,
             generate_embedding_description,
             get_embedding_model_descriptions,
             register_embedding,
@@ -194,21 +196,21 @@ class SupervisorActor(xo.StatelessActor):
             unregister_flexible_model,
         )
         from ..model.image import (
-            CustomImageModelFamilyV1,
+            CustomImageModelFamilyV2,
             generate_image_description,
             get_image_model_descriptions,
             register_image,
             unregister_image,
         )
         from ..model.llm import (
-            CustomLLMFamilyV1,
-            generate_llm_description,
-            get_llm_model_descriptions,
+            CustomLLMFamilyV2,
+            generate_llm_version_info,
+            get_llm_version_infos,
             register_llm,
             unregister_llm,
         )
         from ..model.rerank import (
-            CustomRerankModelSpec,
+            CustomRerankModelFamilyV2,
             generate_rerank_description,
             get_rerank_model_descriptions,
             register_rerank,
@@ -217,31 +219,31 @@ class SupervisorActor(xo.StatelessActor):
 
         self._custom_register_type_to_cls: Dict[str, Tuple] = {  # type: ignore
             "LLM": (
-                CustomLLMFamilyV1,
+                CustomLLMFamilyV2,
                 register_llm,
                 unregister_llm,
-                generate_llm_description,
+                generate_llm_version_info,
             ),
             "embedding": (
-                CustomEmbeddingModelSpec,
+                CustomEmbeddingModelFamilyV2,
                 register_embedding,
                 unregister_embedding,
                 generate_embedding_description,
             ),
             "rerank": (
-                CustomRerankModelSpec,
+                CustomRerankModelFamilyV2,
                 register_rerank,
                 unregister_rerank,
                 generate_rerank_description,
             ),
             "image": (
-                CustomImageModelFamilyV1,
+                CustomImageModelFamilyV2,
                 register_image,
                 unregister_image,
                 generate_image_description,
             ),
             "audio": (
-                CustomAudioModelFamilyV1,
+                CustomAudioModelFamilyV2,
                 register_audio,
                 unregister_audio,
                 generate_audio_description,
@@ -256,7 +258,7 @@ class SupervisorActor(xo.StatelessActor):
 
         # record model version
         model_version_infos: Dict[str, List[Dict]] = {}  # type: ignore
-        model_version_infos.update(get_llm_model_descriptions())
+        model_version_infos.update(get_llm_version_infos())
         model_version_infos.update(get_embedding_model_descriptions())
         model_version_infos.update(get_rerank_model_descriptions())
         model_version_infos.update(get_image_model_descriptions())
@@ -405,9 +407,9 @@ class SupervisorActor(xo.StatelessActor):
         }
 
     async def _to_llm_reg(
-        self, llm_family: "LLMFamilyV1", is_builtin: bool
+        self, llm_family: "LLMFamilyV2", is_builtin: bool
     ) -> Dict[str, Any]:
-        from ..model.llm import get_cache_status
+        from ..model.llm.cache_manager import LLMCacheManager
 
         instance_cnt = await self.get_instance_count(llm_family.model_name)
         version_cnt = await self.get_model_version_count(llm_family.model_name)
@@ -415,9 +417,17 @@ class SupervisorActor(xo.StatelessActor):
         if self.is_local_deployment():
             specs = []
             # TODO: does not work when the supervisor and worker are running on separate nodes.
-            for spec in llm_family.model_specs:
-                cache_status = get_cache_status(llm_family, spec)
-                specs.append({**spec.dict(), "cache_status": cache_status})
+            _llm_family = llm_family.copy()
+            for spec in [
+                _spec
+                for _spec in llm_family.model_specs
+                if _spec.model_hub == "huggingface"
+            ]:
+                _llm_family.model_specs = [spec]
+                cache_manager = LLMCacheManager(_llm_family)
+                specs.append(
+                    {**spec.dict(), "cache_status": cache_manager.get_cache_status()}
+                )
             res = {**llm_family.dict(), "is_builtin": is_builtin, "model_specs": specs}
         else:
             res = {**llm_family.dict(), "is_builtin": is_builtin}
@@ -426,24 +436,37 @@ class SupervisorActor(xo.StatelessActor):
         return res
 
     async def _to_embedding_model_reg(
-        self, model_spec: "EmbeddingModelSpec", is_builtin: bool
+        self, model_family: "EmbeddingModelFamilyV2", is_builtin: bool
     ) -> Dict[str, Any]:
-        from ..model.embedding import get_cache_status
+        from ..model.embedding.cache_manager import EmbeddingCacheManager
 
-        instance_cnt = await self.get_instance_count(model_spec.model_name)
-        version_cnt = await self.get_model_version_count(model_spec.model_name)
+        instance_cnt = await self.get_instance_count(model_family.model_name)
+        version_cnt = await self.get_model_version_count(model_family.model_name)
 
         if self.is_local_deployment():
+            _family = model_family.copy()
+            specs = []
             # TODO: does not work when the supervisor and worker are running on separate nodes.
-            cache_status = get_cache_status(model_spec)
+            for spec in [
+                x for x in model_family.model_specs if x.model_hub == "huggingface"
+            ]:
+                _family.model_specs = [spec]
+                specs.append(
+                    {
+                        **spec.dict(),
+                        "cache_status": EmbeddingCacheManager(
+                            _family
+                        ).get_cache_status(),
+                    }
+                )
             res = {
-                **model_spec.dict(),
-                "cache_status": cache_status,
+                **model_family.dict(),
                 "is_builtin": is_builtin,
+                "model_specs": specs,
             }
         else:
             res = {
-                **model_spec.dict(),
+                **model_family.dict(),
                 "is_builtin": is_builtin,
             }
         res["model_version_count"] = version_cnt
@@ -451,16 +474,17 @@ class SupervisorActor(xo.StatelessActor):
         return res
 
     async def _to_rerank_model_reg(
-        self, model_spec: "RerankModelSpec", is_builtin: bool
+        self, model_spec: "RerankModelFamilyV2", is_builtin: bool
     ) -> Dict[str, Any]:
-        from ..model.rerank import get_cache_status
+        from ..model.cache_manager import CacheManager
 
         instance_cnt = await self.get_instance_count(model_spec.model_name)
         version_cnt = await self.get_model_version_count(model_spec.model_name)
+        cache_manager = CacheManager(model_spec)
 
         if self.is_local_deployment():
             # TODO: does not work when the supervisor and worker are running on separate nodes.
-            cache_status = get_cache_status(model_spec)
+            cache_status = cache_manager.get_cache_status()
             res = {
                 **model_spec.dict(),
                 "cache_status": cache_status,
@@ -476,19 +500,19 @@ class SupervisorActor(xo.StatelessActor):
         return res
 
     async def _to_image_model_reg(
-        self, model_family: "ImageModelFamilyV1", is_builtin: bool
+        self, model_family: "ImageModelFamilyV2", is_builtin: bool
     ) -> Dict[str, Any]:
-        from ..model.image import get_cache_status
+        from ..model.image.cache_manager import ImageCacheManager
 
         instance_cnt = await self.get_instance_count(model_family.model_name)
         version_cnt = await self.get_model_version_count(model_family.model_name)
 
         if self.is_local_deployment():
             # TODO: does not work when the supervisor and worker are running on separate nodes.
-            cache_status = get_cache_status(model_family)
+            cache_manager = ImageCacheManager(model_family)
             res = {
                 **model_family.dict(),
-                "cache_status": cache_status,
+                "cache_status": cache_manager.get_cache_status(),
                 "is_builtin": is_builtin,
             }
         else:
@@ -501,19 +525,19 @@ class SupervisorActor(xo.StatelessActor):
         return res
 
     async def _to_audio_model_reg(
-        self, model_family: "AudioModelFamilyV1", is_builtin: bool
+        self, model_family: "AudioModelFamilyV2", is_builtin: bool
     ) -> Dict[str, Any]:
-        from ..model.audio import get_cache_status
+        from ..model.cache_manager import CacheManager
 
         instance_cnt = await self.get_instance_count(model_family.model_name)
         version_cnt = await self.get_model_version_count(model_family.model_name)
+        cache_manager = CacheManager(model_family)
 
         if self.is_local_deployment():
             # TODO: does not work when the supervisor and worker are running on separate nodes.
-            cache_status = get_cache_status(model_family)
             res = {
                 **model_family.dict(),
-                "cache_status": cache_status,
+                "cache_status": cache_manager.get_cache_status(),
                 "is_builtin": is_builtin,
             }
         else:
@@ -526,19 +550,19 @@ class SupervisorActor(xo.StatelessActor):
         return res
 
     async def _to_video_model_reg(
-        self, model_family: "VideoModelFamilyV1", is_builtin: bool
+        self, model_family: "VideoModelFamilyV2", is_builtin: bool
     ) -> Dict[str, Any]:
-        from ..model.video import get_cache_status
+        from ..model.cache_manager import CacheManager
 
         instance_cnt = await self.get_instance_count(model_family.model_name)
         version_cnt = await self.get_model_version_count(model_family.model_name)
+        cache_manager = CacheManager(model_family)
 
         if self.is_local_deployment():
             # TODO: does not work when the supervisor and worker are running on separate nodes.
-            cache_status = get_cache_status(model_family)
             res = {
                 **model_family.dict(),
-                "cache_status": cache_status,
+                "cache_status": cache_manager.get_cache_status(),
                 "is_builtin": is_builtin,
             }
         else:
@@ -630,8 +654,9 @@ class SupervisorActor(xo.StatelessActor):
             from ..model.image import BUILTIN_IMAGE_MODELS
             from ..model.image.custom import get_user_defined_images
 
-            for model_name, family in BUILTIN_IMAGE_MODELS.items():
+            for model_name, families in BUILTIN_IMAGE_MODELS.items():
                 if detailed:
+                    family = [x for x in families if x.model_hub == "huggingface"][0]
                     ret.append(await self._to_image_model_reg(family, is_builtin=True))
                 else:
                     ret.append({"model_name": model_name, "is_builtin": True})
@@ -652,8 +677,9 @@ class SupervisorActor(xo.StatelessActor):
             from ..model.audio import BUILTIN_AUDIO_MODELS
             from ..model.audio.custom import get_user_defined_audios
 
-            for model_name, family in BUILTIN_AUDIO_MODELS.items():
+            for model_name, families in BUILTIN_AUDIO_MODELS.items():
                 if detailed:
+                    family = [x for x in families if x.model_hub == "huggingface"][0]
                     ret.append(await self._to_audio_model_reg(family, is_builtin=True))
                 else:
                     ret.append({"model_name": model_name, "is_builtin": True})
@@ -673,8 +699,9 @@ class SupervisorActor(xo.StatelessActor):
         elif model_type == "video":
             from ..model.video import BUILTIN_VIDEO_MODELS
 
-            for model_name, family in BUILTIN_VIDEO_MODELS.items():
+            for model_name, families in BUILTIN_VIDEO_MODELS.items():
                 if detailed:
+                    family = [x for x in families if x.model_hub == "huggingface"][0]
                     ret.append(await self._to_video_model_reg(family, is_builtin=True))
                 else:
                     ret.append({"model_name": model_name, "is_builtin": True})
@@ -685,8 +712,9 @@ class SupervisorActor(xo.StatelessActor):
             from ..model.rerank import BUILTIN_RERANK_MODELS
             from ..model.rerank.custom import get_user_defined_reranks
 
-            for model_name, family in BUILTIN_RERANK_MODELS.items():
+            for model_name, families in BUILTIN_RERANK_MODELS.items():
                 if detailed:
+                    family = [x for x in families if x.model_hub == "huggingface"][0]
                     ret.append(await self._to_rerank_model_reg(family, is_builtin=True))
                 else:
                     ret.append({"model_name": model_name, "is_builtin": True})
@@ -755,25 +783,46 @@ class SupervisorActor(xo.StatelessActor):
             from ..model.image import BUILTIN_IMAGE_MODELS
             from ..model.image.custom import get_user_defined_images
 
-            for f in list(BUILTIN_IMAGE_MODELS.values()) + get_user_defined_images():
-                if f.model_name == model_name:
-                    return f
+            if model_name in BUILTIN_IMAGE_MODELS:
+                return [
+                    x
+                    for x in BUILTIN_IMAGE_MODELS[model_name]
+                    if x.model_hub == "huggingface"
+                ][0]
+            else:
+                for f in get_user_defined_images():
+                    if f.model_name == model_name:
+                        return f
             raise ValueError(f"Model {model_name} not found")
         elif model_type == "audio":
             from ..model.audio import BUILTIN_AUDIO_MODELS
             from ..model.audio.custom import get_user_defined_audios
 
-            for f in list(BUILTIN_AUDIO_MODELS.values()) + get_user_defined_audios():
-                if f.model_name == model_name:
-                    return f
+            if model_name in BUILTIN_AUDIO_MODELS:
+                return [
+                    x
+                    for x in BUILTIN_AUDIO_MODELS[model_name]
+                    if x.model_hub == "huggingface"
+                ][0]
+            else:
+                for f in get_user_defined_audios():
+                    if f.model_name == model_name:
+                        return f
             raise ValueError(f"Model {model_name} not found")
         elif model_type == "rerank":
             from ..model.rerank import BUILTIN_RERANK_MODELS
             from ..model.rerank.custom import get_user_defined_reranks
 
-            for f in list(BUILTIN_RERANK_MODELS.values()) + get_user_defined_reranks():
-                if f.model_name == model_name:
-                    return f
+            if model_name in BUILTIN_RERANK_MODELS:
+                return [
+                    x
+                    for x in BUILTIN_RERANK_MODELS[model_name]
+                    if x.model_hub == "huggingface"
+                ][0]
+            else:
+                for f in get_user_defined_reranks():
+                    if f.model_name == model_name:
+                        return f
             raise ValueError(f"Model {model_name} not found")
         elif model_type == "flexible":
             from ..model.flexible import get_flexible_models