xinference 1.9.1__py3-none-any.whl → 1.10.1__py3-none-any.whl

xinference/core/supervisor.py

@@ -31,6 +31,7 @@ from typing import (
     Literal,
     Optional,
     Tuple,
+    Type,
     Union,
 )
 
@@ -406,6 +407,26 @@ class SupervisorActor(xo.StatelessActor):
             "workers": self._worker_status,
         }
 
+    def _get_spec_dicts(
+        self, model_family: Any, cache_manager_cls: Type
+    ) -> Tuple[List[dict], List[str]]:
+        specs = []
+        download_hubs: Dict[str, None] = dict()
+        for spec in model_family.model_specs:
+            model_hub = spec.model_hub
+            if model_hub not in download_hubs:
+                download_hubs[model_hub] = None
+            if model_hub != "huggingface":
+                # since we only need to know all specs
+                # thus filter huggingface specs only
+                continue
+            model_family.model_specs = [spec]
+            cache_manager = cache_manager_cls(model_family)
+            specs.append(
+                {**spec.dict(), "cache_status": cache_manager.get_cache_status()}
+            )
+        return specs, list(download_hubs)
+
     async def _to_llm_reg(
         self, llm_family: "LLMFamilyV2", is_builtin: bool
     ) -> Dict[str, Any]:
@@ -415,20 +436,15 @@ class SupervisorActor(xo.StatelessActor):
         version_cnt = await self.get_model_version_count(llm_family.model_name)
 
         if self.is_local_deployment():
-            specs = []
             # TODO: does not work when the supervisor and worker are running on separate nodes.
             _llm_family = llm_family.copy()
-            for spec in [
-                _spec
-                for _spec in llm_family.model_specs
-                if _spec.model_hub == "huggingface"
-            ]:
-                _llm_family.model_specs = [spec]
-                cache_manager = LLMCacheManager(_llm_family)
-                specs.append(
-                    {**spec.dict(), "cache_status": cache_manager.get_cache_status()}
-                )
-            res = {**llm_family.dict(), "is_builtin": is_builtin, "model_specs": specs}
+            specs, download_hubs = self._get_spec_dicts(_llm_family, LLMCacheManager)
+            res = {
+                **llm_family.dict(),
+                "is_builtin": is_builtin,
+                "model_specs": specs,
+                "download_hubs": download_hubs,
+            }
         else:
             res = {**llm_family.dict(), "is_builtin": is_builtin}
         res["model_version_count"] = version_cnt
@@ -445,24 +461,13 @@ class SupervisorActor(xo.StatelessActor):
 
         if self.is_local_deployment():
             _family = model_family.copy()
-            specs = []
             # TODO: does not work when the supervisor and worker are running on separate nodes.
-            for spec in [
-                x for x in model_family.model_specs if x.model_hub == "huggingface"
-            ]:
-                _family.model_specs = [spec]
-                specs.append(
-                    {
-                        **spec.dict(),
-                        "cache_status": EmbeddingCacheManager(
-                            _family
-                        ).get_cache_status(),
-                    }
-                )
+            specs, download_hubs = self._get_spec_dicts(_family, EmbeddingCacheManager)
             res = {
                 **model_family.dict(),
                 "is_builtin": is_builtin,
                 "model_specs": specs,
+                "download_hubs": download_hubs,
             }
         else:
             res = {
@@ -474,25 +479,26 @@ class SupervisorActor(xo.StatelessActor):
         return res
 
     async def _to_rerank_model_reg(
-        self, model_spec: "RerankModelFamilyV2", is_builtin: bool
+        self, model_family: "RerankModelFamilyV2", is_builtin: bool
     ) -> Dict[str, Any]:
-        from ..model.rerank.cache_manager import RerankCacheManager as CacheManager
+        from ..model.rerank.cache_manager import RerankCacheManager
 
-        instance_cnt = await self.get_instance_count(model_spec.model_name)
-        version_cnt = await self.get_model_version_count(model_spec.model_name)
-        cache_manager = CacheManager(model_spec)
+        instance_cnt = await self.get_instance_count(model_family.model_name)
+        version_cnt = await self.get_model_version_count(model_family.model_name)
 
         if self.is_local_deployment():
+            _family = model_family.copy()
             # TODO: does not work when the supervisor and worker are running on separate nodes.
-            cache_status = cache_manager.get_cache_status()
+            specs, download_hubs = self._get_spec_dicts(_family, RerankCacheManager)
             res = {
-                **model_spec.dict(),
-                "cache_status": cache_status,
+                **model_family.dict(),
                 "is_builtin": is_builtin,
+                "model_specs": specs,
+                "download_hubs": download_hubs,
             }
         else:
             res = {
-                **model_spec.dict(),
+                **model_family.dict(),
                 "is_builtin": is_builtin,
             }
         res["model_version_count"] = version_cnt
@@ -657,7 +663,9 @@ class SupervisorActor(xo.StatelessActor):
             for model_name, families in BUILTIN_IMAGE_MODELS.items():
                 if detailed:
                     family = [x for x in families if x.model_hub == "huggingface"][0]
-                    ret.append(await self._to_image_model_reg(family, is_builtin=True))
+                    info = await self._to_image_model_reg(family, is_builtin=True)
+                    info["download_hubs"] = [x.model_hub for x in families]
+                    ret.append(info)
                 else:
                     ret.append({"model_name": model_name, "is_builtin": True})
 
@@ -680,7 +688,9 @@ class SupervisorActor(xo.StatelessActor):
             for model_name, families in BUILTIN_AUDIO_MODELS.items():
                 if detailed:
                     family = [x for x in families if x.model_hub == "huggingface"][0]
-                    ret.append(await self._to_audio_model_reg(family, is_builtin=True))
+                    info = await self._to_audio_model_reg(family, is_builtin=True)
+                    info["download_hubs"] = [x.model_hub for x in families]
+                    ret.append(info)
                 else:
                     ret.append({"model_name": model_name, "is_builtin": True})
 
@@ -702,7 +712,9 @@ class SupervisorActor(xo.StatelessActor):
             for model_name, families in BUILTIN_VIDEO_MODELS.items():
                 if detailed:
                     family = [x for x in families if x.model_hub == "huggingface"][0]
-                    ret.append(await self._to_video_model_reg(family, is_builtin=True))
+                    info = await self._to_video_model_reg(family, is_builtin=True)
+                    info["download_hubs"] = [x.model_hub for x in families]
+                    ret.append(info)
                 else:
                     ret.append({"model_name": model_name, "is_builtin": True})
 
@@ -812,16 +824,9 @@ class SupervisorActor(xo.StatelessActor):
             from ..model.rerank import BUILTIN_RERANK_MODELS
             from ..model.rerank.custom import get_user_defined_reranks
 
-            if model_name in BUILTIN_RERANK_MODELS:
-                return [
-                    x
-                    for x in BUILTIN_RERANK_MODELS[model_name]
-                    if x.model_hub == "huggingface"
-                ][0]
-            else:
-                for f in get_user_defined_reranks():
-                    if f.model_name == model_name:
-                        return f
+            for f in list(BUILTIN_RERANK_MODELS.values()) + get_user_defined_reranks():
+                if f.model_name == model_name:
+                    return f
             raise ValueError(f"Model {model_name} not found")
         elif model_type == "flexible":
             from ..model.flexible import get_flexible_models
@@ -830,6 +835,16 @@ class SupervisorActor(xo.StatelessActor):
                 if f.model_name == model_name:
                     return f
             raise ValueError(f"Model {model_name} not found")
+        elif model_type == "video":
+            from ..model.video import BUILTIN_VIDEO_MODELS
+
+            if model_name in BUILTIN_VIDEO_MODELS:
+                return [
+                    x
+                    for x in BUILTIN_VIDEO_MODELS[model_name]
+                    if x.model_hub == "huggingface"
+                ][0]
+            raise ValueError(f"Model {model_name} not found")
         else:
             raise ValueError(f"Unsupported model type: {model_type}")
 
@@ -864,6 +879,26 @@ class SupervisorActor(xo.StatelessActor):
                 generate_fn,
             ) = self._custom_register_type_to_cls[model_type]
 
+            model_spec = model_spec_cls.parse_raw(model)
+
+            # check if model already registered
+            try:
+                model = await self.get_model_registration(
+                    model_type, model_spec.model_name
+                )
+                if model is not None:
+                    raise ValueError(
+                        f"Model {model_spec.model_name} already registered"
+                    )
+            except ValueError as e:
+                if "not found" in str(e):
+                    pass
+                else:
+                    raise e
+            except Exception:
+                logger.error("Get model registration failed.", exc_info=True)
+                raise
+
             target_ip_worker_ref = (
                 self._get_worker_ref_by_ip(worker_ip) if worker_ip is not None else None
             )
@@ -880,12 +915,15 @@ class SupervisorActor(xo.StatelessActor):
                 await target_ip_worker_ref.register_model(model_type, model, persist)
                 return
 
-            model_spec = model_spec_cls.parse_raw(model)
             try:
                 register_fn(model_spec, persist)
                 await self._cache_tracker_ref.record_model_version(
                     generate_fn(model_spec), self.address
                 )
+                await self._sync_register_model(
+                    model_type, model, persist, model_spec.model_name
+                )
+
             except ValueError as e:
                 raise e
             except Exception as e:
@@ -894,6 +932,30 @@ class SupervisorActor(xo.StatelessActor):
         else:
             raise ValueError(f"Unsupported model type: {model_type}")
 
+    async def _sync_register_model(
+        self, model_type: str, model: str, persist: bool, model_name: str
+    ):
+        logger.info(f"begin sync model: {model_name} to worker")
+        try:
+            # Sync model to all workers.
+            for name, worker in self._worker_address_to_worker.items():
+                logger.info(f"sync model: {model_name} to {name}")
+                if name == self.address:
+                    # Ignore: when worker and supervisor at the same node.
+                    logger.info(
+                        f"ignore sync model: {model_name} to {name} for same node"
+                    )
+                else:
+                    await worker.register_model(model_type, model, persist)
+                    logger.info(f"success sync model: {model_name} to {name}")
+        except Exception as e:
+            # If sync fails, unregister the model in all workers.
+            for name, worker in self._worker_address_to_worker.items():
+                logger.warning(f"ready to unregister model for {name}")
+                await worker.unregister_model(model_type, model_name)
+                logger.warning(f"finish unregister model: {model} for {name}")
+            raise e
+
     @log_async(logger=logger)
     async def unregister_model(self, model_type: str, model_name: str):
         if model_type in self._custom_register_type_to_cls:
@@ -1014,7 +1076,7 @@ class SupervisorActor(xo.StatelessActor):
             )
 
         # search in worker first
-        if not self.is_local_deployment():
+        if not self.is_local_deployment() and worker_ip is None:
             workers = list(self._worker_address_to_worker.values())
             for worker in workers:
                 res = await worker.get_model_registration(model_type, model_name)

xinference/core/worker.py

@@ -710,6 +710,16 @@ class WorkerActor(xo.StatelessActor):
             for model_spec in get_user_defined_reranks():
                 ret.append({"model_name": model_spec.model_name, "is_builtin": False})
 
+            ret.sort(key=sort_helper)
+            return ret
+        elif model_type == "flexible":
+            from ..model.flexible.custom import get_flexible_models
+
+            ret = []
+
+            for model_spec in get_flexible_models():
+                ret.append({"model_name": model_spec.model_name, "is_builtin": False})
+
             ret.sort(key=sort_helper)
             return ret
         else:

xinference/deploy/cmdline.py

@@ -576,6 +576,21 @@ def list_model_registrations(
             ),
             file=sys.stderr,
         )
+    elif model_type == "flexible":
+        for registration in registrations:
+            model_name = registration["model_name"]
+            model_family = client.get_model_registration(model_type, model_name)
+            table.append(
+                [
+                    model_type,
+                    model_family["model_name"],
+                    registration["is_builtin"],
+                ]
+            )
+        print(
+            tabulate(table, headers=["Type", "Name", "Is-built-in"]),
+            file=sys.stderr,
+        )
     else:
         raise NotImplementedError(f"List {model_type} is not implemented.")
 

xinference/model/audio/core.py

@@ -23,8 +23,10 @@ from .f5tts import F5TTSModel
 from .f5tts_mlx import F5TTSMLXModel
 from .fish_speech import FishSpeechModel
 from .funasr import FunASRModel
+from .indextts2 import Indextts2
 from .kokoro import KokoroModel
 from .kokoro_mlx import KokoroMLXModel
+from .kokoro_zh import KokoroZHModel
 from .megatts import MegaTTSModel
 from .melotts import MeloTTSModel
 from .whisper import WhisperModel
@@ -106,13 +108,23 @@ def match_audio(
 
     if model_name in BUILTIN_AUDIO_MODELS:
         model_families = BUILTIN_AUDIO_MODELS[model_name]
-        if download_hub == "modelscope" or download_from_modelscope():
-            return (
-                [x for x in model_families if x.model_hub == "modelscope"]
-                + [x for x in model_families if x.model_hub == "huggingface"]
-            )[0]
+        if download_hub is not None:
+            if download_hub == "modelscope":
+                return (
+                    [x for x in model_families if x.model_hub == "modelscope"]
+                    + [x for x in model_families if x.model_hub == "huggingface"]
+                )[0]
+            else:
+                return [x for x in model_families if x.model_hub == download_hub][0]
         else:
-            return [x for x in model_families if x.model_hub == "huggingface"][0]
+            if download_from_modelscope():
+                return (
+                    [x for x in model_families if x.model_hub == "modelscope"]
+                    + [x for x in model_families if x.model_hub == "huggingface"]
+                )[0]
+            else:
+                return [x for x in model_families if x.model_hub == "huggingface"][0]
+
     else:
         raise ValueError(
             f"Audio model {model_name} not found, available"
@@ -140,7 +152,9 @@ def create_audio_model_instance(
     MeloTTSModel,
     KokoroModel,
     KokoroMLXModel,
+    KokoroZHModel,
     MegaTTSModel,
+    Indextts2,
 ]:
     from ..cache_manager import CacheManager
 
@@ -160,7 +174,9 @@ def create_audio_model_instance(
         MeloTTSModel,
         KokoroModel,
         KokoroMLXModel,
+        KokoroZHModel,
         MegaTTSModel,
+        Indextts2,
     ]
     if model_spec.model_family == "whisper":
         if not model_spec.engine:
@@ -183,10 +199,14 @@ def create_audio_model_instance(
         model = MeloTTSModel(model_uid, model_path, model_spec, **kwargs)
     elif model_spec.model_family == "Kokoro":
         model = KokoroModel(model_uid, model_path, model_spec, **kwargs)
+    elif model_spec.model_family == "Kokoro-zh":
+        model = KokoroZHModel(model_uid, model_path, model_spec, **kwargs)
     elif model_spec.model_family == "Kokoro-MLX":
         model = KokoroMLXModel(model_uid, model_path, model_spec, **kwargs)
     elif model_spec.model_family == "MegaTTS":
         model = MegaTTSModel(model_uid, model_path, model_spec, **kwargs)
+    elif model_spec.model_family == "IndexTTS2":
+        model = Indextts2(model_uid, model_path, model_spec, **kwargs)
     else:
         raise Exception(f"Unsupported audio model family: {model_spec.model_family}")
     return model

xinference/model/audio/indextts2.py

@@ -0,0 +1,166 @@
+# Copyright 2022-2025 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+import os
+import sys
+from typing import TYPE_CHECKING, Optional
+
+from ..utils import set_all_random_seed
+
+if TYPE_CHECKING:
+    from .core import AudioModelFamilyV2
+
+logger = logging.getLogger(__name__)
+
+
+class Indextts2:
+    def __init__(
+        self,
+        model_uid: str,
+        model_path: str,
+        model_spec: "AudioModelFamilyV2",
+        device: Optional[str] = None,
+        **kwargs,
+    ):
+        self.model_family = model_spec
+        self._model_uid = model_uid
+        self._model_path = model_path
+        self._model_spec = model_spec
+        self._device = device
+        self._model = None
+        self._kwargs = kwargs
+
+    @property
+    def model_ability(self):
+        return self._model_spec.model_ability
+
+    def load(self):
+        # The yaml config loaded from model has hard-coded the import paths
+        thirdparty_dir = os.path.join(os.path.dirname(__file__), "../../thirdparty")
+        sys.path.insert(0, thirdparty_dir)
+
+        from indextts.infer_v2 import IndexTTS2
+
+        config_path = os.path.join(self._model_path, "config.yaml")
+        use_fp16 = self._kwargs.get("use_fp16", False)
+        use_deepspeed = self._kwargs.get("use_deepspeed", False)
+
+        logger.info("Loading IndexTTS2 model...")
+        self._model = IndexTTS2(
+            cfg_path=config_path,
+            model_dir=self._model_path,
+            use_fp16=use_fp16,
+            device=self._device,
+            use_deepspeed=use_deepspeed,
+        )
+
+    def speech(
+        self,
+        input: str,
+        voice: str,
+        response_format: str = "mp3",
+        speed: float = 1.0,
+        stream: bool = False,
+        **kwargs,
+    ):
+        from io import BytesIO
+
+        import soundfile
+
+        if stream:
+            raise Exception("IndexTTS2 does not support stream generation.")
+
+        prompt_speech: Optional[bytes] = kwargs.pop("prompt_speech", None)
+        emo_prompt_speech: Optional[bytes] = kwargs.pop("emo_prompt_speech", None)
+        emo_alpha: float = kwargs.pop("emo_alpha", 1.0)
+        emo_text: Optional[str] = kwargs.pop("emo_text", None)
+        use_random: bool = kwargs.pop("use_random", False)
+        emo_vector: Optional[list] = kwargs.pop("emo_vector", None)
+        seed: Optional[int] = kwargs.pop("seed", 0)
+        use_emo_text: bool = kwargs.pop("use_emo_text", False)
+
+        if prompt_speech is None:
+            # IndexTTS2 requires reference audio for voice cloning
+            # We'll provide a helpful error message with usage examples
+            raise ValueError(
+                "IndexTTS2 requires a reference audio for voice cloning.\n"
+                "Please provide a short audio sample (3-10 seconds) as 'prompt_speech' parameter.\n"
+                "Example usage:\n"
+                "  with open('reference.wav', 'rb') as f:\n"
+                "      prompt_speech = f.read()\n"
+                "  audio_bytes = model.speech(\n"
+                "      input='Hello, world!',\n"
+                "      voice='default',\n"
+                "      prompt_speech=prompt_speech"
+                "  )\n\n"
+                "For emotion control, you can also add:\n"
+                "  emo_prompt_speech=emotion_audio_bytes  # Optional: emotion reference\n"
+                "  emo_text='happy and cheerful'  # Optional: emotion description\n"
+                "  emo_alpha=1.5  # Optional: emotion intensity"
+            )
+
+        assert self._model is not None
+
+        set_all_random_seed(seed)
+
+        # Save prompt speech to temp file
+        import tempfile
+
+        with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as temp_prompt:
+            temp_prompt.write(prompt_speech)
+            temp_prompt_path = temp_prompt.name
+
+        emo_prompt_path = None
+        if emo_prompt_speech is not None:
+            with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as temp_emo:
+                temp_emo.write(emo_prompt_speech)
+                emo_prompt_path = temp_emo.name
+
+        try:
+            # Generate audio
+            with tempfile.NamedTemporaryFile(
+                suffix=".wav", delete=False
+            ) as temp_output:
+                output_path = temp_output.name
+
+            self._model.infer(
+                spk_audio_prompt=temp_prompt_path,
+                text=input,
+                output_path=output_path,
+                emo_audio_prompt=emo_prompt_path,
+                emo_alpha=emo_alpha,
+                emo_text=emo_text,
+                use_random=use_random,
+                emo_vector=emo_vector,
+                use_emo_text=use_emo_text,
+            )
+
+            # Read generated audio and convert to requested format
+            audio, sample_rate = soundfile.read(output_path)
+
+            with BytesIO() as out:
+                with soundfile.SoundFile(
+                    out, "w", sample_rate, 1, format=response_format.upper()
+                ) as f:
+                    f.write(audio)
+                return out.getvalue()
+        finally:
+            # Clean up temp files
+            try:
+                os.unlink(temp_prompt_path)
+                os.unlink(output_path)
+                if emo_prompt_path:
+                    os.unlink(emo_prompt_path)
+            except:
+                pass

xinference/model/audio/kokoro.py

@@ -81,7 +81,7 @@ class KokoroModel:
         logger.info("Launching Kokoro model with language code: %s", lang_code)
         self._model = KPipeline(
             lang_code=lang_code,
-            model=KModel(config=config_path, model=model_path),
+            model=KModel(config=config_path, model=model_path).to(self._device),
             device=self._device,
         )