xinference 1.10.0__py3-none-any.whl → 1.11.0__py3-none-any.whl

xinference/core/supervisor.py

@@ -31,6 +31,7 @@ from typing import (
     Literal,
     Optional,
     Tuple,
+    Type,
     Union,
 )
 
@@ -406,6 +407,26 @@ class SupervisorActor(xo.StatelessActor):
             "workers": self._worker_status,
         }
 
+    def _get_spec_dicts(
+        self, model_family: Any, cache_manager_cls: Type
+    ) -> Tuple[List[dict], List[str]]:
+        specs = []
+        download_hubs: Dict[str, None] = dict()
+        for spec in model_family.model_specs:
+            model_hub = spec.model_hub
+            if model_hub not in download_hubs:
+                download_hubs[model_hub] = None
+            if model_hub != "huggingface":
+                # since we only need to know all specs
+                # thus filter huggingface specs only
+                continue
+            model_family.model_specs = [spec]
+            cache_manager = cache_manager_cls(model_family)
+            specs.append(
+                {**spec.dict(), "cache_status": cache_manager.get_cache_status()}
+            )
+        return specs, list(download_hubs)
+
     async def _to_llm_reg(
         self, llm_family: "LLMFamilyV2", is_builtin: bool
     ) -> Dict[str, Any]:
@@ -415,20 +436,15 @@ class SupervisorActor(xo.StatelessActor):
         version_cnt = await self.get_model_version_count(llm_family.model_name)
 
         if self.is_local_deployment():
-            specs = []
             # TODO: does not work when the supervisor and worker are running on separate nodes.
             _llm_family = llm_family.copy()
-            for spec in [
-                _spec
-                for _spec in llm_family.model_specs
-                if _spec.model_hub == "huggingface"
-            ]:
-                _llm_family.model_specs = [spec]
-                cache_manager = LLMCacheManager(_llm_family)
-                specs.append(
-                    {**spec.dict(), "cache_status": cache_manager.get_cache_status()}
-                )
-            res = {**llm_family.dict(), "is_builtin": is_builtin, "model_specs": specs}
+            specs, download_hubs = self._get_spec_dicts(_llm_family, LLMCacheManager)
+            res = {
+                **llm_family.dict(),
+                "is_builtin": is_builtin,
+                "model_specs": specs,
+                "download_hubs": download_hubs,
+            }
         else:
             res = {**llm_family.dict(), "is_builtin": is_builtin}
         res["model_version_count"] = version_cnt
@@ -445,24 +461,13 @@ class SupervisorActor(xo.StatelessActor):
 
         if self.is_local_deployment():
             _family = model_family.copy()
-            specs = []
             # TODO: does not work when the supervisor and worker are running on separate nodes.
-            for spec in [
-                x for x in model_family.model_specs if x.model_hub == "huggingface"
-            ]:
-                _family.model_specs = [spec]
-                specs.append(
-                    {
-                        **spec.dict(),
-                        "cache_status": EmbeddingCacheManager(
-                            _family
-                        ).get_cache_status(),
-                    }
-                )
+            specs, download_hubs = self._get_spec_dicts(_family, EmbeddingCacheManager)
             res = {
                 **model_family.dict(),
                 "is_builtin": is_builtin,
                 "model_specs": specs,
+                "download_hubs": download_hubs,
             }
         else:
             res = {
@@ -474,25 +479,26 @@ class SupervisorActor(xo.StatelessActor):
         return res
 
     async def _to_rerank_model_reg(
-        self, model_spec: "RerankModelFamilyV2", is_builtin: bool
+        self, model_family: "RerankModelFamilyV2", is_builtin: bool
     ) -> Dict[str, Any]:
-        from ..model.rerank.cache_manager import RerankCacheManager as CacheManager
+        from ..model.rerank.cache_manager import RerankCacheManager
 
-        instance_cnt = await self.get_instance_count(model_spec.model_name)
-        version_cnt = await self.get_model_version_count(model_spec.model_name)
-        cache_manager = CacheManager(model_spec)
+        instance_cnt = await self.get_instance_count(model_family.model_name)
+        version_cnt = await self.get_model_version_count(model_family.model_name)
 
         if self.is_local_deployment():
+            _family = model_family.copy()
             # TODO: does not work when the supervisor and worker are running on separate nodes.
-            cache_status = cache_manager.get_cache_status()
+            specs, download_hubs = self._get_spec_dicts(_family, RerankCacheManager)
             res = {
-                **model_spec.dict(),
-                "cache_status": cache_status,
+                **model_family.dict(),
                 "is_builtin": is_builtin,
+                "model_specs": specs,
+                "download_hubs": download_hubs,
             }
         else:
             res = {
-                **model_spec.dict(),
+                **model_family.dict(),
                 "is_builtin": is_builtin,
             }
         res["model_version_count"] = version_cnt
@@ -657,7 +663,9 @@ class SupervisorActor(xo.StatelessActor):
             for model_name, families in BUILTIN_IMAGE_MODELS.items():
                 if detailed:
                     family = [x for x in families if x.model_hub == "huggingface"][0]
-                    ret.append(await self._to_image_model_reg(family, is_builtin=True))
+                    info = await self._to_image_model_reg(family, is_builtin=True)
+                    info["download_hubs"] = [x.model_hub for x in families]
+                    ret.append(info)
                 else:
                     ret.append({"model_name": model_name, "is_builtin": True})
 
@@ -680,7 +688,9 @@ class SupervisorActor(xo.StatelessActor):
             for model_name, families in BUILTIN_AUDIO_MODELS.items():
                 if detailed:
                     family = [x for x in families if x.model_hub == "huggingface"][0]
-                    ret.append(await self._to_audio_model_reg(family, is_builtin=True))
+                    info = await self._to_audio_model_reg(family, is_builtin=True)
+                    info["download_hubs"] = [x.model_hub for x in families]
+                    ret.append(info)
                 else:
                     ret.append({"model_name": model_name, "is_builtin": True})
 
@@ -702,7 +712,9 @@ class SupervisorActor(xo.StatelessActor):
             for model_name, families in BUILTIN_VIDEO_MODELS.items():
                 if detailed:
                     family = [x for x in families if x.model_hub == "huggingface"][0]
-                    ret.append(await self._to_video_model_reg(family, is_builtin=True))
+                    info = await self._to_video_model_reg(family, is_builtin=True)
+                    info["download_hubs"] = [x.model_hub for x in families]
+                    ret.append(info)
                 else:
                     ret.append({"model_name": model_name, "is_builtin": True})
 
@@ -812,16 +824,9 @@ class SupervisorActor(xo.StatelessActor):
             from ..model.rerank import BUILTIN_RERANK_MODELS
             from ..model.rerank.custom import get_user_defined_reranks
 
-            if model_name in BUILTIN_RERANK_MODELS:
-                return [
-                    x
-                    for x in BUILTIN_RERANK_MODELS[model_name]
-                    if x.model_hub == "huggingface"
-                ][0]
-            else:
-                for f in get_user_defined_reranks():
-                    if f.model_name == model_name:
-                        return f
+            for f in list(BUILTIN_RERANK_MODELS.values()) + get_user_defined_reranks():
+                if f.model_name == model_name:
+                    return f
             raise ValueError(f"Model {model_name} not found")
         elif model_type == "flexible":
             from ..model.flexible import get_flexible_models
@@ -830,6 +835,16 @@ class SupervisorActor(xo.StatelessActor):
                 if f.model_name == model_name:
                     return f
             raise ValueError(f"Model {model_name} not found")
+        elif model_type == "video":
+            from ..model.video import BUILTIN_VIDEO_MODELS
+
+            if model_name in BUILTIN_VIDEO_MODELS:
+                return [
+                    x
+                    for x in BUILTIN_VIDEO_MODELS[model_name]
+                    if x.model_hub == "huggingface"
+                ][0]
+            raise ValueError(f"Model {model_name} not found")
         else:
             raise ValueError(f"Unsupported model type: {model_type}")
 
@@ -864,6 +879,26 @@ class SupervisorActor(xo.StatelessActor):
                 generate_fn,
             ) = self._custom_register_type_to_cls[model_type]
 
+            model_spec = model_spec_cls.parse_raw(model)
+
+            # check if model already registered
+            try:
+                model = await self.get_model_registration(
+                    model_type, model_spec.model_name
+                )
+                if model is not None:
+                    raise ValueError(
+                        f"Model {model_spec.model_name} already registered"
+                    )
+            except ValueError as e:
+                if "not found" in str(e):
+                    pass
+                else:
+                    raise e
+            except Exception:
+                logger.error("Get model registration failed.", exc_info=True)
+                raise
+
             target_ip_worker_ref = (
                 self._get_worker_ref_by_ip(worker_ip) if worker_ip is not None else None
             )
@@ -880,7 +915,6 @@ class SupervisorActor(xo.StatelessActor):
                 await target_ip_worker_ref.register_model(model_type, model, persist)
                 return
 
-            model_spec = model_spec_cls.parse_raw(model)
             try:
                 register_fn(model_spec, persist)
                 await self._cache_tracker_ref.record_model_version(
@@ -901,25 +935,25 @@ class SupervisorActor(xo.StatelessActor):
     async def _sync_register_model(
         self, model_type: str, model: str, persist: bool, model_name: str
     ):
-        logger.info(f"begin sync model:{model_name} to worker")
+        logger.info(f"begin sync model: {model_name} to worker")
         try:
             # Sync model to all workers.
             for name, worker in self._worker_address_to_worker.items():
-                logger.info(f"sync model:{model_name} to {name}")
+                logger.info(f"sync model: {model_name} to {name}")
                 if name == self.address:
                     # Ignore: when worker and supervisor at the same node.
                     logger.info(
-                        f"ignore sync model:{model_name} to {name} for same node"
+                        f"ignore sync model: {model_name} to {name} for same node"
                     )
                 else:
                     await worker.register_model(model_type, model, persist)
-                    logger.info(f"success sync model:{model_name} to {name}")
+                    logger.info(f"success sync model: {model_name} to {name}")
         except Exception as e:
             # If sync fails, unregister the model in all workers.
             for name, worker in self._worker_address_to_worker.items():
                 logger.warning(f"ready to unregister model for {name}")
                 await worker.unregister_model(model_type, model_name)
-                logger.warning(f"finish unregister model:{model} for {name}")
+                logger.warning(f"finish unregister model: {model} for {name}")
             raise e
 
     @log_async(logger=logger)
@@ -1627,6 +1661,9 @@ class SupervisorActor(xo.StatelessActor):
         if isinstance(worker_ref, list):
             # get first worker to fetch information if model across workers
             worker_ref = worker_ref[0]
+        assert not isinstance(
+            worker_ref, (list, tuple)
+        ), "worker_ref must be a single worker"
         return await worker_ref.get_model(model_uid=replica_model_uid)
 
     @log_async(logger=logger)
@@ -1639,6 +1676,9 @@ class SupervisorActor(xo.StatelessActor):
         if isinstance(worker_ref, list):
             # get status from first shard if model has multiple shards across workers
             worker_ref = worker_ref[0]
+        assert not isinstance(
+            worker_ref, (list, tuple)
+        ), "worker_ref must be a single worker"
         return await worker_ref.get_model_status(replica_model_uid)
 
     @log_async(logger=logger)
@@ -1657,6 +1697,9 @@ class SupervisorActor(xo.StatelessActor):
         if isinstance(worker_ref, list):
             # get status from first shard if model has multiple shards across workers
             worker_ref = worker_ref[0]
+        assert not isinstance(
+            worker_ref, (list, tuple)
+        ), "worker_ref must be a single worker"
         info = await worker_ref.describe_model(model_uid=replica_model_uid)
         info["replica"] = replica_info.replica
         return info
@@ -1732,6 +1775,9 @@ class SupervisorActor(xo.StatelessActor):
             if isinstance(worker_ref, list):
                 # get status from first shard if model has multiple shards across workers
                 worker_ref = worker_ref[0]
+            assert not isinstance(
+                worker_ref, (list, tuple)
+            ), "worker_ref must be a single worker"
             model_ref = await worker_ref.get_model(model_uid=rep_mid)
             result_info = await model_ref.abort_request(request_id, block_duration)
             res["msg"] = result_info

xinference/core/worker.py

@@ -710,6 +710,16 @@ class WorkerActor(xo.StatelessActor):
             for model_spec in get_user_defined_reranks():
                 ret.append({"model_name": model_spec.model_name, "is_builtin": False})
 
+            ret.sort(key=sort_helper)
+            return ret
+        elif model_type == "flexible":
+            from ..model.flexible.custom import get_flexible_models
+
+            ret = []
+
+            for model_spec in get_flexible_models():
+                ret.append({"model_name": model_spec.model_name, "is_builtin": False})
+
             ret.sort(key=sort_helper)
             return ret
         else:

xinference/deploy/cmdline.py

@@ -576,6 +576,21 @@ def list_model_registrations(
             ),
             file=sys.stderr,
         )
+    elif model_type == "flexible":
+        for registration in registrations:
+            model_name = registration["model_name"]
+            model_family = client.get_model_registration(model_type, model_name)
+            table.append(
+                [
+                    model_type,
+                    model_family["model_name"],
+                    registration["is_builtin"],
+                ]
+            )
+        print(
+            tabulate(table, headers=["Type", "Name", "Is-built-in"]),
+            file=sys.stderr,
+        )
     else:
         raise NotImplementedError(f"List {model_type} is not implemented.")
 

xinference/model/audio/core.py

@@ -23,6 +23,7 @@ from .f5tts import F5TTSModel
 from .f5tts_mlx import F5TTSMLXModel
 from .fish_speech import FishSpeechModel
 from .funasr import FunASRModel
+from .indextts2 import Indextts2
 from .kokoro import KokoroModel
 from .kokoro_mlx import KokoroMLXModel
 from .kokoro_zh import KokoroZHModel
@@ -107,13 +108,23 @@ def match_audio(
 
     if model_name in BUILTIN_AUDIO_MODELS:
         model_families = BUILTIN_AUDIO_MODELS[model_name]
-        if download_hub == "modelscope" or download_from_modelscope():
-            return (
-                [x for x in model_families if x.model_hub == "modelscope"]
-                + [x for x in model_families if x.model_hub == "huggingface"]
-            )[0]
+        if download_hub is not None:
+            if download_hub == "modelscope":
+                return (
+                    [x for x in model_families if x.model_hub == "modelscope"]
+                    + [x for x in model_families if x.model_hub == "huggingface"]
+                )[0]
+            else:
+                return [x for x in model_families if x.model_hub == download_hub][0]
         else:
-            return [x for x in model_families if x.model_hub == "huggingface"][0]
+            if download_from_modelscope():
+                return (
+                    [x for x in model_families if x.model_hub == "modelscope"]
+                    + [x for x in model_families if x.model_hub == "huggingface"]
+                )[0]
+            else:
+                return [x for x in model_families if x.model_hub == "huggingface"][0]
+
     else:
         raise ValueError(
             f"Audio model {model_name} not found, available"
@@ -143,6 +154,7 @@ def create_audio_model_instance(
     KokoroMLXModel,
     KokoroZHModel,
     MegaTTSModel,
+    Indextts2,
 ]:
     from ..cache_manager import CacheManager
 
@@ -164,6 +176,7 @@ def create_audio_model_instance(
         KokoroMLXModel,
         KokoroZHModel,
         MegaTTSModel,
+        Indextts2,
     ]
     if model_spec.model_family == "whisper":
         if not model_spec.engine:
@@ -192,6 +205,8 @@ def create_audio_model_instance(
         model = KokoroMLXModel(model_uid, model_path, model_spec, **kwargs)
     elif model_spec.model_family == "MegaTTS":
         model = MegaTTSModel(model_uid, model_path, model_spec, **kwargs)
+    elif model_spec.model_family == "IndexTTS2":
+        model = Indextts2(model_uid, model_path, model_spec, **kwargs)
     else:
         raise Exception(f"Unsupported audio model family: {model_spec.model_family}")
     return model

xinference/model/audio/indextts2.py

@@ -0,0 +1,166 @@
+# Copyright 2022-2025 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+import os
+import sys
+from typing import TYPE_CHECKING, Optional
+
+from ..utils import set_all_random_seed
+
+if TYPE_CHECKING:
+    from .core import AudioModelFamilyV2
+
+logger = logging.getLogger(__name__)
+
+
+class Indextts2:
+    def __init__(
+        self,
+        model_uid: str,
+        model_path: str,
+        model_spec: "AudioModelFamilyV2",
+        device: Optional[str] = None,
+        **kwargs,
+    ):
+        self.model_family = model_spec
+        self._model_uid = model_uid
+        self._model_path = model_path
+        self._model_spec = model_spec
+        self._device = device
+        self._model = None
+        self._kwargs = kwargs
+
+    @property
+    def model_ability(self):
+        return self._model_spec.model_ability
+
+    def load(self):
+        # The yaml config loaded from model has hard-coded the import paths
+        thirdparty_dir = os.path.join(os.path.dirname(__file__), "../../thirdparty")
+        sys.path.insert(0, thirdparty_dir)
+
+        from indextts.infer_v2 import IndexTTS2
+
+        config_path = os.path.join(self._model_path, "config.yaml")
+        use_fp16 = self._kwargs.get("use_fp16", False)
+        use_deepspeed = self._kwargs.get("use_deepspeed", False)
+
+        logger.info("Loading IndexTTS2 model...")
+        self._model = IndexTTS2(
+            cfg_path=config_path,
+            model_dir=self._model_path,
+            use_fp16=use_fp16,
+            device=self._device,
+            use_deepspeed=use_deepspeed,
+        )
+
+    def speech(
+        self,
+        input: str,
+        voice: str,
+        response_format: str = "mp3",
+        speed: float = 1.0,
+        stream: bool = False,
+        **kwargs,
+    ):
+        from io import BytesIO
+
+        import soundfile
+
+        if stream:
+            raise Exception("IndexTTS2 does not support stream generation.")
+
+        prompt_speech: Optional[bytes] = kwargs.pop("prompt_speech", None)
+        emo_prompt_speech: Optional[bytes] = kwargs.pop("emo_prompt_speech", None)
+        emo_alpha: float = kwargs.pop("emo_alpha", 1.0)
+        emo_text: Optional[str] = kwargs.pop("emo_text", None)
+        use_random: bool = kwargs.pop("use_random", False)
+        emo_vector: Optional[list] = kwargs.pop("emo_vector", None)
+        seed: Optional[int] = kwargs.pop("seed", 0)
+        use_emo_text: bool = kwargs.pop("use_emo_text", False)
+
+        if prompt_speech is None:
+            # IndexTTS2 requires reference audio for voice cloning
+            # We'll provide a helpful error message with usage examples
+            raise ValueError(
+                "IndexTTS2 requires a reference audio for voice cloning.\n"
+                "Please provide a short audio sample (3-10 seconds) as 'prompt_speech' parameter.\n"
+                "Example usage:\n"
+                "  with open('reference.wav', 'rb') as f:\n"
+                "      prompt_speech = f.read()\n"
+                "  audio_bytes = model.speech(\n"
+                "      input='Hello, world!',\n"
+                "      voice='default',\n"
+                "      prompt_speech=prompt_speech"
+                "  )\n\n"
+                "For emotion control, you can also add:\n"
+                "  emo_prompt_speech=emotion_audio_bytes  # Optional: emotion reference\n"
+                "  emo_text='happy and cheerful'  # Optional: emotion description\n"
+                "  emo_alpha=1.5  # Optional: emotion intensity"
+            )
+
+        assert self._model is not None
+
+        set_all_random_seed(seed)
+
+        # Save prompt speech to temp file
+        import tempfile
+
+        with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as temp_prompt:
+            temp_prompt.write(prompt_speech)
+            temp_prompt_path = temp_prompt.name
+
+        emo_prompt_path = None
+        if emo_prompt_speech is not None:
+            with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as temp_emo:
+                temp_emo.write(emo_prompt_speech)
+                emo_prompt_path = temp_emo.name
+
+        try:
+            # Generate audio
+            with tempfile.NamedTemporaryFile(
+                suffix=".wav", delete=False
+            ) as temp_output:
+                output_path = temp_output.name
+
+            self._model.infer(
+                spk_audio_prompt=temp_prompt_path,
+                text=input,
+                output_path=output_path,
+                emo_audio_prompt=emo_prompt_path,
+                emo_alpha=emo_alpha,
+                emo_text=emo_text,
+                use_random=use_random,
+                emo_vector=emo_vector,
+                use_emo_text=use_emo_text,
+            )
+
+            # Read generated audio and convert to requested format
+            audio, sample_rate = soundfile.read(output_path)
+
+            with BytesIO() as out:
+                with soundfile.SoundFile(
+                    out, "w", sample_rate, 1, format=response_format.upper()
+                ) as f:
+                    f.write(audio)
+                return out.getvalue()
+        finally:
+            # Clean up temp files
+            try:
+                os.unlink(temp_prompt_path)
+                os.unlink(output_path)
+                if emo_prompt_path:
+                    os.unlink(emo_prompt_path)
+            except:
+                pass

xinference/model/audio/model_spec.json

@@ -685,7 +685,7 @@
         "model_revision": "4dcc16f297f2ff98a17b3726b16f5de5a5e45672"
       },
       "modelscope": {
-        "model_id": "SWivid/F5-TTS_Emilia-ZH-EN",
+        "model_id": "AI-ModelScope/F5-TTS",
         "model_revision": "master"
       }
     }
@@ -862,26 +862,6 @@
         "model_revision": "master"
       }
     }
-  },
-    {
-    "version": 2,
-    "model_name": "Kokoro-82M-v1.1-zh",
-    "model_family": "Kokoro-zh",
-    "model_ability": [
-      "text2audio",
-      "text2audio_zero_shot"
-    ],
-    "multilingual": false,
-    "model_src": {
-      "huggingface": {
-        "model_id": "hexgrad/Kokoro-82M-v1.1-zh",
-        "model_revision": "01e7505bd6a7a2ac4975463114c3a7650a9f7218"
-      },
-      "modelscope": {
-        "model_id": "AI-ModelScope/Kokoro-82M-v1.1-zh",
-        "model_revision": "master"
-      }
-    }
   },
   {
     "version": 2,
@@ -931,5 +911,62 @@
         "model_revision": "master"
       }
     }
+  },
+  {
+    "version": 2,
+    "model_name": "Kokoro-82M-v1.1-zh",
+    "model_family": "Kokoro-zh",
+    "model_ability": [
+      "text2audio",
+      "text2audio_zero_shot"
+    ],
+    "multilingual": false,
+    "model_src": {
+      "huggingface": {
+        "model_id": "hexgrad/Kokoro-82M-v1.1-zh",
+        "model_revision": "01e7505bd6a7a2ac4975463114c3a7650a9f7218"
+      },
+      "modelscope": {
+        "model_id": "AI-ModelScope/Kokoro-82M-v1.1-zh",
+        "model_revision": "master"
+      }
+    }
+  },
+  {
+    "version": 2,
+    "model_name": "IndexTTS2",
+    "model_family": "IndexTTS2",
+    "model_ability": [
+      "text2audio",
+      "text2audio_zero_shot",
+      "text2audio_voice_cloning",
+      "text2audio_emotion_control"
+    ],
+    "multilingual": true,
+    "virtualenv": {
+      "packages": [
+        "transformers==4.52.1",
+        "#system_torch#",
+        "#system_numpy#",
+        "json5",
+        "munch",
+        "matplotlib",
+        "flatten_dict",
+        "julius",
+        "tensorboard",
+        "randomname",
+        "argbind"
+      ]
+    },
+    "model_src": {
+      "huggingface": {
+        "model_id": "IndexTeam/IndexTTS-2",
+        "model_revision": "main"
+      },
+      "modelscope": {
+        "model_id": "IndexTeam/IndexTTS-2",
+        "model_revision": "master"
+      }
+    }
   }
 ]