xinference 1.7.1__py3-none-any.whl → 1.8.0__py3-none-any.whl

xinference/core/worker.py

@@ -54,7 +54,7 @@ from ..constants import (
 from ..core.model import ModelActor
 from ..core.status_guard import LaunchStatus
 from ..device_utils import get_available_device_env_name, gpu_count
-from ..model.core import ModelDescription, VirtualEnvSettings, create_model_instance
+from ..model.core import VirtualEnvSettings, create_model_instance
 from ..model.utils import CancellableDownloader, get_engine_params_by_name
 from ..types import PeftModelConfig
 from ..utils import get_pip_config_args, get_real_path
@@ -131,14 +131,14 @@ class WorkerActor(xo.StatelessActor):
         self._model_uid_launching_guard: Dict[str, LaunchInfo] = {}
         # attributes maintained after model launched:
         self._model_uid_to_model: Dict[str, xo.ActorRefType["ModelActor"]] = {}
-        self._model_uid_to_model_spec: Dict[str, ModelDescription] = {}
+        self._model_uid_to_model_spec: Dict[str, Dict[str, Any]] = {}
         self._model_uid_to_model_status: Dict[str, ModelStatus] = {}
         self._gpu_to_model_uid: Dict[int, str] = {}
         self._gpu_to_embedding_model_uids: Dict[int, Set[str]] = defaultdict(set)
         # Dict structure: gpu_index: {(replica_model_uid, model_type)}
-        self._user_specified_gpu_to_model_uids: Dict[
-            int, Set[Tuple[str, str]]
-        ] = defaultdict(set)
+        self._user_specified_gpu_to_model_uids: Dict[int, Set[Tuple[str, str]]] = (
+            defaultdict(set)
+        )
         self._model_uid_to_addr: Dict[str, str] = {}
         self._model_uid_to_recover_count: Dict[str, Optional[int]] = {}
         self._model_uid_to_launch_args: Dict[str, Dict] = {}
@@ -236,13 +236,13 @@ class WorkerActor(xo.StatelessActor):
 
     async def __post_create__(self):
         from ..model.audio import (
-            CustomAudioModelFamilyV1,
+            CustomAudioModelFamilyV2,
             generate_audio_description,
             register_audio,
             unregister_audio,
         )
         from ..model.embedding import (
-            CustomEmbeddingModelSpec,
+            CustomEmbeddingModelFamilyV2,
             generate_embedding_description,
             register_embedding,
             unregister_embedding,
@@ -254,19 +254,19 @@ class WorkerActor(xo.StatelessActor):
             unregister_flexible_model,
         )
         from ..model.image import (
-            CustomImageModelFamilyV1,
+            CustomImageModelFamilyV2,
             generate_image_description,
             register_image,
             unregister_image,
         )
         from ..model.llm import (
-            CustomLLMFamilyV1,
-            generate_llm_description,
+            CustomLLMFamilyV2,
+            generate_llm_version_info,
             register_llm,
             unregister_llm,
         )
         from ..model.rerank import (
-            CustomRerankModelSpec,
+            CustomRerankModelFamilyV2,
             generate_rerank_description,
             register_rerank,
             unregister_rerank,
@@ -274,31 +274,31 @@ class WorkerActor(xo.StatelessActor):
 
         self._custom_register_type_to_cls: Dict[str, Tuple] = {  # type: ignore
             "LLM": (
-                CustomLLMFamilyV1,
+                CustomLLMFamilyV2,
                 register_llm,
                 unregister_llm,
-                generate_llm_description,
+                generate_llm_version_info,
             ),
             "embedding": (
-                CustomEmbeddingModelSpec,
+                CustomEmbeddingModelFamilyV2,
                 register_embedding,
                 unregister_embedding,
                 generate_embedding_description,
             ),
             "rerank": (
-                CustomRerankModelSpec,
+                CustomRerankModelFamilyV2,
                 register_rerank,
                 unregister_rerank,
                 generate_rerank_description,
             ),
             "image": (
-                CustomImageModelFamilyV1,
+                CustomImageModelFamilyV2,
                 register_image,
                 unregister_image,
                 generate_image_description,
             ),
             "audio": (
-                CustomAudioModelFamilyV1,
+                CustomAudioModelFamilyV2,
                 register_audio,
                 unregister_audio,
                 generate_audio_description,
@@ -396,16 +396,18 @@ class WorkerActor(xo.StatelessActor):
         from ..model.embedding import get_embedding_model_descriptions
         from ..model.flexible import get_flexible_model_descriptions
         from ..model.image import get_image_model_descriptions
-        from ..model.llm import get_llm_model_descriptions
+        from ..model.llm import get_llm_version_infos
         from ..model.rerank import get_rerank_model_descriptions
+        from ..model.video import get_video_model_descriptions
 
         # record model version
         model_version_infos: Dict[str, List[Dict]] = {}  # type: ignore
-        model_version_infos.update(get_llm_model_descriptions())
+        model_version_infos.update(get_llm_version_infos())
         model_version_infos.update(get_embedding_model_descriptions())
         model_version_infos.update(get_rerank_model_descriptions())
         model_version_infos.update(get_image_model_descriptions())
         model_version_infos.update(get_audio_model_descriptions())
+        model_version_infos.update(get_video_model_descriptions())
         model_version_infos.update(get_flexible_model_descriptions())
         await self._cache_tracker_ref.record_model_version(
             model_version_infos, self.address
@@ -774,10 +776,7 @@ class WorkerActor(xo.StatelessActor):
             assert isinstance(model, LLM)
             return model.model_family.model_ability  # type: ignore
 
-    async def update_cache_status(
-        self, model_name: str, model_description: ModelDescription
-    ):
-        version_info = model_description.to_version_info()
+    async def update_cache_status(self, model_name: str, version_info: Any):
         if isinstance(version_info, list):  # image model
             model_path = version_info[0]["model_file_location"]
             await self._cache_tracker_ref.update_cache_status(
@@ -1028,10 +1027,8 @@ class WorkerActor(xo.StatelessActor):
                                 self._upload_download_progress, progressor, downloader
                             )
                         )
-                        model, model_description = await asyncio.to_thread(
+                        model = await asyncio.to_thread(
                             create_model_instance,
-                            subpool_address,
-                            devices,
                             model_uid,
                             model_type,
                             model_name,
@@ -1044,7 +1041,14 @@ class WorkerActor(xo.StatelessActor):
                             model_path,
                             **model_kwargs,
                         )
-                    await self.update_cache_status(model_name, model_description)
+                    model.model_family.address = subpool_address
+                    model.model_family.accelerators = devices
+                    model.model_family.multimodal_projector = model_kwargs.get(
+                        "multimodal_projector", None
+                    )
+                    await self.update_cache_status(
+                        model_name, model.model_family.to_version_info()
+                    )
 
                 def check_cancel():
                     # check downloader first, sometimes download finished
@@ -1063,7 +1067,7 @@ class WorkerActor(xo.StatelessActor):
                     await asyncio.to_thread(
                         self._prepare_virtual_env,
                         virtual_env_manager,
-                        model_description.spec.virtualenv,
+                        model.model_family.virtualenv,
                     )
                     launch_info.virtual_env_manager = virtual_env_manager
 
@@ -1078,7 +1082,6 @@ class WorkerActor(xo.StatelessActor):
                     worker_address=self.address,
                     replica_model_uid=model_uid,
                     model=model,
-                    model_description=model_description,
                     request_limits=request_limits,
                     xavier_config=xavier_config,
                     n_worker=n_worker,
@@ -1125,7 +1128,9 @@ class WorkerActor(xo.StatelessActor):
                         continue
                 raise
             self._model_uid_to_model[model_uid] = model_ref
-            self._model_uid_to_model_spec[model_uid] = model_description
+            self._model_uid_to_model_spec[model_uid] = (
+                model.model_family.to_description()
+            )
             self._model_uid_to_model_status[model_uid] = ModelStatus()
             self._model_uid_to_addr[model_uid] = subpool_address
             self._model_uid_to_recover_count.setdefault(
@@ -1301,12 +1306,7 @@ class WorkerActor(xo.StatelessActor):
 
     @log_async(logger=logger)
     async def list_models(self) -> Dict[str, Dict[str, Any]]:
-        ret = {}
-
-        items = list(self._model_uid_to_model_spec.items())
-        for k, v in items:
-            ret[k] = v.to_dict()
-        return ret
+        return {k: v for k, v in self._model_uid_to_model_spec.items()}
 
     @log_sync(logger=logger)
     def get_model(self, model_uid: str) -> xo.ActorRefType["ModelActor"]:
@@ -1323,7 +1323,7 @@ class WorkerActor(xo.StatelessActor):
         model_desc = self._model_uid_to_model_spec.get(model_uid, None)
         if model_desc is None:
             raise ValueError(f"Model not found in the model list, uid: {model_uid}")
-        return model_desc.to_dict()
+        return model_desc
 
     async def report_status(self):
         status = dict()
@@ -1409,7 +1409,9 @@ class WorkerActor(xo.StatelessActor):
 
     async def confirm_and_remove_model(self, model_version: str) -> bool:
         paths = await self.list_deletable_models(model_version)
+        dir_paths = set()
         for path in paths:
+            dir_paths.add(os.path.dirname(path))
             try:
                 if os.path.islink(path):
                     os.unlink(path)
@@ -1422,6 +1424,16 @@ class WorkerActor(xo.StatelessActor):
             except Exception as e:
                 logger.error(f"Fail to delete {path} with error:{e}.")  # noqa: E231
                 return False
+
+        for _dir in dir_paths:
+            try:
+                shutil.rmtree(_dir)
+            except Exception as e:
+                logger.error(
+                    f"Fail to delete parent dir {_dir} with error:{e}."
+                )  # noqa: E231
+                return False
+
         await self._cache_tracker_ref.confirm_and_remove_model(
             model_version, self.address
         )

xinference/deploy/test/test_cmdline.py

@@ -185,7 +185,7 @@ def test_cmdline_of_custom_model(setup):
 
     # register custom model
     custom_model_desc = """{
-  "version": 1,
+  "version": 2,
   "context_length":2048,
   "model_name": "custom_model",
   "model_lang": [
@@ -200,11 +200,7 @@ def test_cmdline_of_custom_model(setup):
     {
       "model_format": "pytorch",
       "model_size_in_billions": 7,
-      "quantizations": [
-        "4-bit",
-        "8-bit",
-        "none"
-      ],
+      "quantization": "none",
       "model_id": "ziqingyang/chinese-alpaca-2-7b"
     }
   ],

xinference/model/audio/__init__.py

@@ -18,38 +18,41 @@ import os
 import platform
 import sys
 import warnings
-from typing import Any, Dict
+from typing import Dict, List
 
 from ...constants import XINFERENCE_MODEL_DIR
+from ..utils import flatten_model_src
 from .core import (
     AUDIO_MODEL_DESCRIPTIONS,
-    MODEL_NAME_TO_REVISION,
-    AudioModelFamilyV1,
+    AudioModelFamilyV2,
     generate_audio_description,
     get_audio_model_descriptions,
-    get_cache_status,
 )
 from .custom import (
-    CustomAudioModelFamilyV1,
+    CustomAudioModelFamilyV2,
     get_user_defined_audios,
     register_audio,
     unregister_audio,
 )
 
-BUILTIN_AUDIO_MODELS: Dict[str, Any] = {}
-MODELSCOPE_AUDIO_MODELS: Dict[str, Any] = {}
+BUILTIN_AUDIO_MODELS: Dict[str, List["AudioModelFamilyV2"]] = {}
 
 
 def register_custom_model():
+    from ..custom import migrate_from_v1_to_v2
+
+    # migrate from v1 to v2 first
+    migrate_from_v1_to_v2("audio", CustomAudioModelFamilyV2)
+
     # if persist=True, load them when init
-    user_defined_audio_dir = os.path.join(XINFERENCE_MODEL_DIR, "audio")
+    user_defined_audio_dir = os.path.join(XINFERENCE_MODEL_DIR, "v2", "audio")
     if os.path.isdir(user_defined_audio_dir):
         for f in os.listdir(user_defined_audio_dir):
             try:
                 with codecs.open(
                     os.path.join(user_defined_audio_dir, f), encoding="utf-8"
                 ) as fd:
-                    user_defined_audio_family = CustomAudioModelFamilyV1.parse_obj(
+                    user_defined_audio_family = CustomAudioModelFamilyV2.parse_obj(
                         json.load(fd)
                     )
                     register_audio(user_defined_audio_family, persist=False)
@@ -67,13 +70,12 @@ def _need_filter(spec: dict):
 
 def _install():
     load_model_family_from_json("model_spec.json", BUILTIN_AUDIO_MODELS)
-    load_model_family_from_json("model_spec_modelscope.json", MODELSCOPE_AUDIO_MODELS)
 
     # register model description after recording model revision
-    for model_spec_info in [BUILTIN_AUDIO_MODELS, MODELSCOPE_AUDIO_MODELS]:
-        for model_name, model_spec in model_spec_info.items():
-            if model_spec.model_name not in AUDIO_MODEL_DESCRIPTIONS:
-                AUDIO_MODEL_DESCRIPTIONS.update(generate_audio_description(model_spec))
+    for model_name, model_specs in BUILTIN_AUDIO_MODELS.items():
+        model_spec = [x for x in model_specs if x.model_hub == "huggingface"][0]
+        if model_spec.model_name not in AUDIO_MODEL_DESCRIPTIONS:
+            AUDIO_MODEL_DESCRIPTIONS.update(generate_audio_description(model_spec))
 
     register_custom_model()
 
@@ -84,14 +86,15 @@ def _install():
 
 def load_model_family_from_json(json_filename, target_families):
     json_path = os.path.join(os.path.dirname(__file__), json_filename)
-    target_families.update(
-        dict(
-            (spec["model_name"], AudioModelFamilyV1(**spec))
-            for spec in json.load(codecs.open(json_path, "r", encoding="utf-8"))
-            if not _need_filter(spec)
-        )
-    )
-    for model_name, model_spec in target_families.items():
-        MODEL_NAME_TO_REVISION[model_name].append(model_spec.model_revision)
+    flattened_model_specs = []
+    for spec in json.load(codecs.open(json_path, "r", encoding="utf-8")):
+        flattened_model_specs.extend(flatten_model_src(spec))
+
+    for spec in flattened_model_specs:
+        if not _need_filter(spec):
+            if spec["model_name"] not in target_families:
+                target_families[spec["model_name"]] = [AudioModelFamilyV2(**spec)]
+            else:
+                target_families[spec["model_name"]].append(AudioModelFamilyV2(**spec))
 
     del json_path

xinference/model/audio/chattts.py

@@ -20,7 +20,7 @@ from typing import TYPE_CHECKING, Optional
 from ..utils import set_all_random_seed
 
 if TYPE_CHECKING:
-    from .core import AudioModelFamilyV1
+    from .core import AudioModelFamilyV2
 
 logger = logging.getLogger(__name__)
 
@@ -30,10 +30,11 @@ class ChatTTSModel:
         self,
         model_uid: str,
         model_path: str,
-        model_spec: "AudioModelFamilyV1",
+        model_spec: "AudioModelFamilyV2",
         device: Optional[str] = None,
         **kwargs,
     ):
+        self.model_family = model_spec
         self._model_uid = model_uid
         self._model_path = model_path
         self._model_spec = model_spec

xinference/model/audio/core.py

@@ -12,13 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-import os
 from collections import defaultdict
-from typing import Any, Dict, List, Literal, Optional, Tuple, Union
+from typing import Any, Dict, List, Literal, Optional, Union
 
-from ...constants import XINFERENCE_CACHE_DIR
-from ..core import CacheableModelSpec, ModelDescription, VirtualEnvSettings
-from ..utils import valid_model_revision
+from ..core import CacheableModelSpec, VirtualEnvSettings
+from ..utils import ModelInstanceInfoMixin
 from .chattts import ChatTTSModel
 from .cosyvoice import CosyVoiceModel
 from .f5tts import F5TTSModel
@@ -33,9 +31,7 @@ from .whisper_mlx import WhisperMLXModel
 
 logger = logging.getLogger(__name__)
 
-# Used for check whether the model is cached.
 # Init when registering all the builtin models.
-MODEL_NAME_TO_REVISION: Dict[str, List[str]] = defaultdict(list)
 AUDIO_MODEL_DESCRIPTIONS: Dict[str, List[Dict]] = defaultdict(list)
 
 
@@ -45,7 +41,8 @@ def get_audio_model_descriptions():
     return copy.deepcopy(AUDIO_MODEL_DESCRIPTIONS)
 
 
-class AudioModelFamilyV1(CacheableModelSpec):
+class AudioModelFamilyV2(CacheableModelSpec, ModelInstanceInfoMixin):
+    version: Literal[2]
     model_family: str
     model_name: str
     model_id: str
@@ -58,57 +55,37 @@ class AudioModelFamilyV1(CacheableModelSpec):
     engine: Optional[str]
     virtualenv: Optional[VirtualEnvSettings]
 
+    class Config:
+        extra = "allow"
 
-class AudioModelDescription(ModelDescription):
-    def __init__(
-        self,
-        address: Optional[str],
-        devices: Optional[List[str]],
-        model_spec: AudioModelFamilyV1,
-        model_path: Optional[str] = None,
-    ):
-        super().__init__(address, devices, model_path=model_path)
-        self._model_spec = model_spec
-
-    @property
-    def spec(self):
-        return self._model_spec
-
-    def to_dict(self):
+    def to_description(self):
         return {
             "model_type": "audio",
-            "address": self.address,
-            "accelerators": self.devices,
-            "model_name": self._model_spec.model_name,
-            "model_family": self._model_spec.model_family,
-            "model_revision": self._model_spec.model_revision,
-            "model_ability": self._model_spec.model_ability,
+            "address": getattr(self, "address", None),
+            "accelerators": getattr(self, "accelerators", None),
+            "model_name": self.model_name,
+            "model_family": self.model_family,
+            "model_revision": self.model_revision,
+            "model_ability": self.model_ability,
         }
 
     def to_version_info(self):
-        from .utils import get_model_version
+        from ..cache_manager import CacheManager
 
-        if self._model_path is None:
-            is_cached = get_cache_status(self._model_spec)
-            file_location = get_cache_dir(self._model_spec)
-        else:
-            is_cached = True
-            file_location = self._model_path
+        cache_manager = CacheManager(self)
 
         return {
-            "model_version": get_model_version(self._model_spec),
-            "model_file_location": file_location,
-            "cache_status": is_cached,
+            "model_version": self.model_name,
+            "model_file_location": cache_manager.get_cache_dir(),
+            "cache_status": cache_manager.get_cache_status(),
         }
 
 
 def generate_audio_description(
-    image_model: AudioModelFamilyV1,
+    audio_model: AudioModelFamilyV2,
 ) -> Dict[str, List[Dict]]:
     res = defaultdict(list)
-    res[image_model.model_name].append(
-        AudioModelDescription(None, None, image_model).to_version_info()
-    )
+    res[audio_model.model_name].append(audio_model.to_version_info())
     return res
 
 
@@ -117,27 +94,24 @@ def match_audio(
     download_hub: Optional[
         Literal["huggingface", "modelscope", "openmind_hub", "csghub"]
     ] = None,
-) -> AudioModelFamilyV1:
+) -> AudioModelFamilyV2:
     from ..utils import download_from_modelscope
-    from . import BUILTIN_AUDIO_MODELS, MODELSCOPE_AUDIO_MODELS
+    from . import BUILTIN_AUDIO_MODELS
     from .custom import get_user_defined_audios
 
     for model_spec in get_user_defined_audios():
         if model_spec.model_name == model_name:
             return model_spec
 
-    if download_hub == "huggingface" and model_name in BUILTIN_AUDIO_MODELS:
-        logger.debug(f"Audio model {model_name} found in huggingface.")
-        return BUILTIN_AUDIO_MODELS[model_name]
-    elif download_hub == "modelscope" and model_name in MODELSCOPE_AUDIO_MODELS:
-        logger.debug(f"Audio model {model_name} found in ModelScope.")
-        return MODELSCOPE_AUDIO_MODELS[model_name]
-    elif download_from_modelscope() and model_name in MODELSCOPE_AUDIO_MODELS:
-        logger.debug(f"Audio model {model_name} found in ModelScope.")
-        return MODELSCOPE_AUDIO_MODELS[model_name]
-    elif model_name in BUILTIN_AUDIO_MODELS:
-        logger.debug(f"Audio model {model_name} found in huggingface.")
-        return BUILTIN_AUDIO_MODELS[model_name]
+    if model_name in BUILTIN_AUDIO_MODELS:
+        model_families = BUILTIN_AUDIO_MODELS[model_name]
+        if download_hub == "modelscope" or download_from_modelscope():
+            return (
+                [x for x in model_families if x.model_hub == "modelscope"]
+                + [x for x in model_families if x.model_hub == "huggingface"]
+            )[0]
+        else:
+            return [x for x in model_families if x.model_hub == "huggingface"][0]
     else:
         raise ValueError(
             f"Audio model {model_name} not found, available"
@@ -145,27 +119,7 @@ def match_audio(
         )
 
 
-def cache(model_spec: AudioModelFamilyV1):
-    from ..utils import cache
-
-    return cache(model_spec, AudioModelDescription)
-
-
-def get_cache_dir(model_spec: AudioModelFamilyV1):
-    return os.path.realpath(os.path.join(XINFERENCE_CACHE_DIR, model_spec.model_name))
-
-
-def get_cache_status(
-    model_spec: AudioModelFamilyV1,
-) -> bool:
-    cache_dir = get_cache_dir(model_spec)
-    meta_path = os.path.join(cache_dir, "__valid_download")
-    return valid_model_revision(meta_path, model_spec.model_revision)
-
-
 def create_audio_model_instance(
-    subpool_addr: str,
-    devices: List[str],
     model_uid: str,
     model_name: str,
     download_hub: Optional[
@@ -173,25 +127,25 @@ def create_audio_model_instance(
     ] = None,
     model_path: Optional[str] = None,
     **kwargs,
-) -> Tuple[
-    Union[
-        WhisperModel,
-        WhisperMLXModel,
-        FunASRModel,
-        ChatTTSModel,
-        CosyVoiceModel,
-        FishSpeechModel,
-        F5TTSModel,
-        F5TTSMLXModel,
-        MeloTTSModel,
-        KokoroModel,
-        MegaTTSModel,
-    ],
-    AudioModelDescription,
+) -> Union[
+    WhisperModel,
+    WhisperMLXModel,
+    FunASRModel,
+    ChatTTSModel,
+    CosyVoiceModel,
+    FishSpeechModel,
+    F5TTSModel,
+    F5TTSMLXModel,
+    MeloTTSModel,
+    KokoroModel,
+    MegaTTSModel,
 ]:
+    from ..cache_manager import CacheManager
+
     model_spec = match_audio(model_name, download_hub)
     if model_path is None:
-        model_path = cache(model_spec)
+        cache_manager = CacheManager(model_spec)
+        model_path = cache_manager.cache()
     model: Union[
         WhisperModel,
         WhisperMLXModel,
@@ -230,7 +184,4 @@ def create_audio_model_instance(
         model = MegaTTSModel(model_uid, model_path, model_spec, **kwargs)
     else:
         raise Exception(f"Unsupported audio model family: {model_spec.model_family}")
-    model_description = AudioModelDescription(
-        subpool_addr, devices, model_spec, model_path
-    )
-    return model, model_description
+    return model

xinference/model/audio/cosyvoice.py

@@ -18,7 +18,7 @@ from typing import TYPE_CHECKING, Optional
 from ..utils import set_all_random_seed
 
 if TYPE_CHECKING:
-    from .core import AudioModelFamilyV1
+    from .core import AudioModelFamilyV2
 
 logger = logging.getLogger(__name__)
 
@@ -28,10 +28,11 @@ class CosyVoiceModel:
         self,
         model_uid: str,
         model_path: str,
-        model_spec: "AudioModelFamilyV1",
+        model_spec: "AudioModelFamilyV2",
         device: Optional[str] = None,
         **kwargs,
     ):
+        self.model_family = model_spec
         self._model_uid = model_uid
         self._model_path = model_path
         self._model_spec = model_spec