xinference 0.10.1__py3-none-any.whl → 0.10.2.post1__py3-none-any.whl

xinference/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2024-04-11T15:35:46+0800",
+ "date": "2024-04-19T14:40:59+0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "e3a947ebddfc53b5e8ec723c1f632c2b895edef1",
- "version": "0.10.1"
+ "full-revisionid": "500171569de25d49f6ddb3c167d9fc0e55cd66c7",
+ "version": "0.10.2.post1"
 }
 '''  # END VERSION_JSON
 

xinference/api/restful_api.py

@@ -64,6 +64,7 @@ from ..types import (
     CreateChatCompletion,
     CreateCompletion,
     ImageList,
+    PeftModelConfig,
     max_tokens_field,
 )
 from .oauth2.auth_service import AuthService
@@ -692,9 +693,7 @@ class RESTfulAPI:
         replica = payload.get("replica", 1)
         n_gpu = payload.get("n_gpu", "auto")
         request_limits = payload.get("request_limits", None)
-        peft_model_path = payload.get("peft_model_path", None)
-        image_lora_load_kwargs = payload.get("image_lora_load_kwargs", None)
-        image_lora_fuse_kwargs = payload.get("image_lora_fuse_kwargs", None)
+        peft_model_config = payload.get("peft_model_config", None)
         worker_ip = payload.get("worker_ip", None)
         gpu_idx = payload.get("gpu_idx", None)
 
@@ -708,9 +707,7 @@ class RESTfulAPI:
             "replica",
             "n_gpu",
             "request_limits",
-            "peft_model_path",
-            "image_lora_load_kwargs",
-            "image_lora_fuse_kwargs",
+            "peft_model_config",
             "worker_ip",
             "gpu_idx",
         }
@@ -725,6 +722,11 @@ class RESTfulAPI:
                 detail="Invalid input. Please specify the model name",
             )
 
+        if peft_model_config is not None:
+            peft_model_config = PeftModelConfig.from_dict(peft_model_config)
+        else:
+            peft_model_config = None
+
         try:
             model_uid = await (await self._get_supervisor_ref()).launch_builtin_model(
                 model_uid=model_uid,
@@ -737,9 +739,7 @@ class RESTfulAPI:
                 n_gpu=n_gpu,
                 request_limits=request_limits,
                 wait_ready=wait_ready,
-                peft_model_path=peft_model_path,
-                image_lora_load_kwargs=image_lora_load_kwargs,
-                image_lora_fuse_kwargs=image_lora_fuse_kwargs,
+                peft_model_config=peft_model_config,
                 worker_ip=worker_ip,
                 gpu_idx=gpu_idx,
                 **kwargs,

xinference/client/restful/restful_client.py

@@ -35,6 +35,17 @@ if TYPE_CHECKING:
     )
 
 
+def convert_float_to_int_or_str(model_size: float) -> Union[int, str]:
+    """convert float to int or string
+
+    if float can be presented as int, convert it to int, otherwise convert it to string
+    """
+    if int(model_size) == model_size:
+        return int(model_size)
+    else:
+        return str(model_size)
+
+
 def _get_error_string(response: requests.Response) -> str:
     try:
         if response.content:
@@ -746,7 +757,7 @@ class Client:
     def launch_speculative_llm(
         self,
         model_name: str,
-        model_size_in_billions: Optional[int],
+        model_size_in_billions: Optional[Union[int, str, float]],
         quantization: Optional[str],
         draft_model_name: str,
         draft_model_size_in_billions: Optional[int],
@@ -767,6 +778,10 @@ class Client:
             "`launch_speculative_llm` is an experimental feature and the API may change in the future."
         )
 
+        # convert float to int or string since the RESTful API does not accept float.
+        if isinstance(model_size_in_billions, float):
+            model_size_in_billions = convert_float_to_int_or_str(model_size_in_billions)
+
         payload = {
             "model_uid": None,
             "model_name": model_name,
@@ -794,15 +809,13 @@ class Client:
         model_name: str,
         model_type: str = "LLM",
         model_uid: Optional[str] = None,
-        model_size_in_billions: Optional[Union[int, str]] = None,
+        model_size_in_billions: Optional[Union[int, str, float]] = None,
         model_format: Optional[str] = None,
         quantization: Optional[str] = None,
         replica: int = 1,
         n_gpu: Optional[Union[int, str]] = "auto",
+        peft_model_config: Optional[Dict] = None,
         request_limits: Optional[int] = None,
-        peft_model_path: Optional[str] = None,
-        image_lora_load_kwargs: Optional[Dict] = None,
-        image_lora_fuse_kwargs: Optional[Dict] = None,
         worker_ip: Optional[str] = None,
         gpu_idx: Optional[Union[int, List[int]]] = None,
         **kwargs,
@@ -818,7 +831,7 @@ class Client:
             type of model.
         model_uid: str
             UID of model, auto generate a UUID if is None.
-        model_size_in_billions: Optional[int]
+        model_size_in_billions: Optional[Union[int, str, float]]
             The size (in billions) of the model.
         model_format: Optional[str]
             The format of the model.
@@ -829,15 +842,13 @@ class Client:
         n_gpu: Optional[Union[int, str]],
             The number of GPUs used by the model, default is "auto".
             ``n_gpu=None`` means cpu only, ``n_gpu=auto`` lets the system automatically determine the best number of GPUs to use.
+        peft_model_config: Optional[Dict]
+            - "lora_list": A List of PEFT (Parameter-Efficient Fine-Tuning) model and path.
+            - "image_lora_load_kwargs": A Dict of lora load parameters for image model
+            - "image_lora_fuse_kwargs": A Dict of lora fuse parameters for image model
         request_limits: Optional[int]
-            The number of request limits for this model， default is None.
+            The number of request limits for this model, default is None.
             ``request_limits=None`` means no limits for this model.
-        peft_model_path: Optional[str]
-            PEFT (Parameter-Efficient Fine-Tuning) model path.
-        image_lora_load_kwargs: Optional[Dict]
-            lora load parameters for image model
-        image_lora_fuse_kwargs: Optional[Dict]
-            lora fuse parameters for image model
         worker_ip: Optional[str]
             Specify the worker ip where the model is located in a distributed scenario.
         gpu_idx: Optional[Union[int, List[int]]]
@@ -854,9 +865,14 @@ class Client:
 
         url = f"{self.base_url}/v1/models"
 
+        # convert float to int or string since the RESTful API does not accept float.
+        if isinstance(model_size_in_billions, float):
+            model_size_in_billions = convert_float_to_int_or_str(model_size_in_billions)
+
         payload = {
             "model_uid": model_uid,
             "model_name": model_name,
+            "peft_model_config": peft_model_config,
             "model_type": model_type,
             "model_size_in_billions": model_size_in_billions,
             "model_format": model_format,
@@ -864,9 +880,6 @@ class Client:
             "replica": replica,
             "n_gpu": n_gpu,
             "request_limits": request_limits,
-            "peft_model_path": peft_model_path,
-            "image_lora_load_kwargs": image_lora_load_kwargs,
-            "image_lora_fuse_kwargs": image_lora_fuse_kwargs,
             "worker_ip": worker_ip,
             "gpu_idx": gpu_idx,
         }

xinference/core/supervisor.py

@@ -30,6 +30,7 @@ from ..constants import (
 )
 from ..core import ModelActor
 from ..core.status_guard import InstanceInfo, LaunchStatus
+from ..types import PeftModelConfig
 from .metrics import record_metrics
 from .resource import GPUStatus, ResourceStatus
 from .utils import (
@@ -135,6 +136,13 @@ class SupervisorActor(xo.StatelessActor):
             EventCollectorActor, address=self.address, uid=EventCollectorActor.uid()
         )
 
+        from ..model.audio import (
+            CustomAudioModelFamilyV1,
+            generate_audio_description,
+            get_audio_model_descriptions,
+            register_audio,
+            unregister_audio,
+        )
         from ..model.embedding import (
             CustomEmbeddingModelSpec,
             generate_embedding_description,
@@ -177,6 +185,12 @@ class SupervisorActor(xo.StatelessActor):
                 unregister_rerank,
                 generate_rerank_description,
             ),
+            "audio": (
+                CustomAudioModelFamilyV1,
+                register_audio,
+                unregister_audio,
+                generate_audio_description,
+            ),
         }
 
         # record model version
@@ -185,6 +199,7 @@ class SupervisorActor(xo.StatelessActor):
         model_version_infos.update(get_embedding_model_descriptions())
         model_version_infos.update(get_rerank_model_descriptions())
         model_version_infos.update(get_image_model_descriptions())
+        model_version_infos.update(get_audio_model_descriptions())
         await self._cache_tracker_ref.record_model_version(
             model_version_infos, self.address
         )
@@ -483,6 +498,7 @@ class SupervisorActor(xo.StatelessActor):
             return ret
         elif model_type == "audio":
             from ..model.audio import BUILTIN_AUDIO_MODELS
+            from ..model.audio.custom import get_user_defined_audios
 
             ret = []
             for model_name, family in BUILTIN_AUDIO_MODELS.items():
@@ -491,6 +507,16 @@ class SupervisorActor(xo.StatelessActor):
                 else:
                     ret.append({"model_name": model_name, "is_builtin": True})
 
+            for model_spec in get_user_defined_audios():
+                if detailed:
+                    ret.append(
+                        await self._to_audio_model_reg(model_spec, is_builtin=False)
+                    )
+                else:
+                    ret.append(
+                        {"model_name": model_spec.model_name, "is_builtin": False}
+                    )
+
             ret.sort(key=sort_helper)
             return ret
         elif model_type == "rerank":
@@ -548,8 +574,9 @@ class SupervisorActor(xo.StatelessActor):
             raise ValueError(f"Model {model_name} not found")
         elif model_type == "audio":
             from ..model.audio import BUILTIN_AUDIO_MODELS
+            from ..model.audio.custom import get_user_defined_audios
 
-            for f in BUILTIN_AUDIO_MODELS.values():
+            for f in list(BUILTIN_AUDIO_MODELS.values()) + get_user_defined_audios():
                 if f.model_name == model_name:
                     return f
             raise ValueError(f"Model {model_name} not found")
@@ -654,7 +681,7 @@ class SupervisorActor(xo.StatelessActor):
         self,
         model_uid: Optional[str],
         model_name: str,
-        model_size_in_billions: Optional[int],
+        model_size_in_billions: Optional[Union[int, str]],
         quantization: Optional[str],
         draft_model_name: str,
         draft_model_size_in_billions: Optional[int],
@@ -714,7 +741,7 @@ class SupervisorActor(xo.StatelessActor):
         self,
         model_uid: Optional[str],
         model_name: str,
-        model_size_in_billions: Optional[int],
+        model_size_in_billions: Optional[Union[int, str]],
         model_format: Optional[str],
         quantization: Optional[str],
         model_type: Optional[str],
@@ -723,9 +750,7 @@ class SupervisorActor(xo.StatelessActor):
         request_limits: Optional[int] = None,
         wait_ready: bool = True,
         model_version: Optional[str] = None,
-        peft_model_path: Optional[str] = None,
-        image_lora_load_kwargs: Optional[Dict] = None,
-        image_lora_fuse_kwargs: Optional[Dict] = None,
+        peft_model_config: Optional[PeftModelConfig] = None,
         worker_ip: Optional[str] = None,
         gpu_idx: Optional[Union[int, List[int]]] = None,
         **kwargs,
@@ -777,9 +802,7 @@ class SupervisorActor(xo.StatelessActor):
                 model_type=model_type,
                 n_gpu=n_gpu,
                 request_limits=request_limits,
-                peft_model_path=peft_model_path,
-                image_lora_load_kwargs=image_lora_load_kwargs,
-                image_lora_fuse_kwargs=image_lora_fuse_kwargs,
+                peft_model_config=peft_model_config,
                 gpu_idx=gpu_idx,
                 **kwargs,
             )

xinference/core/worker.py

@@ -36,6 +36,7 @@ from ..core import ModelActor
 from ..core.status_guard import LaunchStatus
 from ..device_utils import gpu_count
 from ..model.core import ModelDescription, create_model_instance
+from ..types import PeftModelConfig
 from .event import Event, EventCollectorActor, EventType
 from .metrics import launch_metrics_export_server, record_metrics
 from .resource import gather_node_info
@@ -195,6 +196,12 @@ class WorkerActor(xo.StatelessActor):
         logger.info("Purge cache directory: %s", XINFERENCE_CACHE_DIR)
         purge_dir(XINFERENCE_CACHE_DIR)
 
+        from ..model.audio import (
+            CustomAudioModelFamilyV1,
+            get_audio_model_descriptions,
+            register_audio,
+            unregister_audio,
+        )
         from ..model.embedding import (
             CustomEmbeddingModelSpec,
             get_embedding_model_descriptions,
@@ -223,6 +230,7 @@ class WorkerActor(xo.StatelessActor):
                 unregister_embedding,
             ),
             "rerank": (CustomRerankModelSpec, register_rerank, unregister_rerank),
+            "audio": (CustomAudioModelFamilyV1, register_audio, unregister_audio),
         }
 
         # record model version
@@ -231,6 +239,7 @@ class WorkerActor(xo.StatelessActor):
         model_version_infos.update(get_embedding_model_descriptions())
         model_version_infos.update(get_rerank_model_descriptions())
         model_version_infos.update(get_image_model_descriptions())
+        model_version_infos.update(get_audio_model_descriptions())
         await self._cache_tracker_ref.record_model_version(
             model_version_infos, self.address
         )
@@ -593,14 +602,12 @@ class WorkerActor(xo.StatelessActor):
         self,
         model_uid: str,
         model_name: str,
-        model_size_in_billions: Optional[int],
+        model_size_in_billions: Optional[Union[int, str]],
         model_format: Optional[str],
         quantization: Optional[str],
         model_type: str = "LLM",
         n_gpu: Optional[Union[int, str]] = "auto",
-        peft_model_path: Optional[str] = None,
-        image_lora_load_kwargs: Optional[Dict] = None,
-        image_lora_fuse_kwargs: Optional[Dict] = None,
+        peft_model_config: Optional[PeftModelConfig] = None,
         request_limits: Optional[int] = None,
         gpu_idx: Optional[Union[int, List[int]]] = None,
         **kwargs,
@@ -638,7 +645,7 @@ class WorkerActor(xo.StatelessActor):
             if isinstance(n_gpu, str) and n_gpu != "auto":
                 raise ValueError("Currently `n_gpu` only supports `auto`.")
 
-        if peft_model_path is not None:
+        if peft_model_config is not None:
             if model_type in ("embedding", "rerank"):
                 raise ValueError(
                     f"PEFT adaptors cannot be applied to embedding or rerank models."
@@ -669,9 +676,7 @@ class WorkerActor(xo.StatelessActor):
                 model_format,
                 model_size_in_billions,
                 quantization,
-                peft_model_path,
-                image_lora_load_kwargs,
-                image_lora_fuse_kwargs,
+                peft_model_config,
                 is_local_deployment,
                 **kwargs,
             )

xinference/deploy/cmdline.py

@@ -640,10 +640,11 @@ def list_model_registrations(
     help='The number of GPUs used by the model, default is "auto".',
 )
 @click.option(
-    "--peft-model-path",
-    default=None,
-    type=str,
-    help="PEFT model path.",
+    "--lora-modules",
+    "-lm",
+    multiple=True,
+    type=(str, str),
+    help="LoRA module configurations in the format name=path. Multiple modules can be specified.",
 )
 @click.option(
     "--image-lora-load-kwargs",
@@ -696,7 +697,7 @@ def model_launch(
     quantization: str,
     replica: int,
     n_gpu: str,
-    peft_model_path: Optional[str],
+    lora_modules: Optional[Tuple],
     image_lora_load_kwargs: Optional[Tuple],
     image_lora_fuse_kwargs: Optional[Tuple],
     worker_ip: Optional[str],
@@ -729,6 +730,18 @@ def model_launch(
         else None
     )
 
+    lora_list = (
+        [{"lora_name": k, "local_path": v} for k, v in dict(lora_modules).items()]
+        if lora_modules
+        else []
+    )
+
+    peft_model_config = {
+        "image_lora_load_kwargs": image_lora_load_params,
+        "image_lora_fuse_kwargs": image_lora_fuse_params,
+        "lora_list": lora_list,
+    }
+
     _gpu_idx: Optional[List[int]] = (
         None if gpu_idx is None else [int(idx) for idx in gpu_idx.split(",")]
     )
@@ -736,7 +749,9 @@ def model_launch(
     endpoint = get_endpoint(endpoint)
     model_size: Optional[Union[str, int]] = (
         size_in_billions
-        if size_in_billions is None or "_" in size_in_billions
+        if size_in_billions is None
+        or "_" in size_in_billions
+        or "." in size_in_billions
         else int(size_in_billions)
     )
     client = RESTfulClient(base_url=endpoint, api_key=api_key)
@@ -752,9 +767,7 @@ def model_launch(
         quantization=quantization,
         replica=replica,
         n_gpu=_n_gpu,
-        peft_model_path=peft_model_path,
-        image_lora_load_kwargs=image_lora_load_params,
-        image_lora_fuse_kwargs=image_lora_fuse_params,
+        peft_model_config=peft_model_config,
         worker_ip=worker_ip,
         gpu_idx=_gpu_idx,
         trust_remote_code=trust_remote_code,

xinference/model/audio/__init__.py

@@ -16,12 +16,51 @@ import codecs
 import json
 import os
 
-from .core import AudioModelFamilyV1, generate_audio_description, get_cache_status
+from .core import (
+    AUDIO_MODEL_DESCRIPTIONS,
+    MODEL_NAME_TO_REVISION,
+    AudioModelFamilyV1,
+    generate_audio_description,
+    get_audio_model_descriptions,
+    get_cache_status,
+)
+from .custom import (
+    CustomAudioModelFamilyV1,
+    get_user_defined_audios,
+    register_audio,
+    unregister_audio,
+)
 
 _model_spec_json = os.path.join(os.path.dirname(__file__), "model_spec.json")
 BUILTIN_AUDIO_MODELS = dict(
     (spec["model_name"], AudioModelFamilyV1(**spec))
     for spec in json.load(codecs.open(_model_spec_json, "r", encoding="utf-8"))
 )
+for model_name, model_spec in BUILTIN_AUDIO_MODELS.items():
+    MODEL_NAME_TO_REVISION[model_name].append(model_spec.model_revision)
+
+# register model description after recording model revision
+for model_spec_info in [BUILTIN_AUDIO_MODELS]:
+    for model_name, model_spec in model_spec_info.items():
+        if model_spec.model_name not in AUDIO_MODEL_DESCRIPTIONS:
+            AUDIO_MODEL_DESCRIPTIONS.update(generate_audio_description(model_spec))
+
+from ...constants import XINFERENCE_MODEL_DIR
+
+# if persist=True, load them when init
+user_defined_audio_dir = os.path.join(XINFERENCE_MODEL_DIR, "audio")
+if os.path.isdir(user_defined_audio_dir):
+    for f in os.listdir(user_defined_audio_dir):
+        with codecs.open(
+            os.path.join(user_defined_audio_dir, f), encoding="utf-8"
+        ) as fd:
+            user_defined_audio_family = CustomAudioModelFamilyV1.parse_obj(
+                json.load(fd)
+            )
+            register_audio(user_defined_audio_family, persist=False)
+
+# register model description
+for ud_audio in get_user_defined_audios():
+    AUDIO_MODEL_DESCRIPTIONS.update(generate_audio_description(ud_audio))
 
 del _model_spec_json

xinference/model/audio/core.py

@@ -16,9 +16,8 @@ import os
 from collections import defaultdict
 from typing import Dict, List, Optional, Tuple
 
-from ..._compat import BaseModel
 from ...constants import XINFERENCE_CACHE_DIR
-from ..core import ModelDescription
+from ..core import CacheableModelSpec, ModelDescription
 from ..utils import valid_model_revision
 from .whisper import WhisperModel
 
@@ -26,8 +25,19 @@ MAX_ATTEMPTS = 3
 
 logger = logging.getLogger(__name__)
 
+# Used for check whether the model is cached.
+# Init when registering all the builtin models.
+MODEL_NAME_TO_REVISION: Dict[str, List[str]] = defaultdict(list)
+AUDIO_MODEL_DESCRIPTIONS: Dict[str, List[Dict]] = defaultdict(list)
 
-class AudioModelFamilyV1(BaseModel):
+
+def get_audio_model_descriptions():
+    import copy
+
+    return copy.deepcopy(AUDIO_MODEL_DESCRIPTIONS)
+
+
+class AudioModelFamilyV1(CacheableModelSpec):
     model_family: str
     model_name: str
     model_id: str
@@ -77,63 +87,33 @@ def generate_audio_description(
     image_model: AudioModelFamilyV1,
 ) -> Dict[str, List[Dict]]:
     res = defaultdict(list)
-    res[image_model.model_name].extend(
-        AudioModelDescription(None, None, image_model).to_dict()
+    res[image_model.model_name].append(
+        AudioModelDescription(None, None, image_model).to_version_info()
     )
     return res
 
 
-def match_model(model_name: str) -> AudioModelFamilyV1:
+def match_audio(model_name: str) -> AudioModelFamilyV1:
     from . import BUILTIN_AUDIO_MODELS
+    from .custom import get_user_defined_audios
+
+    for model_spec in get_user_defined_audios():
+        if model_spec.model_name == model_name:
+            return model_spec
 
     if model_name in BUILTIN_AUDIO_MODELS:
         return BUILTIN_AUDIO_MODELS[model_name]
     else:
         raise ValueError(
-            f"Image model {model_name} not found, available"
+            f"Audio model {model_name} not found, available"
             f"model list: {BUILTIN_AUDIO_MODELS.keys()}"
         )
 
 
 def cache(model_spec: AudioModelFamilyV1):
-    # TODO: cache from uri
-    import huggingface_hub
-
-    cache_dir = get_cache_dir(model_spec)
-    if not os.path.exists(cache_dir):
-        os.makedirs(cache_dir, exist_ok=True)
-
-    meta_path = os.path.join(cache_dir, "__valid_download")
-    if valid_model_revision(meta_path, model_spec.model_revision):
-        return cache_dir
-
-    for current_attempt in range(1, MAX_ATTEMPTS + 1):
-        try:
-            huggingface_hub.snapshot_download(
-                model_spec.model_id,
-                revision=model_spec.model_revision,
-                local_dir=cache_dir,
-                local_dir_use_symlinks=True,
-                resume_download=True,
-            )
-            break
-        except huggingface_hub.utils.LocalEntryNotFoundError:
-            remaining_attempts = MAX_ATTEMPTS - current_attempt
-            logger.warning(
-                f"Attempt {current_attempt} failed. Remaining attempts: {remaining_attempts}"
-            )
-    else:
-        raise RuntimeError(
-            f"Failed to download model '{model_spec.model_name}' after {MAX_ATTEMPTS} attempts"
-        )
-
-    with open(meta_path, "w") as f:
-        import json
-
-        desc = AudioModelDescription(None, None, model_spec)
-        json.dump(desc.to_dict(), f)
+    from ..utils import cache
 
-    return cache_dir
+    return cache(model_spec, AudioModelDescription)
 
 
 def get_cache_dir(model_spec: AudioModelFamilyV1):
@@ -151,7 +131,7 @@ def get_cache_status(
 def create_audio_model_instance(
     subpool_addr: str, devices: List[str], model_uid: str, model_name: str, **kwargs
 ) -> Tuple[WhisperModel, AudioModelDescription]:
-    model_spec = match_model(model_name)
+    model_spec = match_audio(model_name)
     model_path = cache(model_spec)
     model = WhisperModel(model_uid, model_path, model_spec, **kwargs)
     model_description = AudioModelDescription(