xinference 0.11.3__py3-none-any.whl → 0.12.1__py3-none-any.whl

xinference/core/supervisor.py

@@ -982,34 +982,59 @@ class SupervisorActor(xo.StatelessActor):
         )
 
     @log_async(logger=logger)
-    async def list_cached_models(self) -> List[Dict[str, Any]]:
+    async def list_cached_models(
+        self, model_name: Optional[str] = None, worker_ip: Optional[str] = None
+    ) -> List[Dict[str, Any]]:
+        target_ip_worker_ref = (
+            self._get_worker_ref_by_ip(worker_ip) if worker_ip is not None else None
+        )
+        if (
+            worker_ip is not None
+            and not self.is_local_deployment()
+            and target_ip_worker_ref is None
+        ):
+            raise ValueError(f"Worker ip address {worker_ip} is not in the cluster.")
+
+        # search assigned worker and return
+        if target_ip_worker_ref:
+            cached_models = await target_ip_worker_ref.list_cached_models(model_name)
+            cached_models = sorted(cached_models, key=lambda x: x["model_name"])
+            return cached_models
+
+        # search all worker
         cached_models = []
         for worker in self._worker_address_to_worker.values():
-            ret = await worker.list_cached_models()
-            for model_version in ret:
-                model_name = model_version.get("model_name", None)
-                model_format = model_version.get("model_format", None)
-                model_size_in_billions = model_version.get(
-                    "model_size_in_billions", None
-                )
-                quantizations = model_version.get("quantization", None)
-                actor_ip_address = model_version.get("actor_ip_address", None)
-                path = model_version.get("path", None)
-                real_path = model_version.get("real_path", None)
-
-                cache_entry = {
-                    "model_name": model_name,
-                    "model_format": model_format,
-                    "model_size_in_billions": model_size_in_billions,
-                    "quantizations": quantizations,
-                    "path": path,
-                    "Actor IP Address": actor_ip_address,
-                    "real_path": real_path,
-                }
-
-                cached_models.append(cache_entry)
+            res = await worker.list_cached_models(model_name)
+            cached_models.extend(res)
+        cached_models = sorted(cached_models, key=lambda x: x["model_name"])
         return cached_models
 
+    @log_async(logger=logger)
+    async def abort_request(self, model_uid: str, request_id: str) -> Dict:
+        from .scheduler import AbortRequestMessage
+
+        res = {"msg": AbortRequestMessage.NO_OP.name}
+        replica_info = self._model_uid_to_replica_info.get(model_uid, None)
+        if not replica_info:
+            return res
+        replica_cnt = replica_info.replica
+
+        # Query all replicas
+        for rep_mid in iter_replica_model_uid(model_uid, replica_cnt):
+            worker_ref = self._replica_model_uid_to_worker.get(rep_mid, None)
+            if worker_ref is None:
+                continue
+            model_ref = await worker_ref.get_model(model_uid=rep_mid)
+            result_info = await model_ref.abort_request(request_id)
+            res["msg"] = result_info
+            if result_info == AbortRequestMessage.DONE.name:
+                break
+            elif result_info == AbortRequestMessage.NOT_FOUND.name:
+                logger.debug(f"Request id: {request_id} not found for model {rep_mid}")
+            else:
+                logger.debug(f"No-op for model {rep_mid}")
+        return res
+
     @log_async(logger=logger)
     async def add_worker(self, worker_address: str):
         from .worker import WorkerActor
@@ -1057,6 +1082,56 @@ class SupervisorActor(xo.StatelessActor):
             worker_status.update_time = time.time()
             worker_status.status = status
 
+    async def list_deletable_models(
+        self, model_version: str, worker_ip: Optional[str] = None
+    ) -> List[str]:
+        target_ip_worker_ref = (
+            self._get_worker_ref_by_ip(worker_ip) if worker_ip is not None else None
+        )
+        if (
+            worker_ip is not None
+            and not self.is_local_deployment()
+            and target_ip_worker_ref is None
+        ):
+            raise ValueError(f"Worker ip address {worker_ip} is not in the cluster.")
+
+        ret = []
+        if target_ip_worker_ref:
+            ret = await target_ip_worker_ref.list_deletable_models(
+                model_version=model_version,
+            )
+            return ret
+
+        for worker in self._worker_address_to_worker.values():
+            path = await worker.list_deletable_models(model_version=model_version)
+            ret.extend(path)
+        return ret
+
+    async def confirm_and_remove_model(
+        self, model_version: str, worker_ip: Optional[str] = None
+    ) -> bool:
+        target_ip_worker_ref = (
+            self._get_worker_ref_by_ip(worker_ip) if worker_ip is not None else None
+        )
+        if (
+            worker_ip is not None
+            and not self.is_local_deployment()
+            and target_ip_worker_ref is None
+        ):
+            raise ValueError(f"Worker ip address {worker_ip} is not in the cluster.")
+
+        if target_ip_worker_ref:
+            ret = await target_ip_worker_ref.confirm_and_remove_model(
+                model_version=model_version,
+            )
+            return ret
+        ret = True
+        for worker in self._worker_address_to_worker.values():
+            ret = ret and await worker.confirm_and_remove_model(
+                model_version=model_version,
+            )
+        return ret
+
     @staticmethod
     def record_metrics(name, op, kwargs):
         record_metrics(name, op, kwargs)

xinference/core/worker.py

@@ -16,6 +16,7 @@ import asyncio
 import os
 import platform
 import queue
+import shutil
 import signal
 import threading
 import time
@@ -786,8 +787,73 @@ class WorkerActor(xo.StatelessActor):
             except asyncio.CancelledError:  # pragma: no cover
                 break
 
-    async def list_cached_models(self) -> List[Dict[Any, Any]]:
-        return self._cache_tracker_ref.list_cached_models()
+    async def list_cached_models(
+        self, model_name: Optional[str] = None
+    ) -> List[Dict[Any, Any]]:
+        lists = await self._cache_tracker_ref.list_cached_models(
+            self.address, model_name
+        )
+        cached_models = []
+        for list in lists:
+            cached_model = {
+                "model_name": list.get("model_name"),
+                "model_size_in_billions": list.get("model_size_in_billions"),
+                "model_format": list.get("model_format"),
+                "quantization": list.get("quantization"),
+                "model_version": list.get("model_version"),
+            }
+            path = list.get("model_file_location")
+            cached_model["path"] = path
+            # parsing soft links
+            if os.path.isdir(path):
+                files = os.listdir(path)
+                # dir has files
+                if files:
+                    resolved_file = os.path.realpath(os.path.join(path, files[0]))
+                    if resolved_file:
+                        cached_model["real_path"] = os.path.dirname(resolved_file)
+            else:
+                cached_model["real_path"] = os.path.realpath(path)
+            cached_model["actor_ip_address"] = self.address
+            cached_models.append(cached_model)
+        return cached_models
+
+    async def list_deletable_models(self, model_version: str) -> List[str]:
+        paths = set()
+        path = await self._cache_tracker_ref.list_deletable_models(
+            model_version, self.address
+        )
+        if os.path.isfile(path):
+            path = os.path.dirname(path)
+
+        if os.path.isdir(path):
+            files = os.listdir(path)
+            paths.update([os.path.join(path, file) for file in files])
+            # search real path
+            if paths:
+                paths.update([os.path.realpath(path) for path in paths])
+
+        return list(paths)
+
+    async def confirm_and_remove_model(self, model_version: str) -> bool:
+        paths = await self.list_deletable_models(model_version)
+        for path in paths:
+            try:
+                if os.path.islink(path):
+                    os.unlink(path)
+                elif os.path.isfile(path):
+                    os.remove(path)
+                elif os.path.isdir(path):
+                    shutil.rmtree(path)
+                else:
+                    logger.debug(f"{path} is not a valid path.")
+            except Exception as e:
+                logger.error(f"Fail to delete {path} with error:{e}.")
+                return False
+        await self._cache_tracker_ref.confirm_and_remove_model(
+            model_version, self.address
+        )
+        return True
 
     @staticmethod
     def record_metrics(name, op, kwargs):

xinference/deploy/cmdline.py

@@ -577,6 +577,18 @@ def list_model_registrations(
     type=str,
     help="Xinference endpoint.",
 )
+@click.option(
+    "--model_name",
+    "-n",
+    type=str,
+    help="Provide the name of the models to be removed.",
+)
+@click.option(
+    "--worker-ip",
+    default=None,
+    type=str,
+    help="Specify which worker this model runs on by ip, for distributed situation.",
+)
 @click.option(
     "--api-key",
     "-ak",
@@ -587,6 +599,8 @@ def list_model_registrations(
 def list_cached_models(
     endpoint: Optional[str],
     api_key: Optional[str],
+    model_name: Optional[str],
+    worker_ip: Optional[str],
 ):
     from tabulate import tabulate
 
@@ -595,10 +609,13 @@ def list_cached_models(
     if api_key is None:
         client._set_token(get_stored_token(endpoint, client))
 
-    cached_models = client.list_cached_models()
+    cached_models = client.list_cached_models(model_name, worker_ip)
+    if not cached_models:
+        print("There are no cache files.")
+        return
+    headers = list(cached_models[0].keys())
 
     print("cached_model: ")
-    headers = list(cached_models[0].keys())
     table_data = []
     for model in cached_models:
         row_data = [
@@ -608,6 +625,73 @@ def list_cached_models(
     print(tabulate(table_data, headers=headers, tablefmt="pretty"))
 
 
+@cli.command("remove-cache", help="Remove selected cached models in Xinference.")
+@click.option(
+    "--endpoint",
+    "-e",
+    type=str,
+    help="Xinference endpoint.",
+)
+@click.option(
+    "--model_version",
+    "-n",
+    type=str,
+    help="Provide the version of the models to be removed.",
+)
+@click.option(
+    "--worker-ip",
+    default=None,
+    type=str,
+    help="Specify which worker this model runs on by ip, for distributed situation.",
+)
+@click.option(
+    "--api-key",
+    "-ak",
+    default=None,
+    type=str,
+    help="Api-Key for access xinference api with authorization.",
+)
+@click.option("--check", is_flag=True, help="Confirm the deletion of the cache.")
+def remove_cache(
+    endpoint: Optional[str],
+    model_version: str,
+    api_key: Optional[str],
+    check: bool,
+    worker_ip: Optional[str] = None,
+):
+    endpoint = get_endpoint(endpoint)
+    client = RESTfulClient(base_url=endpoint, api_key=api_key)
+    if api_key is None:
+        client._set_token(get_stored_token(endpoint, client))
+
+    if not check:
+        response = client.list_deletable_models(
+            model_version=model_version, worker_ip=worker_ip
+        )
+        paths: List[str] = response.get("paths", [])
+        if not paths:
+            click.echo(f"There is no model version named {model_version}.")
+            return
+        click.echo(f"Model {model_version} cache directory to be deleted:")
+        for path in response.get("paths", []):
+            click.echo(f"{path}")
+
+        if click.confirm("Do you want to proceed with the deletion?", abort=True):
+            check = True
+    try:
+        result = client.confirm_and_remove_model(
+            model_version=model_version, worker_ip=worker_ip
+        )
+        if result:
+            click.echo(f"Cache directory {model_version} has been deleted.")
+        else:
+            click.echo(
+                f"Cache directory {model_version} fail to be deleted. Please check the log."
+            )
+    except Exception as e:
+        click.echo(f"An error occurred while deleting the cache: {e}")
+
+
 @cli.command(
     "launch",
     help="Launch a model with the Xinference framework with the given parameters.",

xinference/deploy/test/test_cmdline.py

@@ -26,6 +26,7 @@ from ..cmdline import (
     model_list,
     model_terminate,
     register_model,
+    remove_cache,
     unregister_model,
 )
 
@@ -287,18 +288,26 @@ def test_list_cached_models(setup):
 
     result = runner.invoke(
         list_cached_models,
-        [
-            "--endpoint",
-            endpoint,
-        ],
+        ["--endpoint", endpoint, "--model_name", "orca"],
     )
-    assert result.exit_code == 0
-    assert "cached_model: " in result.stdout
-
-    # check if the output is in tabular format
     assert "model_name" in result.stdout
     assert "model_format" in result.stdout
     assert "model_size_in_billions" in result.stdout
-    assert "quantizations" in result.stdout
+    assert "quantization" in result.stdout
+    assert "model_version" in result.stdout
     assert "path" in result.stdout
-    assert "Actor IP Address" in result.stdout
+    assert "actor_ip_address" in result.stdout
+
+
+def test_remove_cache(setup):
+    endpoint, _ = setup
+    runner = CliRunner()
+
+    result = runner.invoke(
+        remove_cache,
+        ["--endpoint", endpoint, "--model_version", "orca"],
+        input="y\n",
+    )
+
+    assert result.exit_code == 0
+    assert "Cache directory orca has been deleted."

xinference/isolation.py

@@ -19,13 +19,19 @@ from typing import Any, Coroutine
 
 class Isolation:
     # TODO: better move isolation to xoscar.
-    def __init__(self, loop: asyncio.AbstractEventLoop, threaded: bool = True):
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop,
+        threaded: bool = True,
+        daemon: bool = True,
+    ):
         self._loop = loop
         self._threaded = threaded
 
         self._stopped = None
         self._thread = None
         self._thread_ident = None
+        self._daemon = daemon
 
     def _run(self):
         asyncio.set_event_loop(self._loop)
@@ -35,7 +41,8 @@ class Isolation:
     def start(self):
         if self._threaded:
             self._thread = thread = threading.Thread(target=self._run)
-            thread.daemon = True
+            if self._daemon:
+                thread.daemon = True
             thread.start()
             self._thread_ident = thread.ident
 

xinference/model/audio/__init__.py

@@ -32,6 +32,9 @@ from .custom import (
 )
 
 _model_spec_json = os.path.join(os.path.dirname(__file__), "model_spec.json")
+_model_spec_modelscope_json = os.path.join(
+    os.path.dirname(__file__), "model_spec_modelscope.json"
+)
 BUILTIN_AUDIO_MODELS = dict(
     (spec["model_name"], AudioModelFamilyV1(**spec))
     for spec in json.load(codecs.open(_model_spec_json, "r", encoding="utf-8"))
@@ -39,8 +42,17 @@ BUILTIN_AUDIO_MODELS = dict(
 for model_name, model_spec in BUILTIN_AUDIO_MODELS.items():
     MODEL_NAME_TO_REVISION[model_name].append(model_spec.model_revision)
 
+MODELSCOPE_AUDIO_MODELS = dict(
+    (spec["model_name"], AudioModelFamilyV1(**spec))
+    for spec in json.load(
+        codecs.open(_model_spec_modelscope_json, "r", encoding="utf-8")
+    )
+)
+for model_name, model_spec in MODELSCOPE_AUDIO_MODELS.items():
+    MODEL_NAME_TO_REVISION[model_name].append(model_spec.model_revision)
+
 # register model description after recording model revision
-for model_spec_info in [BUILTIN_AUDIO_MODELS]:
+for model_spec_info in [BUILTIN_AUDIO_MODELS, MODELSCOPE_AUDIO_MODELS]:
     for model_name, model_spec in model_spec_info.items():
         if model_spec.model_name not in AUDIO_MODEL_DESCRIPTIONS:
             AUDIO_MODEL_DESCRIPTIONS.update(generate_audio_description(model_spec))
@@ -64,3 +76,4 @@ for ud_audio in get_user_defined_audios():
     AUDIO_MODEL_DESCRIPTIONS.update(generate_audio_description(ud_audio))
 
 del _model_spec_json
+del _model_spec_modelscope_json

xinference/model/audio/chattts.py

@@ -0,0 +1,84 @@
+# Copyright 2022-2023 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+from io import BytesIO
+from typing import TYPE_CHECKING, Optional
+
+if TYPE_CHECKING:
+    from .core import AudioModelFamilyV1
+
+logger = logging.getLogger(__name__)
+
+
+class ChatTTSModel:
+    def __init__(
+        self,
+        model_uid: str,
+        model_path: str,
+        model_spec: "AudioModelFamilyV1",
+        device: Optional[str] = None,
+        **kwargs,
+    ):
+        self._model_uid = model_uid
+        self._model_path = model_path
+        self._model_spec = model_spec
+        self._device = device
+        self._model = None
+        self._kwargs = kwargs
+
+    def load(self):
+        import torch
+
+        from xinference.thirdparty import ChatTTS
+
+        torch._dynamo.config.cache_size_limit = 64
+        torch._dynamo.config.suppress_errors = True
+        torch.set_float32_matmul_precision("high")
+        self._model = ChatTTS.Chat()
+        self._model.load_models(
+            source="local", local_path=self._model_path, compile=True
+        )
+
+    def speech(
+        self, input: str, voice: str, response_format: str = "mp3", speed: float = 1.0
+    ):
+        import numpy as np
+        import torch
+        import torchaudio
+        import xxhash
+
+        seed = xxhash.xxh32_intdigest(voice)
+
+        torch.manual_seed(seed)
+        np.random.seed(seed)
+        torch.cuda.manual_seed(seed)
+        torch.backends.cudnn.deterministic = True
+        torch.backends.cudnn.benchmark = False
+
+        assert self._model is not None
+        rnd_spk_emb = self._model.sample_random_speaker()
+
+        default = 5
+        infer_speed = int(default * speed)
+        params_infer_code = {"spk_emb": rnd_spk_emb, "prompt": f"[speed_{infer_speed}]"}
+
+        assert self._model is not None
+        wavs = self._model.infer([input], params_infer_code=params_infer_code)
+
+        # Save the generated audio
+        with BytesIO() as out:
+            torchaudio.save(
+                out, torch.from_numpy(wavs[0]), 24000, format=response_format
+            )
+            return out.getvalue()

xinference/model/audio/core.py

@@ -14,11 +14,12 @@
 import logging
 import os
 from collections import defaultdict
-from typing import Dict, List, Optional, Tuple
+from typing import Dict, List, Optional, Tuple, Union
 
 from ...constants import XINFERENCE_CACHE_DIR
 from ..core import CacheableModelSpec, ModelDescription
 from ..utils import valid_model_revision
+from .chattts import ChatTTSModel
 from .whisper import WhisperModel
 
 MAX_ATTEMPTS = 3
@@ -94,13 +95,24 @@ def generate_audio_description(
 
 
 def match_audio(model_name: str) -> AudioModelFamilyV1:
-    from . import BUILTIN_AUDIO_MODELS
+    from ..utils import download_from_modelscope
+    from . import BUILTIN_AUDIO_MODELS, MODELSCOPE_AUDIO_MODELS
     from .custom import get_user_defined_audios
 
     for model_spec in get_user_defined_audios():
         if model_spec.model_name == model_name:
             return model_spec
 
+    if download_from_modelscope():
+        if model_name in MODELSCOPE_AUDIO_MODELS:
+            logger.debug(f"Audio model {model_name} found in ModelScope.")
+            return MODELSCOPE_AUDIO_MODELS[model_name]
+        else:
+            logger.debug(
+                f"Audio model {model_name} not found in ModelScope, "
+                f"now try to load it via builtin way."
+            )
+
     if model_name in BUILTIN_AUDIO_MODELS:
         return BUILTIN_AUDIO_MODELS[model_name]
     else:
@@ -130,10 +142,16 @@ def get_cache_status(
 
 def create_audio_model_instance(
     subpool_addr: str, devices: List[str], model_uid: str, model_name: str, **kwargs
-) -> Tuple[WhisperModel, AudioModelDescription]:
+) -> Tuple[Union[WhisperModel, ChatTTSModel], AudioModelDescription]:
     model_spec = match_audio(model_name)
     model_path = cache(model_spec)
-    model = WhisperModel(model_uid, model_path, model_spec, **kwargs)
+    model: Union[WhisperModel, ChatTTSModel]
+    if model_spec.model_family == "whisper":
+        model = WhisperModel(model_uid, model_path, model_spec, **kwargs)
+    elif model_spec.model_family == "ChatTTS":
+        model = ChatTTSModel(model_uid, model_path, model_spec, **kwargs)
+    else:
+        raise Exception(f"Unsupported audio model family: {model_spec.model_family}")
     model_description = AudioModelDescription(
         subpool_addr, devices, model_spec, model_path=model_path
     )

xinference/model/audio/custom.py

@@ -83,15 +83,17 @@ def get_user_defined_audios() -> List[CustomAudioModelFamilyV1]:
 def register_audio(model_spec: CustomAudioModelFamilyV1, persist: bool):
     from ...constants import XINFERENCE_MODEL_DIR
     from ..utils import is_valid_model_name, is_valid_model_uri
-    from . import BUILTIN_AUDIO_MODELS
+    from . import BUILTIN_AUDIO_MODELS, MODELSCOPE_AUDIO_MODELS
 
     if not is_valid_model_name(model_spec.model_name):
         raise ValueError(f"Invalid model name {model_spec.model_name}.")
 
     with UD_AUDIO_LOCK:
-        for model_name in list(BUILTIN_AUDIO_MODELS.keys()) + [
-            spec.model_name for spec in UD_AUDIOS
-        ]:
+        for model_name in (
+            list(BUILTIN_AUDIO_MODELS.keys())
+            + list(MODELSCOPE_AUDIO_MODELS.keys())
+            + [spec.model_name for spec in UD_AUDIOS]
+        ):
             if model_spec.model_name == model_name:
                 raise ValueError(
                     f"Model name conflicts with existing model {model_spec.model_name}"