xinference 0.12.0__py3-none-any.whl → 0.12.1__py3-none-any.whl

xinference/core/worker.py

@@ -16,6 +16,7 @@ import asyncio
 import os
 import platform
 import queue
+import shutil
 import signal
 import threading
 import time
@@ -786,8 +787,73 @@ class WorkerActor(xo.StatelessActor):
             except asyncio.CancelledError:  # pragma: no cover
                 break
 
-    async def list_cached_models(self) -> List[Dict[Any, Any]]:
-        return self._cache_tracker_ref.list_cached_models()
+    async def list_cached_models(
+        self, model_name: Optional[str] = None
+    ) -> List[Dict[Any, Any]]:
+        lists = await self._cache_tracker_ref.list_cached_models(
+            self.address, model_name
+        )
+        cached_models = []
+        for list in lists:
+            cached_model = {
+                "model_name": list.get("model_name"),
+                "model_size_in_billions": list.get("model_size_in_billions"),
+                "model_format": list.get("model_format"),
+                "quantization": list.get("quantization"),
+                "model_version": list.get("model_version"),
+            }
+            path = list.get("model_file_location")
+            cached_model["path"] = path
+            # parsing soft links
+            if os.path.isdir(path):
+                files = os.listdir(path)
+                # dir has files
+                if files:
+                    resolved_file = os.path.realpath(os.path.join(path, files[0]))
+                    if resolved_file:
+                        cached_model["real_path"] = os.path.dirname(resolved_file)
+            else:
+                cached_model["real_path"] = os.path.realpath(path)
+            cached_model["actor_ip_address"] = self.address
+            cached_models.append(cached_model)
+        return cached_models
+
+    async def list_deletable_models(self, model_version: str) -> List[str]:
+        paths = set()
+        path = await self._cache_tracker_ref.list_deletable_models(
+            model_version, self.address
+        )
+        if os.path.isfile(path):
+            path = os.path.dirname(path)
+
+        if os.path.isdir(path):
+            files = os.listdir(path)
+            paths.update([os.path.join(path, file) for file in files])
+            # search real path
+            if paths:
+                paths.update([os.path.realpath(path) for path in paths])
+
+        return list(paths)
+
+    async def confirm_and_remove_model(self, model_version: str) -> bool:
+        paths = await self.list_deletable_models(model_version)
+        for path in paths:
+            try:
+                if os.path.islink(path):
+                    os.unlink(path)
+                elif os.path.isfile(path):
+                    os.remove(path)
+                elif os.path.isdir(path):
+                    shutil.rmtree(path)
+                else:
+                    logger.debug(f"{path} is not a valid path.")
+            except Exception as e:
+                logger.error(f"Fail to delete {path} with error:{e}.")
+                return False
+        await self._cache_tracker_ref.confirm_and_remove_model(
+            model_version, self.address
+        )
+        return True
 
     @staticmethod
     def record_metrics(name, op, kwargs):

xinference/deploy/cmdline.py

@@ -577,6 +577,18 @@ def list_model_registrations(
     type=str,
     help="Xinference endpoint.",
 )
+@click.option(
+    "--model_name",
+    "-n",
+    type=str,
+    help="Provide the name of the models to be removed.",
+)
+@click.option(
+    "--worker-ip",
+    default=None,
+    type=str,
+    help="Specify which worker this model runs on by ip, for distributed situation.",
+)
 @click.option(
     "--api-key",
     "-ak",
@@ -587,6 +599,8 @@ def list_model_registrations(
 def list_cached_models(
     endpoint: Optional[str],
     api_key: Optional[str],
+    model_name: Optional[str],
+    worker_ip: Optional[str],
 ):
     from tabulate import tabulate
 
@@ -595,10 +609,13 @@ def list_cached_models(
     if api_key is None:
         client._set_token(get_stored_token(endpoint, client))
 
-    cached_models = client.list_cached_models()
+    cached_models = client.list_cached_models(model_name, worker_ip)
+    if not cached_models:
+        print("There are no cache files.")
+        return
+    headers = list(cached_models[0].keys())
 
     print("cached_model: ")
-    headers = list(cached_models[0].keys())
     table_data = []
     for model in cached_models:
         row_data = [
@@ -608,6 +625,73 @@ def list_cached_models(
     print(tabulate(table_data, headers=headers, tablefmt="pretty"))
 
 
+@cli.command("remove-cache", help="Remove selected cached models in Xinference.")
+@click.option(
+    "--endpoint",
+    "-e",
+    type=str,
+    help="Xinference endpoint.",
+)
+@click.option(
+    "--model_version",
+    "-n",
+    type=str,
+    help="Provide the version of the models to be removed.",
+)
+@click.option(
+    "--worker-ip",
+    default=None,
+    type=str,
+    help="Specify which worker this model runs on by ip, for distributed situation.",
+)
+@click.option(
+    "--api-key",
+    "-ak",
+    default=None,
+    type=str,
+    help="Api-Key for access xinference api with authorization.",
+)
+@click.option("--check", is_flag=True, help="Confirm the deletion of the cache.")
+def remove_cache(
+    endpoint: Optional[str],
+    model_version: str,
+    api_key: Optional[str],
+    check: bool,
+    worker_ip: Optional[str] = None,
+):
+    endpoint = get_endpoint(endpoint)
+    client = RESTfulClient(base_url=endpoint, api_key=api_key)
+    if api_key is None:
+        client._set_token(get_stored_token(endpoint, client))
+
+    if not check:
+        response = client.list_deletable_models(
+            model_version=model_version, worker_ip=worker_ip
+        )
+        paths: List[str] = response.get("paths", [])
+        if not paths:
+            click.echo(f"There is no model version named {model_version}.")
+            return
+        click.echo(f"Model {model_version} cache directory to be deleted:")
+        for path in response.get("paths", []):
+            click.echo(f"{path}")
+
+        if click.confirm("Do you want to proceed with the deletion?", abort=True):
+            check = True
+    try:
+        result = client.confirm_and_remove_model(
+            model_version=model_version, worker_ip=worker_ip
+        )
+        if result:
+            click.echo(f"Cache directory {model_version} has been deleted.")
+        else:
+            click.echo(
+                f"Cache directory {model_version} fail to be deleted. Please check the log."
+            )
+    except Exception as e:
+        click.echo(f"An error occurred while deleting the cache: {e}")
+
+
 @cli.command(
     "launch",
     help="Launch a model with the Xinference framework with the given parameters.",

xinference/deploy/test/test_cmdline.py

@@ -26,6 +26,7 @@ from ..cmdline import (
     model_list,
     model_terminate,
     register_model,
+    remove_cache,
     unregister_model,
 )
 
@@ -287,18 +288,26 @@ def test_list_cached_models(setup):
 
     result = runner.invoke(
         list_cached_models,
-        [
-            "--endpoint",
-            endpoint,
-        ],
+        ["--endpoint", endpoint, "--model_name", "orca"],
     )
-    assert result.exit_code == 0
-    assert "cached_model: " in result.stdout
-
-    # check if the output is in tabular format
     assert "model_name" in result.stdout
     assert "model_format" in result.stdout
     assert "model_size_in_billions" in result.stdout
-    assert "quantizations" in result.stdout
+    assert "quantization" in result.stdout
+    assert "model_version" in result.stdout
     assert "path" in result.stdout
-    assert "Actor IP Address" in result.stdout
+    assert "actor_ip_address" in result.stdout
+
+
+def test_remove_cache(setup):
+    endpoint, _ = setup
+    runner = CliRunner()
+
+    result = runner.invoke(
+        remove_cache,
+        ["--endpoint", endpoint, "--model_version", "orca"],
+        input="y\n",
+    )
+
+    assert result.exit_code == 0
+    assert "Cache directory orca has been deleted."

xinference/model/audio/__init__.py

@@ -32,6 +32,9 @@ from .custom import (
 )
 
 _model_spec_json = os.path.join(os.path.dirname(__file__), "model_spec.json")
+_model_spec_modelscope_json = os.path.join(
+    os.path.dirname(__file__), "model_spec_modelscope.json"
+)
 BUILTIN_AUDIO_MODELS = dict(
     (spec["model_name"], AudioModelFamilyV1(**spec))
     for spec in json.load(codecs.open(_model_spec_json, "r", encoding="utf-8"))
@@ -39,8 +42,17 @@ BUILTIN_AUDIO_MODELS = dict(
 for model_name, model_spec in BUILTIN_AUDIO_MODELS.items():
     MODEL_NAME_TO_REVISION[model_name].append(model_spec.model_revision)
 
+MODELSCOPE_AUDIO_MODELS = dict(
+    (spec["model_name"], AudioModelFamilyV1(**spec))
+    for spec in json.load(
+        codecs.open(_model_spec_modelscope_json, "r", encoding="utf-8")
+    )
+)
+for model_name, model_spec in MODELSCOPE_AUDIO_MODELS.items():
+    MODEL_NAME_TO_REVISION[model_name].append(model_spec.model_revision)
+
 # register model description after recording model revision
-for model_spec_info in [BUILTIN_AUDIO_MODELS]:
+for model_spec_info in [BUILTIN_AUDIO_MODELS, MODELSCOPE_AUDIO_MODELS]:
     for model_name, model_spec in model_spec_info.items():
         if model_spec.model_name not in AUDIO_MODEL_DESCRIPTIONS:
             AUDIO_MODEL_DESCRIPTIONS.update(generate_audio_description(model_spec))
@@ -64,3 +76,4 @@ for ud_audio in get_user_defined_audios():
     AUDIO_MODEL_DESCRIPTIONS.update(generate_audio_description(ud_audio))
 
 del _model_spec_json
+del _model_spec_modelscope_json

xinference/model/audio/core.py

@@ -95,13 +95,24 @@ def generate_audio_description(
 
 
 def match_audio(model_name: str) -> AudioModelFamilyV1:
-    from . import BUILTIN_AUDIO_MODELS
+    from ..utils import download_from_modelscope
+    from . import BUILTIN_AUDIO_MODELS, MODELSCOPE_AUDIO_MODELS
     from .custom import get_user_defined_audios
 
     for model_spec in get_user_defined_audios():
         if model_spec.model_name == model_name:
             return model_spec
 
+    if download_from_modelscope():
+        if model_name in MODELSCOPE_AUDIO_MODELS:
+            logger.debug(f"Audio model {model_name} found in ModelScope.")
+            return MODELSCOPE_AUDIO_MODELS[model_name]
+        else:
+            logger.debug(
+                f"Audio model {model_name} not found in ModelScope, "
+                f"now try to load it via builtin way."
+            )
+
     if model_name in BUILTIN_AUDIO_MODELS:
         return BUILTIN_AUDIO_MODELS[model_name]
     else:

xinference/model/audio/custom.py

@@ -83,15 +83,17 @@ def get_user_defined_audios() -> List[CustomAudioModelFamilyV1]:
 def register_audio(model_spec: CustomAudioModelFamilyV1, persist: bool):
     from ...constants import XINFERENCE_MODEL_DIR
     from ..utils import is_valid_model_name, is_valid_model_uri
-    from . import BUILTIN_AUDIO_MODELS
+    from . import BUILTIN_AUDIO_MODELS, MODELSCOPE_AUDIO_MODELS
 
     if not is_valid_model_name(model_spec.model_name):
         raise ValueError(f"Invalid model name {model_spec.model_name}.")
 
     with UD_AUDIO_LOCK:
-        for model_name in list(BUILTIN_AUDIO_MODELS.keys()) + [
-            spec.model_name for spec in UD_AUDIOS
-        ]:
+        for model_name in (
+            list(BUILTIN_AUDIO_MODELS.keys())
+            + list(MODELSCOPE_AUDIO_MODELS.keys())
+            + [spec.model_name for spec in UD_AUDIOS]
+        ):
             if model_spec.model_name == model_name:
                 raise ValueError(
                     f"Model name conflicts with existing model {model_spec.model_name}"

xinference/model/audio/model_spec_modelscope.json

@@ -0,0 +1,20 @@
+[
+  {
+    "model_name": "whisper-large-v3",
+    "model_family": "whisper",
+    "model_hub": "modelscope",
+    "model_id": "AI-ModelScope/whisper-large-v3",
+    "model_revision": "master",
+    "ability": "audio-to-text",
+    "multilingual": true
+  },
+  {
+    "model_name": "ChatTTS",
+    "model_family": "ChatTTS",
+    "model_hub": "modelscope",
+    "model_id": "pzc163/chatTTS",
+    "model_revision": "master",
+    "ability": "text-to-audio",
+    "multilingual": true
+  }
+]

xinference/model/llm/__init__.py

@@ -25,6 +25,7 @@ from .core import (
     get_llm_model_descriptions,
 )
 from .llm_family import (
+    BUILTIN_CSGHUB_LLM_FAMILIES,
     BUILTIN_LLM_FAMILIES,
     BUILTIN_LLM_MODEL_CHAT_FAMILIES,
     BUILTIN_LLM_MODEL_GENERATE_FAMILIES,
@@ -221,13 +222,44 @@ def _install():
         if "tools" in model_spec.model_ability:
             BUILTIN_LLM_MODEL_TOOL_CALL_FAMILIES.add(model_spec.model_name)
 
-    for llm_specs in [BUILTIN_LLM_FAMILIES, BUILTIN_MODELSCOPE_LLM_FAMILIES]:
+    csghub_json_path = os.path.join(
+        os.path.dirname(os.path.abspath(__file__)), "llm_family_csghub.json"
+    )
+    for json_obj in json.load(codecs.open(csghub_json_path, "r", encoding="utf-8")):
+        model_spec = LLMFamilyV1.parse_obj(json_obj)
+        BUILTIN_CSGHUB_LLM_FAMILIES.append(model_spec)
+
+        # register prompt style, in case that we have something missed
+        # if duplicated with huggingface json, keep it as the huggingface style
+        if (
+            "chat" in model_spec.model_ability
+            and isinstance(model_spec.prompt_style, PromptStyleV1)
+            and model_spec.model_name not in BUILTIN_LLM_PROMPT_STYLE
+        ):
+            BUILTIN_LLM_PROMPT_STYLE[model_spec.model_name] = model_spec.prompt_style
+        # register model family
+        if "chat" in model_spec.model_ability:
+            BUILTIN_LLM_MODEL_CHAT_FAMILIES.add(model_spec.model_name)
+        else:
+            BUILTIN_LLM_MODEL_GENERATE_FAMILIES.add(model_spec.model_name)
+        if "tools" in model_spec.model_ability:
+            BUILTIN_LLM_MODEL_TOOL_CALL_FAMILIES.add(model_spec.model_name)
+
+    for llm_specs in [
+        BUILTIN_LLM_FAMILIES,
+        BUILTIN_MODELSCOPE_LLM_FAMILIES,
+        BUILTIN_CSGHUB_LLM_FAMILIES,
+    ]:
         for llm_spec in llm_specs:
             if llm_spec.model_name not in LLM_MODEL_DESCRIPTIONS:
                 LLM_MODEL_DESCRIPTIONS.update(generate_llm_description(llm_spec))
 
     # traverse all families and add engine parameters corresponding to the model name
-    for families in [BUILTIN_LLM_FAMILIES, BUILTIN_MODELSCOPE_LLM_FAMILIES]:
+    for families in [
+        BUILTIN_LLM_FAMILIES,
+        BUILTIN_MODELSCOPE_LLM_FAMILIES,
+        BUILTIN_CSGHUB_LLM_FAMILIES,
+    ]:
         for family in families:
             generate_engine_config_by_model_family(family)
 

xinference/model/llm/llm_family.json

@@ -939,6 +939,8 @@
         "model_format": "pytorch",
         "model_size_in_billions": 9,
         "quantizations": [
+          "4-bit",
+          "8-bit",
           "none"
         ],
         "model_id": "THUDM/glm-4v-9b",

xinference/model/llm/llm_family.py

@@ -32,10 +32,15 @@ from ..._compat import (
     load_str_bytes,
     validator,
 )
-from ...constants import XINFERENCE_CACHE_DIR, XINFERENCE_MODEL_DIR
+from ...constants import (
+    XINFERENCE_CACHE_DIR,
+    XINFERENCE_ENV_CSG_TOKEN,
+    XINFERENCE_MODEL_DIR,
+)
 from ..utils import (
     IS_NEW_HUGGINGFACE_HUB,
     create_symlink,
+    download_from_csghub,
     download_from_modelscope,
     is_valid_model_uri,
     parse_uri,
@@ -232,6 +237,7 @@ LLAMA_CLASSES: List[Type[LLM]] = []
 
 BUILTIN_LLM_FAMILIES: List["LLMFamilyV1"] = []
 BUILTIN_MODELSCOPE_LLM_FAMILIES: List["LLMFamilyV1"] = []
+BUILTIN_CSGHUB_LLM_FAMILIES: List["LLMFamilyV1"] = []
 
 SGLANG_CLASSES: List[Type[LLM]] = []
 TRANSFORMERS_CLASSES: List[Type[LLM]] = []
@@ -292,6 +298,9 @@ def cache(
             elif llm_spec.model_hub == "modelscope":
                 logger.info(f"Caching from Modelscope: {llm_spec.model_id}")
                 return cache_from_modelscope(llm_family, llm_spec, quantization)
+            elif llm_spec.model_hub == "csghub":
+                logger.info(f"Caching from CSGHub: {llm_spec.model_id}")
+                return cache_from_csghub(llm_family, llm_spec, quantization)
             else:
                 raise ValueError(f"Unknown model hub: {llm_spec.model_hub}")
 
@@ -566,6 +575,7 @@ def _skip_download(
             "modelscope": _get_meta_path(
                 cache_dir, model_format, "modelscope", quantization
             ),
+            "csghub": _get_meta_path(cache_dir, model_format, "csghub", quantization),
         }
         if valid_model_revision(model_hub_to_meta_path[model_hub], model_revision):
             logger.info(f"Cache {cache_dir} exists")
@@ -650,6 +660,75 @@ def _merge_cached_files(
     logger.info(f"Merge complete.")
 
 
+def cache_from_csghub(
+    llm_family: LLMFamilyV1,
+    llm_spec: "LLMSpecV1",
+    quantization: Optional[str] = None,
+) -> str:
+    """
+    Cache model from CSGHub. Return the cache directory.
+    """
+    from pycsghub.file_download import file_download
+    from pycsghub.snapshot_download import snapshot_download
+
+    cache_dir = _get_cache_dir(llm_family, llm_spec)
+
+    if _skip_download(
+        cache_dir,
+        llm_spec.model_format,
+        llm_spec.model_hub,
+        llm_spec.model_revision,
+        quantization,
+    ):
+        return cache_dir
+
+    if llm_spec.model_format in ["pytorch", "gptq", "awq"]:
+        download_dir = retry_download(
+            snapshot_download,
+            llm_family.model_name,
+            {
+                "model_size": llm_spec.model_size_in_billions,
+                "model_format": llm_spec.model_format,
+            },
+            llm_spec.model_id,
+            endpoint="https://hub-stg.opencsg.com",
+            token=os.environ.get(XINFERENCE_ENV_CSG_TOKEN),
+        )
+        create_symlink(download_dir, cache_dir)
+
+    elif llm_spec.model_format in ["ggmlv3", "ggufv2"]:
+        file_names, final_file_name, need_merge = _generate_model_file_names(
+            llm_spec, quantization
+        )
+
+        for filename in file_names:
+            download_path = retry_download(
+                file_download,
+                llm_family.model_name,
+                {
+                    "model_size": llm_spec.model_size_in_billions,
+                    "model_format": llm_spec.model_format,
+                },
+                llm_spec.model_id,
+                file_name=filename,
+                endpoint="https://hub-stg.opencsg.com",
+                token=os.environ.get(XINFERENCE_ENV_CSG_TOKEN),
+            )
+            symlink_local_file(download_path, cache_dir, filename)
+
+        if need_merge:
+            _merge_cached_files(cache_dir, file_names, final_file_name)
+    else:
+        raise ValueError(f"Unsupported format: {llm_spec.model_format}")
+
+    meta_path = _get_meta_path(
+        cache_dir, llm_spec.model_format, llm_spec.model_hub, quantization
+    )
+    _generate_meta_file(meta_path, llm_family, llm_spec, quantization)
+
+    return cache_dir
+
+
 def cache_from_modelscope(
     llm_family: LLMFamilyV1,
     llm_spec: "LLMSpecV1",
@@ -931,6 +1010,12 @@ def match_llm(
             + BUILTIN_LLM_FAMILIES
             + user_defined_llm_families
         )
+    elif download_from_csghub():
+        all_families = (
+            BUILTIN_CSGHUB_LLM_FAMILIES
+            + BUILTIN_LLM_FAMILIES
+            + user_defined_llm_families
+        )
     else:
         all_families = BUILTIN_LLM_FAMILIES + user_defined_llm_families
 

xinference/model/llm/llm_family_csghub.json

@@ -0,0 +1,66 @@
+[
+  {
+    "version": 1,
+    "context_length": 32768,
+    "model_name": "qwen2-instruct",
+    "model_lang": [
+      "en",
+      "zh"
+    ],
+    "model_ability": [
+      "chat",
+      "tools"
+    ],
+    "model_description": "Qwen2 is the new series of Qwen large language models",
+    "model_specs": [
+      {
+        "model_format": "pytorch",
+        "model_size_in_billions": "0_5",
+        "quantizations": [
+          "4-bit",
+          "8-bit",
+          "none"
+        ],
+        "model_id": "Qwen/Qwen2-0.5B-Instruct",
+        "model_hub": "csghub"
+      },
+      {
+        "model_format": "ggufv2",
+        "model_size_in_billions": "0_5",
+        "quantizations": [
+          "q2_k",
+          "q3_k_m",
+          "q4_0",
+          "q4_k_m",
+          "q5_0",
+          "q5_k_m",
+          "q6_k",
+          "q8_0",
+          "fp16"
+        ],
+        "model_id": "qwen/Qwen2-0.5B-Instruct-GGUF",
+        "model_file_name_template": "qwen2-0_5b-instruct-{quantization}.gguf",
+        "model_hub": "csghub"
+      }
+    ],
+    "prompt_style": {
+      "style_name": "QWEN",
+      "system_prompt": "You are a helpful assistant.",
+      "roles": [
+        "user",
+        "assistant"
+      ],
+      "intra_message_sep": "\n",
+      "stop_token_ids": [
+        151643,
+        151644,
+        151645
+      ],
+      "stop": [
+        "<|endoftext|>",
+        "<|im_start|>",
+        "<|im_end|>"
+      ]
+    }
+  }
+]

xinference/model/llm/llm_family_modelscope.json

@@ -632,6 +632,8 @@
         "model_format": "pytorch",
         "model_size_in_billions": 9,
         "quantizations": [
+          "4-bit",
+          "8-bit",
           "none"
         ],
         "model_hub": "modelscope",

xinference/model/llm/pytorch/chatglm.py

@@ -89,24 +89,30 @@ class ChatglmPytorchChatModel(PytorchChatModel):
             return False
         return True
 
-    @staticmethod
-    def _handle_tools(generate_config) -> Optional[dict]:
+    def _handle_tools(self, generate_config) -> Optional[dict]:
         """Convert openai tools to ChatGLM tools."""
         if generate_config is None:
             return None
         tools = generate_config.pop("tools", None)
         if tools is None:
             return None
-        chatglm_tools = []
-        for elem in tools:
-            if elem.get("type") != "function" or "function" not in elem:
-                raise ValueError("ChatGLM tools only support function type.")
-            chatglm_tools.append(elem["function"])
-        return {
-            "role": "system",
-            "content": f"Answer the following questions as best as you can. You have access to the following tools:",
-            "tools": chatglm_tools,
-        }
+        if self.model_family.model_name == "glm4-chat":
+            return {
+                "role": "system",
+                "content": None,
+                "tools": tools,
+            }
+        else:
+            chatglm_tools = []
+            for elem in tools:
+                if elem.get("type") != "function" or "function" not in elem:
+                    raise ValueError("ChatGLM tools only support function type.")
+                chatglm_tools.append(elem["function"])
+            return {
+                "role": "system",
+                "content": f"Answer the following questions as best as you can. You have access to the following tools:",
+                "tools": chatglm_tools,
+            }
 
     def chat(
         self,