xinference 0.13.0__py3-none-any.whl → 0.13.1__py3-none-any.whl

xinference/model/flexible/core.py

@@ -0,0 +1,228 @@
+# Copyright 2022-2024 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import logging
+import os
+from collections import defaultdict
+from threading import Lock
+from typing import Dict, List, Optional, Tuple
+
+from ...constants import XINFERENCE_CACHE_DIR, XINFERENCE_MODEL_DIR
+from ..core import CacheableModelSpec, ModelDescription
+from .utils import get_launcher
+
+logger = logging.getLogger(__name__)
+
+FLEXIBLE_MODEL_LOCK = Lock()
+
+
+class FlexibleModelSpec(CacheableModelSpec):
+    model_id: Optional[str]  # type: ignore
+    model_description: Optional[str]
+    model_uri: Optional[str]
+    launcher: str
+    launcher_args: Optional[str]
+
+    def parser_args(self):
+        return json.loads(self.launcher_args)
+
+
+class FlexibleModelDescription(ModelDescription):
+    def __init__(
+        self,
+        address: Optional[str],
+        devices: Optional[List[str]],
+        model_spec: FlexibleModelSpec,
+        model_path: Optional[str] = None,
+    ):
+        super().__init__(address, devices, model_path=model_path)
+        self._model_spec = model_spec
+
+    def to_dict(self):
+        return {
+            "model_type": "flexible",
+            "address": self.address,
+            "accelerators": self.devices,
+            "model_name": self._model_spec.model_name,
+            "launcher": self._model_spec.launcher,
+            "launcher_args": self._model_spec.launcher_args,
+        }
+
+    def get_model_version(self) -> str:
+        return f"{self._model_spec.model_name}"
+
+    def to_version_info(self):
+        return {
+            "model_version": self.get_model_version(),
+            "cache_status": True,
+            "model_file_location": self._model_spec.model_uri,
+            "launcher": self._model_spec.launcher,
+            "launcher_args": self._model_spec.launcher_args,
+        }
+
+
+def generate_flexible_model_description(
+    model_spec: FlexibleModelSpec,
+) -> Dict[str, List[Dict]]:
+    res = defaultdict(list)
+    res[model_spec.model_name].append(
+        FlexibleModelDescription(None, None, model_spec).to_version_info()
+    )
+    return res
+
+
+FLEXIBLE_MODELS: List[FlexibleModelSpec] = []
+FLEXIBLE_MODEL_DESCRIPTIONS: Dict[str, List[Dict]] = defaultdict(list)
+
+
+def get_flexible_models():
+    with FLEXIBLE_MODEL_LOCK:
+        return FLEXIBLE_MODELS.copy()
+
+
+def get_flexible_model_descriptions():
+    import copy
+
+    return copy.deepcopy(FLEXIBLE_MODEL_DESCRIPTIONS)
+
+
+def register_flexible_model(model_spec: FlexibleModelSpec, persist: bool):
+    from ..utils import is_valid_model_name
+
+    if not is_valid_model_name(model_spec.model_name):
+        raise ValueError(f"Invalid model name {model_spec.model_name}.")
+
+    if model_spec.launcher_args:
+        try:
+            model_spec.parser_args()
+        except Exception:
+            raise ValueError(f"Invalid model launcher args {model_spec.launcher_args}.")
+
+    with FLEXIBLE_MODEL_LOCK:
+        for model_name in [spec.model_name for spec in FLEXIBLE_MODELS]:
+            if model_spec.model_name == model_name:
+                raise ValueError(
+                    f"Model name conflicts with existing model {model_spec.model_name}"
+                )
+        FLEXIBLE_MODELS.append(model_spec)
+
+    if persist:
+        persist_path = os.path.join(
+            XINFERENCE_MODEL_DIR, "flexible", f"{model_spec.model_name}.json"
+        )
+        os.makedirs(os.path.dirname(persist_path), exist_ok=True)
+        with open(persist_path, mode="w") as fd:
+            fd.write(model_spec.json())
+
+
+def unregister_flexible_model(model_name: str, raise_error: bool = True):
+    with FLEXIBLE_MODEL_LOCK:
+        model_spec = None
+        for i, f in enumerate(FLEXIBLE_MODELS):
+            if f.model_name == model_name:
+                model_spec = f
+                break
+        if model_spec:
+            FLEXIBLE_MODELS.remove(model_spec)
+
+            persist_path = os.path.join(
+                XINFERENCE_MODEL_DIR, "flexible", f"{model_spec.model_name}.json"
+            )
+            if os.path.exists(persist_path):
+                os.remove(persist_path)
+
+            cache_dir = os.path.join(XINFERENCE_CACHE_DIR, model_spec.model_name)
+            if os.path.exists(cache_dir):
+                logger.warning(
+                    f"Remove the cache of user-defined model {model_spec.model_name}. "
+                    f"Cache directory: {cache_dir}"
+                )
+                if os.path.islink(cache_dir):
+                    os.remove(cache_dir)
+                else:
+                    logger.warning(
+                        f"Cache directory is not a soft link, please remove it manually."
+                    )
+        else:
+            if raise_error:
+                raise ValueError(f"Model {model_name} not found")
+            else:
+                logger.warning(f"Model {model_name} not found")
+
+
+class FlexibleModel:
+    def __init__(
+        self,
+        model_uid: str,
+        model_path: str,
+        device: Optional[str] = None,
+        config: Optional[Dict] = None,
+    ):
+        self._model_uid = model_uid
+        self._model_path = model_path
+        self._device = device
+        self._config = config
+
+    def load(self):
+        """
+        Load the model.
+        """
+
+    def infer(self, **kwargs):
+        """
+        Call model to inference.
+        """
+        raise NotImplementedError("infer method not implemented.")
+
+    @property
+    def model_uid(self):
+        return self._model_uid
+
+    @property
+    def model_path(self):
+        return self._model_path
+
+    @property
+    def device(self):
+        return self._device
+
+    @property
+    def config(self):
+        return self._config
+
+
+def match_flexible_model(model_name):
+    for model_spec in get_flexible_models():
+        if model_name == model_spec.model_name:
+            return model_spec
+
+
+def create_flexible_model_instance(
+    subpool_addr: str, devices: List[str], model_uid: str, model_name: str, **kwargs
+) -> Tuple[FlexibleModel, FlexibleModelDescription]:
+    model_spec = match_flexible_model(model_name)
+    model_path = model_spec.model_uri
+    launcher_name = model_spec.launcher
+    launcher_args = model_spec.parser_args()
+    kwargs.update(launcher_args)
+
+    model = get_launcher(launcher_name)(
+        model_uid=model_uid, model_spec=model_spec, **kwargs
+    )
+
+    model_description = FlexibleModelDescription(
+        subpool_addr, devices, model_spec, model_path=model_path
+    )
+    return model, model_description

xinference/model/flexible/launchers/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2022-2024 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .transformers_launcher import launcher as transformers

xinference/model/flexible/launchers/transformers_launcher.py

@@ -0,0 +1,63 @@
+# Copyright 2022-2024 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from transformers import pipeline
+
+from ..core import FlexibleModel, FlexibleModelSpec
+
+
+class MockModel(FlexibleModel):
+    def infer(self, **kwargs):
+        return kwargs
+
+
+class AutoModel(FlexibleModel):
+    def load(self):
+        config = self.config or {}
+        self._pipeline = pipeline(model=self.model_path, device=self.device, **config)
+
+    def infer(self, **kwargs):
+        return self._pipeline(**kwargs)
+
+
+class TransformersTextClassificationModel(FlexibleModel):
+    def load(self):
+        config = self.config or {}
+
+        self._pipeline = pipeline(model=self._model_path, device=self._device, **config)
+
+    def infer(self, **kwargs):
+        return self._pipeline(**kwargs)
+
+
+def launcher(model_uid: str, model_spec: FlexibleModelSpec, **kwargs) -> FlexibleModel:
+    task = kwargs.get("task")
+    device = kwargs.get("device")
+
+    model_path = model_spec.model_uri
+    if model_path is None:
+        raise ValueError("model_path required")
+
+    if task == "text-classification":
+        return TransformersTextClassificationModel(
+            model_uid=model_uid, model_path=model_path, device=device, config=kwargs
+        )
+    elif task == "mock":
+        return MockModel(
+            model_uid=model_uid, model_path=model_path, device=device, config=kwargs
+        )
+    else:
+        return AutoModel(
+            model_uid=model_uid, model_path=model_path, device=device, config=kwargs
+        )

xinference/model/flexible/utils.py

@@ -0,0 +1,33 @@
+# Copyright 2022-2024 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import importlib
+
+
+def get_launcher(launcher_name: str):
+    try:
+        i = launcher_name.rfind(".")
+        if i != -1:
+            module = importlib.import_module(launcher_name[:i])
+            fn = getattr(module, launcher_name[i + 1 :])
+        else:
+            importlib.import_module(launcher_name)
+            fn = locals().get(launcher_name)
+
+        if fn is None:
+            raise ValueError(f"Launcher {launcher_name} not found.")
+
+        return fn
+    except ImportError as e:
+        raise ImportError(f"Failed to import {launcher_name}: {e}")

xinference/model/image/core.py

@@ -15,7 +15,7 @@ import collections.abc
 import logging
 import os
 from collections import defaultdict
-from typing import Dict, List, Optional, Tuple
+from typing import Dict, List, Literal, Optional, Tuple
 
 from ...constants import XINFERENCE_CACHE_DIR
 from ...types import PeftModelConfig
@@ -117,7 +117,10 @@ def generate_image_description(
     return res
 
 
-def match_diffusion(model_name: str) -> ImageModelFamilyV1:
+def match_diffusion(
+    model_name: str,
+    download_hub: Optional[Literal["huggingface", "modelscope", "csghub"]] = None,
+) -> ImageModelFamilyV1:
     from ..utils import download_from_modelscope
     from . import BUILTIN_IMAGE_MODELS, MODELSCOPE_IMAGE_MODELS
     from .custom import get_user_defined_images
@@ -126,17 +129,17 @@ def match_diffusion(model_name: str) -> ImageModelFamilyV1:
         if model_spec.model_name == model_name:
             return model_spec
 
-    if download_from_modelscope():
-        if model_name in MODELSCOPE_IMAGE_MODELS:
-            logger.debug(f"Image model {model_name} found in ModelScope.")
-            return MODELSCOPE_IMAGE_MODELS[model_name]
-        else:
-            logger.debug(
-                f"Image model {model_name} not found in ModelScope, "
-                f"now try to load it via builtin way."
-            )
-
-    if model_name in BUILTIN_IMAGE_MODELS:
+    if download_hub == "modelscope" and model_name in MODELSCOPE_IMAGE_MODELS:
+        logger.debug(f"Image model {model_name} found in ModelScope.")
+        return MODELSCOPE_IMAGE_MODELS[model_name]
+    elif download_hub == "huggingface" and model_name in BUILTIN_IMAGE_MODELS:
+        logger.debug(f"Image model {model_name} found in Huggingface.")
+        return BUILTIN_IMAGE_MODELS[model_name]
+    elif download_from_modelscope() and model_name in MODELSCOPE_IMAGE_MODELS:
+        logger.debug(f"Image model {model_name} found in ModelScope.")
+        return MODELSCOPE_IMAGE_MODELS[model_name]
+    elif model_name in BUILTIN_IMAGE_MODELS:
+        logger.debug(f"Image model {model_name} found in Huggingface.")
         return BUILTIN_IMAGE_MODELS[model_name]
     else:
         raise ValueError(
@@ -183,9 +186,10 @@ def create_image_model_instance(
     model_uid: str,
     model_name: str,
     peft_model_config: Optional[PeftModelConfig] = None,
+    download_hub: Optional[Literal["huggingface", "modelscope", "csghub"]] = None,
     **kwargs,
 ) -> Tuple[DiffusionModel, ImageModelDescription]:
-    model_spec = match_diffusion(model_name)
+    model_spec = match_diffusion(model_name, download_hub)
     controlnet = kwargs.get("controlnet")
     # Handle controlnet
     if controlnet is not None:

xinference/model/image/custom.py

@@ -66,7 +66,7 @@ def register_image(model_spec: CustomImageModelFamilyV1, persist: bool):
             raise ValueError(f"Invalid model URI {model_uri}")
 
         persist_path = os.path.join(
-            XINFERENCE_MODEL_DIR, "image", f"{model_spec.model_id}.json"
+            XINFERENCE_MODEL_DIR, "image", f"{model_spec.model_name}.json"
         )
         os.makedirs(os.path.dirname(persist_path), exist_ok=True)
         with open(persist_path, "w") as f:

xinference/model/llm/__init__.py

@@ -112,7 +112,6 @@ def generate_engine_config_by_model_family(model_family):
 
 
 def _install():
-    from .ggml.chatglm import ChatglmCppChatModel
     from .ggml.llamacpp import LlamaCppChatModel, LlamaCppModel
     from .mlx.core import MLXChatModel, MLXModel
     from .pytorch.baichuan import BaichuanPytorchChatModel
@@ -143,7 +142,6 @@ def _install():
     # register llm classes.
     LLAMA_CLASSES.extend(
         [
-            ChatglmCppChatModel,
             LlamaCppChatModel,
             LlamaCppModel,
         ]

xinference/model/llm/core.py

@@ -20,7 +20,7 @@ import platform
 from abc import abstractmethod
 from collections import defaultdict
 from functools import lru_cache
-from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union
+from typing import TYPE_CHECKING, Dict, List, Literal, Optional, Tuple, Union
 
 from ...core.utils import parse_replica_model_uid
 from ...types import PeftModelConfig
@@ -193,6 +193,7 @@ def create_llm_model_instance(
     model_size_in_billions: Optional[Union[int, str]] = None,
     quantization: Optional[str] = None,
     peft_model_config: Optional[PeftModelConfig] = None,
+    download_hub: Optional[Literal["huggingface", "modelscope", "csghub"]] = None,
     **kwargs,
 ) -> Tuple[LLM, LLMDescription]:
     from .llm_family import cache, check_engine_by_spec_parameters, match_llm
@@ -200,7 +201,7 @@ def create_llm_model_instance(
     if model_engine is None:
         raise ValueError("model_engine is required for LLM model")
     match_result = match_llm(
-        model_name, model_format, model_size_in_billions, quantization
+        model_name, model_format, model_size_in_billions, quantization, download_hub
     )
 
     if not match_result:

xinference/model/llm/ggml/llamacpp.py

@@ -25,7 +25,6 @@ from ....types import (
     CompletionChunk,
     CompletionUsage,
     CreateCompletionLlamaCpp,
-    Embedding,
     LlamaCppGenerateConfig,
     LlamaCppModelConfig,
 )
@@ -65,7 +64,6 @@ class LlamaCppModel(LLM):
 
         if self.model_family.context_length:
             llamacpp_model_config.setdefault("n_ctx", self.model_family.context_length)
-        llamacpp_model_config.setdefault("embedding", True)
         llamacpp_model_config.setdefault("use_mmap", False)
         llamacpp_model_config.setdefault("use_mlock", True)
 
@@ -185,7 +183,7 @@ class LlamaCppModel(LLM):
     ) -> bool:
         if llm_spec.model_format not in ["ggmlv3", "ggufv2"]:
             return False
-        if "chatglm" in llm_family.model_name or "qwen" in llm_family.model_name:
+        if "qwen" in llm_family.model_name:
             return False
         if "generate" not in llm_family.model_ability:
             return False
@@ -261,11 +259,6 @@ class LlamaCppModel(LLM):
         else:
             return generator_wrapper(prompt, generate_config)
 
-    def create_embedding(self, input: Union[str, List[str]]) -> Embedding:
-        assert self._llm is not None
-        embedding = self._llm.create_embedding(input)
-        return embedding
-
 
 class LlamaCppChatModel(LlamaCppModel, ChatModelMixin):
     def __init__(
@@ -292,8 +285,6 @@ class LlamaCppChatModel(LlamaCppModel, ChatModelMixin):
     ) -> bool:
         if llm_spec.model_format not in ["ggmlv3", "ggufv2"]:
             return False
-        if "chatglm" in llm_family.model_name:
-            return False
         if "chat" not in llm_family.model_ability:
             return False
         return True

xinference/model/llm/llm_family.json

@@ -574,19 +574,6 @@
     ],
     "model_description": "ChatGLM is an open-source General Language Model (GLM) based LLM trained on both Chinese and English data.",
     "model_specs": [
-      {
-        "model_format": "ggmlv3",
-        "model_size_in_billions": 6,
-        "quantizations": [
-          "q4_0",
-          "q4_1",
-          "q5_0",
-          "q5_1",
-          "q8_0"
-        ],
-        "model_id": "Xorbits/chatglm-6B-GGML",
-        "model_file_name_template": "chatglm-ggml-{quantization}.bin"
-      },
       {
         "model_format": "pytorch",
         "model_size_in_billions": 6,
@@ -622,19 +609,6 @@
     ],
     "model_description": "ChatGLM2 is the second generation of ChatGLM, still open-source and trained on Chinese and English data.",
     "model_specs": [
-      {
-        "model_format": "ggmlv3",
-        "model_size_in_billions": 6,
-        "quantizations": [
-          "q4_0",
-          "q4_1",
-          "q5_0",
-          "q5_1",
-          "q8_0"
-        ],
-        "model_id": "Xorbits/chatglm2-6B-GGML",
-        "model_file_name_template": "chatglm2-ggml-{quantization}.bin"
-      },
       {
         "model_format": "pytorch",
         "model_size_in_billions": 6,
@@ -706,15 +680,6 @@
     ],
     "model_description": "ChatGLM3 is the third generation of ChatGLM, still open-source and trained on Chinese and English data.",
     "model_specs": [
-      {
-        "model_format": "ggmlv3",
-        "model_size_in_billions": 6,
-        "quantizations": [
-          "q4_0"
-        ],
-        "model_id": "Xorbits/chatglm3-6B-GGML",
-        "model_file_name_template": "chatglm3-ggml-{quantization}.bin"
-      },
       {
         "model_format": "pytorch",
         "model_size_in_billions": 6,
@@ -855,6 +820,32 @@
         ],
         "model_id": "THUDM/glm-4-9b-chat",
         "model_revision": "b84dc74294ccd507a3d78bde8aebf628221af9bd"
+      },
+      {
+        "model_format": "ggufv2",
+        "model_size_in_billions": 9,
+        "quantizations": [
+          "Q2_K",
+          "IQ3_XS",
+          "IQ3_S",
+          "IQ3_M",
+          "Q3_K_S",
+          "Q3_K_L",
+          "Q3_K",
+          "IQ4_XS",
+          "IQ4_NL",
+          "Q4_K_S",
+          "Q4_K",
+          "Q5_K_S",
+          "Q5_K",
+          "Q6_K",
+          "Q8_0",
+          "BF16",
+          "FP16"
+        ],
+        "model_file_name_template": "glm-4-9b-chat.{quantization}.gguf",
+        "model_id": "legraphista/glm-4-9b-chat-GGUF",
+        "model_revision": "0155a14edf0176863e9a003cdd78ce599e4d62c0"
       }
     ],
     "prompt_style": {
@@ -900,6 +891,32 @@
         ],
         "model_id": "THUDM/glm-4-9b-chat-1m",
         "model_revision": "715ddbe91082f976ff6a4ca06d59e5bbff6c3642"
+      },
+      {
+        "model_format": "ggufv2",
+        "model_size_in_billions": 9,
+        "quantizations": [
+          "Q2_K",
+          "IQ3_XS",
+          "IQ3_S",
+          "IQ3_M",
+          "Q3_K_S",
+          "Q3_K_L",
+          "Q3_K",
+          "IQ4_XS",
+          "IQ4_NL",
+          "Q4_K_S",
+          "Q4_K",
+          "Q5_K_S",
+          "Q5_K",
+          "Q6_K",
+          "Q8_0",
+          "BF16",
+          "FP16"
+        ],
+        "model_file_name_template": "glm-4-9b-chat-1m.{quantization}.gguf",
+        "model_id": "legraphista/glm-4-9b-chat-1m-GGUF",
+        "model_revision": "782e28bd5eee3c514c07108da15e0b5e06dcf776"
       }
     ],
     "prompt_style": {