xinference 0.8.2__py3-none-any.whl → 0.8.3__py3-none-any.whl

xinference/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2024-01-26T16:31:51+0800",
+ "date": "2024-02-02T12:27:24+0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "6fa3ee0d57378ca0bbe038a84bd6a3df2010703d",
- "version": "0.8.2"
+ "full-revisionid": "749ef3ff298a94b88c1e67415819fae4fb1de75c",
+ "version": "0.8.3"
 }
 '''  # END VERSION_JSON
 

xinference/api/restful_api.py

@@ -62,6 +62,7 @@ from ..types import (
     CreateChatCompletion,
     CreateCompletion,
     ImageList,
+    max_tokens_field,
 )
 from .oauth2.auth_service import AuthService
 from .oauth2.types import LoginUserForm
@@ -216,6 +217,9 @@ class RESTfulAPI:
         self._router.add_api_route(
             "/v1/models/families", self._get_builtin_families, methods=["GET"]
         )
+        self._router.add_api_route(
+            "/v1/cluster/info", self.get_cluster_device_info, methods=["GET"]
+        )
         self._router.add_api_route(
             "/v1/cluster/devices", self._get_devices_count, methods=["GET"]
         )
@@ -791,6 +795,9 @@ class RESTfulAPI:
         }
         kwargs = body.dict(exclude_unset=True, exclude=exclude)
 
+        if body.max_tokens is None:
+            kwargs["max_tokens"] = max_tokens_field.default
+
         if body.logit_bias is not None:
             raise HTTPException(status_code=501, detail="Not implemented")
 
@@ -1079,6 +1086,9 @@ class RESTfulAPI:
         }
         kwargs = body.dict(exclude_unset=True, exclude=exclude)
 
+        if body.max_tokens is None:
+            kwargs["max_tokens"] = max_tokens_field.default
+
         if body.logit_bias is not None:
             raise HTTPException(status_code=501, detail="Not implemented")
 
@@ -1147,16 +1157,13 @@ class RESTfulAPI:
             raise HTTPException(status_code=500, detail=str(e))
 
         model_name = desc.get("model_name", "")
-        is_chatglm_ggml = (
-            desc.get("model_format") == "ggmlv3" and "chatglm" in model_name
-        )
         function_call_models = ["chatglm3", "gorilla-openfunctions-v1", "qwen-chat"]
 
         is_qwen = desc.get("model_format") == "ggmlv3" and "qwen" in model_name
 
-        if (is_chatglm_ggml or is_qwen) and system_prompt is not None:
+        if is_qwen and system_prompt is not None:
             raise HTTPException(
-                status_code=400, detail="ChatGLM ggml does not have system prompt"
+                status_code=400, detail="Qwen ggml does not have system prompt"
             )
 
         if not any(name in model_name for name in function_call_models):
@@ -1181,7 +1188,7 @@ class RESTfulAPI:
                 iterator = None
                 try:
                     try:
-                        if is_chatglm_ggml or is_qwen:
+                        if is_qwen:
                             iterator = await model.chat(prompt, chat_history, kwargs)
                         else:
                             iterator = await model.chat(
@@ -1201,7 +1208,7 @@ class RESTfulAPI:
             return EventSourceResponse(stream_results())
         else:
             try:
-                if is_chatglm_ggml or is_qwen:
+                if is_qwen:
                     data = await model.chat(prompt, chat_history, kwargs)
                 else:
                     data = await model.chat(prompt, system_prompt, chat_history, kwargs)
@@ -1285,6 +1292,14 @@ class RESTfulAPI:
             logger.error(e, exc_info=True)
             raise HTTPException(status_code=500, detail=str(e))
 
+    async def get_cluster_device_info(self) -> JSONResponse:
+        try:
+            data = await (await self._get_supervisor_ref()).get_cluster_device_info()
+            return JSONResponse(content=data)
+        except Exception as e:
+            logger.error(e, exc_info=True)
+            raise HTTPException(status_code=500, detail=str(e))
+
 
 def run(
     supervisor_address: str,

xinference/client/restful/restful_client.py

@@ -404,6 +404,7 @@ class RESTfulChatglmCppChatModelHandle(RESTfulModelHandle):
     def chat(
         self,
         prompt: str,
+        system_prompt: Optional[str] = None,
         chat_history: Optional[List["ChatCompletionMessage"]] = None,
         tools: Optional[List[Dict]] = None,
         generate_config: Optional["ChatglmCppGenerateConfig"] = None,
@@ -415,6 +416,8 @@ class RESTfulChatglmCppChatModelHandle(RESTfulModelHandle):
         ----------
         prompt: str
             The user's input.
+        system_prompt: Optional[str]
+            The system context provide to Model prior to any chats.
         chat_history: Optional[List["ChatCompletionMessage"]]
             A list of messages comprising the conversation so far.
         tools: Optional[List[Dict]]
@@ -441,6 +444,13 @@ class RESTfulChatglmCppChatModelHandle(RESTfulModelHandle):
         if chat_history is None:
             chat_history = []
 
+        if chat_history and chat_history[0]["role"] == "system":
+            if system_prompt is not None:
+                chat_history[0]["content"] = system_prompt
+        else:
+            if system_prompt is not None:
+                chat_history.insert(0, {"role": "system", "content": system_prompt})
+
         chat_history.append({"role": "user", "content": prompt})
 
         request_body: Dict[str, Any] = {

xinference/constants.py

@@ -18,8 +18,12 @@ from pathlib import Path
 XINFERENCE_ENV_ENDPOINT = "XINFERENCE_ENDPOINT"
 XINFERENCE_ENV_MODEL_SRC = "XINFERENCE_MODEL_SRC"
 XINFERENCE_ENV_HOME_PATH = "XINFERENCE_HOME"
-XINFERENCE_ENV_HEALTH_CHECK_ATTEMPTS = "XINFERENCE_HEALTH_CHECK_ATTEMPTS"
+XINFERENCE_ENV_HEALTH_CHECK_FAILURE_THRESHOLD = (
+    "XINFERENCE_HEALTH_CHECK_FAILURE_THRESHOLD"
+)
 XINFERENCE_ENV_HEALTH_CHECK_INTERVAL = "XINFERENCE_HEALTH_CHECK_INTERVAL"
+XINFERENCE_ENV_HEALTH_CHECK_TIMEOUT = "XINFERENCE_HEALTH_CHECK_TIMEOUT"
+XINFERENCE_ENV_DISABLE_HEALTH_CHECK = "XINFERENCE_DISABLE_HEALTH_CHECK"
 XINFERENCE_ENV_DISABLE_VLLM = "XINFERENCE_DISABLE_VLLM"
 
 
@@ -47,10 +51,16 @@ XINFERENCE_DEFAULT_ENDPOINT_PORT = 9997
 XINFERENCE_DEFAULT_LOG_FILE_NAME = "xinference.log"
 XINFERENCE_LOG_MAX_BYTES = 100 * 1024 * 1024
 XINFERENCE_LOG_BACKUP_COUNT = 30
-XINFERENCE_HEALTH_CHECK_ATTEMPTS = int(
-    os.environ.get(XINFERENCE_ENV_HEALTH_CHECK_ATTEMPTS, 3)
+XINFERENCE_HEALTH_CHECK_FAILURE_THRESHOLD = int(
+    os.environ.get(XINFERENCE_ENV_HEALTH_CHECK_FAILURE_THRESHOLD, 5)
 )
 XINFERENCE_HEALTH_CHECK_INTERVAL = int(
-    os.environ.get(XINFERENCE_ENV_HEALTH_CHECK_INTERVAL, 3)
+    os.environ.get(XINFERENCE_ENV_HEALTH_CHECK_INTERVAL, 5)
+)
+XINFERENCE_HEALTH_CHECK_TIMEOUT = int(
+    os.environ.get(XINFERENCE_ENV_HEALTH_CHECK_TIMEOUT, 10)
+)
+XINFERENCE_DISABLE_HEALTH_CHECK = bool(
+    int(os.environ.get(XINFERENCE_ENV_DISABLE_HEALTH_CHECK, 0))
 )
 XINFERENCE_DISABLE_VLLM = bool(int(os.environ.get(XINFERENCE_ENV_DISABLE_VLLM, 0)))

xinference/core/chat_interface.py

@@ -98,9 +98,16 @@ class GradioInterface:
             return flat_list
 
         def to_chat(lst: List[str]) -> List[ChatCompletionMessage]:
+            from ..model.llm import BUILTIN_LLM_PROMPT_STYLE
+
             res = []
+            prompt_style = BUILTIN_LLM_PROMPT_STYLE.get(self.model_name)
+            if prompt_style is None:
+                roles = ["assistant", "user"]
+            else:
+                roles = prompt_style.roles
             for i in range(len(lst)):
-                role = "assistant" if i % 2 == 1 else "user"
+                role = roles[0] if i % 2 == 1 else roles[1]
                 res.append(ChatCompletionMessage(role=role, content=lst[i]))
             return res
 

xinference/core/resource.py

@@ -13,10 +13,12 @@
 # limitations under the License.
 
 from dataclasses import dataclass
-from typing import Dict
+from typing import Dict, Union
 
 import psutil
 
+from .utils import get_nvidia_gpu_info
+
 
 @dataclass
 class ResourceStatus:
@@ -26,7 +28,14 @@ class ResourceStatus:
     memory_total: float
 
 
-def gather_node_info() -> Dict[str, ResourceStatus]:
+@dataclass
+class GPUStatus:
+    mem_total: float
+    mem_free: float
+    mem_used: float
+
+
+def gather_node_info() -> Dict[str, Union[ResourceStatus, GPUStatus]]:
     node_resource = dict()
     mem_info = psutil.virtual_memory()
     node_resource["cpu"] = ResourceStatus(
@@ -35,13 +44,11 @@ def gather_node_info() -> Dict[str, ResourceStatus]:
         memory_available=mem_info.available,
         memory_total=mem_info.total,
     )
-    # TODO: record GPU stats
-    # for idx, gpu_card_stat in enumerate(resource.cuda_card_stats()):
-    #     node_resource[f"gpu-{idx}"] = ResourceStatus(
-    #         available=gpu_card_stat.gpu_usage / 100.0,
-    #         total=1,
-    #         memory_available=gpu_card_stat.fb_mem_info.available,
-    #         memory_total=gpu_card_stat.fb_mem_info.total,
-    #     )
-
-    return node_resource
+    for gpu_idx, gpu_info in get_nvidia_gpu_info().items():
+        node_resource[gpu_idx] = GPUStatus(  # type: ignore
+            mem_total=gpu_info["total"],
+            mem_used=gpu_info["used"],
+            mem_free=gpu_info["free"],
+        )
+
+    return node_resource  # type: ignore

xinference/core/supervisor.py

@@ -21,10 +21,16 @@ from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Tuple, Un
 
 import xoscar as xo
 
+from ..constants import (
+    XINFERENCE_DISABLE_HEALTH_CHECK,
+    XINFERENCE_HEALTH_CHECK_FAILURE_THRESHOLD,
+    XINFERENCE_HEALTH_CHECK_INTERVAL,
+    XINFERENCE_HEALTH_CHECK_TIMEOUT,
+)
 from ..core import ModelActor
 from ..core.status_guard import InstanceInfo, LaunchStatus
 from .metrics import record_metrics
-from .resource import ResourceStatus
+from .resource import GPUStatus, ResourceStatus
 from .utils import (
     build_replica_model_uid,
     gen_random_string,
@@ -48,7 +54,6 @@ if TYPE_CHECKING:
 logger = getLogger(__name__)
 
 
-DEFAULT_NODE_TIMEOUT = 60
 ASYNC_LAUNCH_TASKS = {}  # type: ignore
 
 
@@ -60,7 +65,8 @@ def callback_for_async_launch(model_uid: str):
 @dataclass
 class WorkerStatus:
     update_time: float
-    status: Dict[str, ResourceStatus]
+    failure_remaining_count: int
+    status: Dict[str, Union[ResourceStatus, GPUStatus]]
 
 
 @dataclass
@@ -87,8 +93,15 @@ class SupervisorActor(xo.StatelessActor):
 
     async def __post_create__(self):
         self._uptime = time.time()
-        # comment this line to avoid worker lost
-        # self._check_dead_nodes_task = asyncio.create_task(self._check_dead_nodes())
+        if not XINFERENCE_DISABLE_HEALTH_CHECK:
+            # Run _check_dead_nodes() in a dedicated thread.
+            from ..isolation import Isolation
+
+            self._isolation = Isolation(asyncio.new_event_loop(), threaded=True)
+            self._isolation.start()
+            asyncio.run_coroutine_threadsafe(
+                self._check_dead_nodes(), loop=self._isolation.loop
+            )
         logger.info(f"Xinference supervisor {self.address} started")
         from .cache_tracker import CacheTrackerActor
         from .status_guard import StatusGuardActor
@@ -166,6 +179,30 @@ class SupervisorActor(xo.StatelessActor):
             model_version_infos, self.address
         )
 
+    async def get_cluster_device_info(self) -> List:
+        supervisor_device_info = {
+            "ip_address": self.address.split(":")[0],
+            "gpu_count": 0,
+            "gpu_vram_total": 0,
+        }
+        res = [{"node_type": "Supervisor", **supervisor_device_info}]
+        for worker_addr, worker_status in self._worker_status.items():
+            vram_total: float = sum(
+                [v.mem_total for k, v in worker_status.status.items() if k != "cpu"]  # type: ignore
+            )
+            total = (
+                vram_total if vram_total == 0 else f"{int(vram_total / 1024 / 1024)}MiB"
+            )
+            res.append(
+                {
+                    "node_type": "Worker",
+                    "ip_address": worker_addr.split(":")[0],
+                    "gpu_count": len(worker_status.status) - 1,
+                    "gpu_vram_total": total,
+                }
+            )
+        return res
+
     @staticmethod
     async def get_builtin_prompts() -> Dict[str, Any]:
         from ..model.llm.llm_family import BUILTIN_LLM_PROMPT_STYLE
@@ -752,27 +789,48 @@ class SupervisorActor(xo.StatelessActor):
 
     async def _check_dead_nodes(self):
         while True:
-            dead_nodes = []
-            for address, status in self._worker_status.items():
-                if time.time() - status.update_time > DEFAULT_NODE_TIMEOUT:
-                    dead_models = []
-                    for model_uid in self._replica_model_uid_to_worker:
-                        if (
-                            self._replica_model_uid_to_worker[model_uid].address
-                            == address
-                        ):
-                            dead_models.append(model_uid)
-                    logger.error(
-                        "Worker timeout. address: %s, influenced models: %s",
-                        address,
-                        dead_models,
-                    )
-                    dead_nodes.append(address)
-
-            for address in dead_nodes:
-                self._worker_status.pop(address)
-                self._worker_address_to_worker.pop(address)
-            await asyncio.sleep(5)
+            try:
+                dead_nodes = []
+                for address, status in self._worker_status.items():
+                    if (
+                        time.time() - status.update_time
+                        > XINFERENCE_HEALTH_CHECK_TIMEOUT
+                    ):
+                        status.failure_remaining_count -= 1
+                    else:
+                        status.failure_remaining_count = (
+                            XINFERENCE_HEALTH_CHECK_FAILURE_THRESHOLD
+                        )
+
+                    if status.failure_remaining_count <= 0:
+                        dead_models = []
+                        for model_uid in self._replica_model_uid_to_worker:
+                            if (
+                                self._replica_model_uid_to_worker[model_uid].address
+                                == address
+                            ):
+                                dead_models.append(model_uid)
+                        logger.error(
+                            "Worker dead. address: %s, influenced models: %s",
+                            address,
+                            dead_models,
+                        )
+                        dead_nodes.append(address)
+                    elif (
+                        status.failure_remaining_count
+                        != XINFERENCE_HEALTH_CHECK_FAILURE_THRESHOLD
+                    ):
+                        logger.error(
+                            "Worker timeout. address: %s, check count remaining %s...",
+                            address,
+                            status.failure_remaining_count,
+                        )
+
+                for address in dead_nodes:
+                    self._worker_status.pop(address, None)
+                    self._worker_address_to_worker.pop(address, None)
+            finally:
+                await asyncio.sleep(XINFERENCE_HEALTH_CHECK_INTERVAL)
 
     @log_async(logger=logger)
     async def terminate_model(self, model_uid: str, suppress_exception=False):
@@ -871,13 +929,19 @@ class SupervisorActor(xo.StatelessActor):
             )
 
     async def report_worker_status(
-        self, worker_address: str, status: Dict[str, ResourceStatus]
+        self, worker_address: str, status: Dict[str, Union[ResourceStatus, GPUStatus]]
     ):
         if worker_address not in self._worker_status:
             logger.debug("Worker %s resources: %s", worker_address, status)
-        self._worker_status[worker_address] = WorkerStatus(
-            update_time=time.time(), status=status
-        )
+            self._worker_status[worker_address] = WorkerStatus(
+                update_time=time.time(),
+                failure_remaining_count=XINFERENCE_HEALTH_CHECK_FAILURE_THRESHOLD,
+                status=status,
+            )
+        else:
+            worker_status = self._worker_status[worker_address]
+            worker_status.update_time = time.time()
+            worker_status.status = status
 
     @staticmethod
     def record_metrics(name, op, kwargs):

xinference/core/utils.py

@@ -16,10 +16,11 @@ import logging
 import os
 import random
 import string
-from typing import Generator, List, Tuple, Union
+from typing import Dict, Generator, List, Tuple, Union
 
 import orjson
 from pydantic import BaseModel
+from pynvml import nvmlDeviceGetCount, nvmlInit, nvmlShutdown
 
 logger = logging.getLogger(__name__)
 
@@ -162,3 +163,30 @@ def parse_model_version(model_version: str, model_type: str) -> Tuple:
         return tuple(results)
     else:
         raise ValueError(f"Not supported model_type: {model_type}")
+
+
+def _get_nvidia_gpu_mem_info(gpu_id: int) -> Dict[str, float]:
+    from pynvml import nvmlDeviceGetHandleByIndex, nvmlDeviceGetMemoryInfo
+
+    handler = nvmlDeviceGetHandleByIndex(gpu_id)
+    mem_info = nvmlDeviceGetMemoryInfo(handler)
+    return {"total": mem_info.total, "used": mem_info.used, "free": mem_info.free}
+
+
+def get_nvidia_gpu_info() -> Dict:
+    try:
+        nvmlInit()
+        device_count = nvmlDeviceGetCount()
+        res = {}
+        for i in range(device_count):
+            res[f"gpu-{i}"] = _get_nvidia_gpu_mem_info(i)
+        return res
+    except:
+        # TODO: add log here
+        # logger.debug(f"Cannot init nvml. Maybe due to lack of NVIDIA GPUs or incorrect installation of CUDA.")
+        return {}
+    finally:
+        try:
+            nvmlShutdown()
+        except:
+            pass

xinference/core/worker.py

@@ -24,6 +24,7 @@ from logging import getLogger
 from typing import Any, Dict, List, Optional, Set, Tuple, Union
 
 import xoscar as xo
+from async_timeout import timeout
 from xoscar import MainActorPoolType
 
 from ..constants import XINFERENCE_CACHE_DIR
@@ -152,6 +153,7 @@ class WorkerActor(xo.StatelessActor):
         return "worker"
 
     async def __post_create__(self):
+        from ..isolation import Isolation
         from .cache_tracker import CacheTrackerActor
         from .status_guard import StatusGuardActor
         from .supervisor import SupervisorActor
@@ -175,7 +177,12 @@ class WorkerActor(xo.StatelessActor):
             address=self._supervisor_address, uid=SupervisorActor.uid()
         )
         await self._supervisor_ref.add_worker(self.address)
-        self._upload_task = asyncio.create_task(self._periodical_report_status())
+        # Run _periodical_report_status() in a dedicated thread.
+        self._isolation = Isolation(asyncio.new_event_loop(), threaded=True)
+        self._isolation.start()
+        asyncio.run_coroutine_threadsafe(
+            self._periodical_report_status(), loop=self._isolation.loop
+        )
         logger.info(f"Xinference worker {self.address} started")
         logger.info("Purge cache directory: %s", XINFERENCE_CACHE_DIR)
         purge_dir(XINFERENCE_CACHE_DIR)
@@ -233,7 +240,7 @@ class WorkerActor(xo.StatelessActor):
             )
 
     async def __pre_destroy__(self):
-        self._upload_task.cancel()
+        self._isolation.stop()
 
     @staticmethod
     def get_devices_count():
@@ -628,7 +635,15 @@ class WorkerActor(xo.StatelessActor):
         return model_desc.to_dict()
 
     async def report_status(self):
-        status = await asyncio.to_thread(gather_node_info)
+        status = dict()
+        try:
+            # asyncio.timeout is only available in Python >= 3.11
+            async with timeout(2):
+                status = await asyncio.to_thread(gather_node_info)
+        except asyncio.CancelledError:
+            raise
+        except Exception:
+            logger.exception("Report status got error.")
         await self._supervisor_ref.report_worker_status(self.address, status)
 
     async def _periodical_report_status(self):

xinference/deploy/local.py

@@ -23,7 +23,7 @@ import xoscar as xo
 from xoscar.utils import get_next_port
 
 from ..constants import (
-    XINFERENCE_HEALTH_CHECK_ATTEMPTS,
+    XINFERENCE_HEALTH_CHECK_FAILURE_THRESHOLD,
     XINFERENCE_HEALTH_CHECK_INTERVAL,
 )
 from ..core.supervisor import SupervisorActor
@@ -116,7 +116,7 @@ def main(
 
     if not health_check(
         address=supervisor_address,
-        max_attempts=XINFERENCE_HEALTH_CHECK_ATTEMPTS,
+        max_attempts=XINFERENCE_HEALTH_CHECK_FAILURE_THRESHOLD,
         sleep_interval=XINFERENCE_HEALTH_CHECK_INTERVAL,
     ):
         raise RuntimeError("Cluster is not available after multiple attempts")

xinference/deploy/supervisor.py

@@ -23,7 +23,7 @@ import xoscar as xo
 from xoscar.utils import get_next_port
 
 from ..constants import (
-    XINFERENCE_HEALTH_CHECK_ATTEMPTS,
+    XINFERENCE_HEALTH_CHECK_FAILURE_THRESHOLD,
     XINFERENCE_HEALTH_CHECK_INTERVAL,
 )
 from ..core.supervisor import SupervisorActor
@@ -82,7 +82,7 @@ def main(
 
     if not health_check(
         address=supervisor_address,
-        max_attempts=XINFERENCE_HEALTH_CHECK_ATTEMPTS,
+        max_attempts=XINFERENCE_HEALTH_CHECK_FAILURE_THRESHOLD,
         sleep_interval=XINFERENCE_HEALTH_CHECK_INTERVAL,
     ):
         raise RuntimeError("Supervisor is not available after multiple attempts")

xinference/model/audio/model_spec.json

@@ -27,6 +27,20 @@
     "model_revision": "911407f4214e0e1d82085af863093ec0b66f9cd6",
     "multilingual": false
   },
+  {
+    "model_name": "whisper-small",
+    "model_family": "whisper",
+    "model_id": "openai/whisper-small",
+    "model_revision": "998cb1a777c20db53d6033a61b977ed4c3792cac",
+    "multilingual": true
+  },
+  {
+    "model_name": "whisper-small.en",
+    "model_family": "whisper",
+    "model_id": "openai/whisper-small.en",
+    "model_revision": "e8727524f962ee844a7319d92be39ac1bd25655a",
+    "multilingual": false
+  },
   {
     "model_name": "whisper-medium",
     "model_family": "whisper",
@@ -47,5 +61,19 @@
     "model_id": "openai/whisper-large-v3",
     "model_revision": "6cdf07a7e3ec3806e5d55f787915b85d4cd020b1",
     "multilingual": true
+  },
+  {
+    "model_name": "Belle-distilwhisper-large-v2-zh",
+    "model_family": "whisper",
+    "model_id": "BELLE-2/Belle-distilwhisper-large-v2-zh",
+    "model_revision": "ed25d13498fa5bac758b2fc479435b698532dfe8",
+    "multilingual": false
+  },
+  {
+    "model_name": "Belle-whisper-large-v2-zh",
+    "model_family": "whisper",
+    "model_id": "BELLE-2/Belle-whisper-large-v2-zh",
+    "model_revision": "ec5bd5d78598545b7585814edde86dac2002b5b9",
+    "multilingual": false
   }
-]
+]

xinference/model/embedding/model_spec.json

@@ -143,6 +143,14 @@
     "model_id": "jinaai/jina-embeddings-v2-base-en",
     "model_revision": "7302ac470bed880590f9344bfeee32ff8722d0e5"
   },
+  {
+    "model_name": "jina-embeddings-v2-base-zh",
+    "dimensions": 768,
+    "max_tokens": 8192,
+    "language": ["zh", "en"],
+    "model_id": "jinaai/jina-embeddings-v2-base-zh",
+    "model_revision": "67974cbef5cf50562eadd745de8afc661c52c96f"
+  },
   {
     "model_name": "text2vec-large-chinese",
     "dimensions": 1024,
@@ -182,5 +190,21 @@
     "language": ["zh"],
     "model_id": "shibing624/text2vec-base-multilingual",
     "model_revision": "f241877385fa56ebcc75f04d1850e1579cfa661d"
+  },
+  {
+    "model_name": "bge-m3",
+    "dimensions": 1024,
+    "max_tokens": 8192,
+    "language": ["zh", "en"],
+    "model_id": "BAAI/bge-m3",
+    "model_revision": "73a15ad29ab604f3bdc31601849a9defe86d563f"
+  },
+  {
+    "model_name": "bce-embedding-base_v1",
+    "dimensions": 768,
+    "max_tokens": 512,
+    "language": ["zh", "en"],
+    "model_id": "maidalun1020/bce-embedding-base_v1",
+    "model_revision": "236d9024fc1b4046f03848723f934521a66a9323"
   }
 ]

xinference/model/embedding/model_spec_modelscope.json

@@ -161,6 +161,14 @@
     "model_revision": "v0.0.1",
     "model_hub": "modelscope"
   },
+  {
+    "model_name": "jina-embeddings-v2-base-zh",
+    "dimensions": 768,
+    "max_tokens": 8192,
+    "language": ["zh", "en"],
+    "model_id": "jinaai/jina-embeddings-v2-base-zh",
+    "model_hub": "modelscope"
+  },
   {
     "model_name": "text2vec-large-chinese",
     "dimensions": 1024,
@@ -184,5 +192,21 @@
     "language": ["zh"],
     "model_id": "mwei23/text2vec-base-chinese-paraphrase",
     "model_hub": "modelscope"
+  },
+  {
+    "model_name": "bge-m3",
+    "dimensions": 1024,
+    "max_tokens": 8192,
+    "language": ["zh", "en"],
+    "model_id": "Xorbits/bge-m3",
+    "model_hub": "modelscope"
+  },
+  {
+    "model_name": "bce-embedding-base_v1",
+    "dimensions": 768,
+    "max_tokens": 512,
+    "language": ["zh", "en"],
+    "model_id": "maidalun/bce-embedding-base_v1",
+    "model_hub": "modelscope"
   }
 ]