xinference 0.5.4__py3-none-any.whl → 0.5.6__py3-none-any.whl

xinference/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2023-10-21T00:06:27+0800",
+ "date": "2023-10-29T22:53:15-0500",
  "dirty": false,
  "error": null,
- "full-revisionid": "a6bf734cc31d019c37fa1e33f48c0e612d3733b6",
- "version": "0.5.4"
+ "full-revisionid": "148d25a8e0f876dd180e2665c9465608e0c1c930",
+ "version": "0.5.6"
 }
 '''  # END VERSION_JSON
 

xinference/constants.py

@@ -15,7 +15,16 @@
 import os
 from pathlib import Path
 
-XINFERENCE_HOME = str(Path.home() / ".xinference")
+XINFERENCE_ENV_ENDPOINT = "XINFERENCE_ENDPOINT"
+XINFERENCE_ENV_MODEL_SRC = "XINFERENCE_MODEL_SRC"
+XINFERENCE_ENV_HOME_PATH = "XINFERENCE_HOME"
+
+
+def get_xinference_home():
+    return os.environ.get(XINFERENCE_ENV_HOME_PATH, str(Path.home() / ".xinference"))
+
+
+XINFERENCE_HOME = get_xinference_home()
 XINFERENCE_CACHE_DIR = os.path.join(XINFERENCE_HOME, "cache")
 XINFERENCE_MODEL_DIR = os.path.join(XINFERENCE_HOME, "model")
 XINFERENCE_LOG_DIR = os.path.join(XINFERENCE_HOME, "logs")
@@ -24,6 +33,3 @@ XINFERENCE_IMAGE_DIR = os.path.join(XINFERENCE_HOME, "image")
 XINFERENCE_DEFAULT_LOCAL_HOST = "127.0.0.1"
 XINFERENCE_DEFAULT_DISTRIBUTED_HOST = "0.0.0.0"
 XINFERENCE_DEFAULT_ENDPOINT_PORT = 9997
-
-XINFERENCE_ENV_ENDPOINT = "XINFERENCE_ENDPOINT"
-XINFERENCE_ENV_MODEL_SRC = "XINFERENCE_MODEL_SRC"

xinference/core/supervisor.py

@@ -436,6 +436,8 @@ class SupervisorActor(xo.Actor):
     async def report_worker_status(
         self, worker_address: str, status: Dict[str, ResourceStatus]
     ):
+        if worker_address not in self._worker_status:
+            logger.debug("Worker %s resources: %s", worker_address, status)
         self._worker_status[worker_address] = WorkerStatus(
             update_time=time.time(), status=status
         )

xinference/model/embedding/core.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import datetime
 import logging
 import os
 from typing import List, Optional, Tuple, Union, no_type_check
@@ -23,6 +22,7 @@ from pydantic import BaseModel
 from ...constants import XINFERENCE_CACHE_DIR
 from ...types import Embedding, EmbeddingData, EmbeddingUsage
 from ..core import ModelDescription
+from ..utils import valid_model_revision
 
 logger = logging.getLogger(__name__)
 
@@ -36,6 +36,21 @@ class EmbeddingModelSpec(BaseModel):
     model_revision: str
 
 
+class EmbeddingModelDescription(ModelDescription):
+    def __init__(self, model_spec: EmbeddingModelSpec):
+        self._model_spec = model_spec
+
+    def to_dict(self):
+        return {
+            "model_type": "embedding",
+            "model_name": self._model_spec.model_name,
+            "dimensions": self._model_spec.dimensions,
+            "max_tokens": self._model_spec.max_tokens,
+            "language": self._model_spec.language,
+            "model_revision": self._model_spec.model_revision,
+        }
+
+
 def cache(model_spec: EmbeddingModelSpec):
     # TODO: cache from uri
     from huggingface_hub import snapshot_download as hf_download
@@ -49,7 +64,7 @@ def cache(model_spec: EmbeddingModelSpec):
     if not os.path.exists(cache_dir):
         os.makedirs(cache_dir, exist_ok=True)
     meta_path = os.path.join(cache_dir, "__valid_download")
-    if os.path.exists(meta_path):
+    if valid_model_revision(meta_path, model_spec.model_revision):
         return cache_dir
 
     from_modelscope: bool = model_spec.model_id.startswith("Xorbits/")
@@ -75,8 +90,11 @@ def cache(model_spec: EmbeddingModelSpec):
             local_dir=cache_dir,
             local_dir_use_symlinks=True,
         )
-        with open(meta_path, "w") as f:
-            f.write(str(datetime.datetime.now()))
+    with open(meta_path, "w") as f:
+        import json
+
+        desc = EmbeddingModelDescription(model_spec)
+        json.dump(desc.to_dict(), f)
     return cache_dir
 
 
@@ -260,21 +278,6 @@ class EmbeddingModel:
         )
 
 
-class EmbeddingModelDescription(ModelDescription):
-    def __init__(self, model_spec: EmbeddingModelSpec):
-        self._model_spec = model_spec
-
-    def to_dict(self):
-        return {
-            "model_type": "embedding",
-            "model_name": self._model_spec.model_name,
-            "dimensions": self._model_spec.dimensions,
-            "max_tokens": self._model_spec.max_tokens,
-            "language": self._model_spec.language,
-            "model_revision": self._model_spec.model_revision,
-        }
-
-
 def match_embedding(model_name: str) -> EmbeddingModelSpec:
     from ..utils import download_from_modelscope
     from . import BUILTIN_EMBEDDING_MODELS, MODELSCOPE_EMBEDDING_MODELS

xinference/model/embedding/model_spec_modelscope.json

@@ -44,8 +44,8 @@
     "dimensions": 1024,
     "max_tokens": 512,
     "language": ["zh"],
-    "model_id": "AI-ModelScope/bge-large-zh",
-    "model_revision": "v1.0.0"
+    "model_id": "Xorbits/bge-large-zh",
+    "model_revision": "v0.0.1"
   },
   {
     "model_name": "bge-large-zh-noinstruct",
@@ -61,7 +61,7 @@
     "max_tokens": 512,
     "language": ["zh"],
     "model_id": "Xorbits/bge-base-zh",
-    "model_revision": "v0.0.1"
+    "model_revision": "v0.0.2"
   },
   {
     "model_name": "multilingual-e5-large",

xinference/model/llm/core.py

@@ -14,6 +14,7 @@
 
 import abc
 import logging
+import os
 import platform
 from abc import abstractmethod
 from typing import TYPE_CHECKING, Optional, Tuple
@@ -68,7 +69,14 @@ class LLM(abc.ABC):
     def _get_cuda_count():
         from xorbits._mars.resource import cuda_count
 
-        return cuda_count()
+        cuda_visible_devices = os.getenv("CUDA_VISIBLE_DEVICES", None)
+        if cuda_visible_devices is None:
+            return cuda_count()
+
+        if cuda_visible_devices == "-1":
+            return 0
+        else:
+            return len(cuda_visible_devices.split(","))
 
     @abstractmethod
     def load(self):

xinference/model/llm/llm_family.json

@@ -8,7 +8,6 @@
       "zh"
     ],
     "model_ability": [
-      "embed",
       "generate"
     ],
     "model_description": "Baichuan is an open-source Transformer based LLM that is trained on both Chinese and English data.",
@@ -68,7 +67,6 @@
       "zh"
     ],
     "model_ability": [
-      "embed",
       "chat"
     ],
     "model_description": "Baichuan-chat is a fine-tuned version of the Baichuan LLM, specializing in chatting.",
@@ -108,7 +106,6 @@
       "en"
     ],
     "model_ability": [
-      "embed",
       "chat"
     ],
     "model_description": "WizardLM is an open-source LLM trained by fine-tuning LLaMA with Evol-Instruct.",
@@ -176,7 +173,6 @@
       "en"
     ],
     "model_ability": [
-      "embed",
       "chat"
     ],
     "model_description": "Vicuna is an open-source LLM trained by fine-tuning LLaMA on data collected from ShareGPT.",
@@ -300,7 +296,6 @@
       "en"
     ],
     "model_ability": [
-      "embed",
       "chat"
     ],
     "model_description": "Orca is an LLM trained by fine-tuning LLaMA on explanation traces obtained from GPT-4.",
@@ -364,7 +359,6 @@
       "zh"
     ],
     "model_ability": [
-      "embed",
       "chat"
     ],
     "model_description": "ChatGLM is an open-source General Language Model (GLM) based LLM trained on both Chinese and English data.",
@@ -413,7 +407,6 @@
       "zh"
     ],
     "model_ability": [
-      "embed",
       "chat"
     ],
     "model_description": "ChatGLM2 is the second generation of ChatGLM, still open-source and trained on Chinese and English data.",
@@ -462,7 +455,6 @@
       "zh"
     ],
     "model_ability": [
-      "embed",
       "chat"
     ],
     "model_description": "ChatGLM2-32k is a special version of ChatGLM2, with a context window of 32k tokens instead of 8k.",
@@ -489,6 +481,74 @@
       "intra_message_sep": "\n\n"
     }
   },
+  {
+    "version": 1,
+    "context_length": 8192,
+    "model_name": "chatglm3",
+    "model_lang": [
+      "en",
+      "zh"
+    ],
+    "model_ability": [
+      "chat"
+    ],
+    "model_description": "ChatGLM3 is the third generation of ChatGLM, still open-source and trained on Chinese and English data.",
+    "model_specs": [
+      {
+        "model_format": "pytorch",
+        "model_size_in_billions": 6,
+        "quantizations": [
+          "4-bit",
+          "8-bit",
+          "none"
+        ],
+        "model_id": "THUDM/chatglm3-6b",
+        "model_revision": "fc3235f807ef5527af598c05f04f2ffd17f48bab"
+      }
+    ],
+    "prompt_style": {
+      "style_name": "CHATGLM3",
+      "system_prompt": "",
+      "roles": [
+        "user",
+        "assistant"
+      ]
+    }
+  },
+  {
+    "version": 1,
+    "context_length": 32768,
+    "model_name": "chatglm3-32k",
+    "model_lang": [
+      "en",
+      "zh"
+    ],
+    "model_ability": [
+      "chat"
+    ],
+    "model_description": "ChatGLM3 is the third generation of ChatGLM, still open-source and trained on Chinese and English data.",
+    "model_specs": [
+      {
+        "model_format": "pytorch",
+        "model_size_in_billions": 6,
+        "quantizations": [
+          "4-bit",
+          "8-bit",
+          "none"
+        ],
+        "model_id": "THUDM/chatglm3-6b-32k",
+        "model_revision": "339f17ff464d47b5077527c2b34e80a7719ede3e"
+      }
+    ],
+    "prompt_style": {
+      "style_name": "CHATGLM3",
+      "system_prompt": "",
+      "roles": [
+        "user",
+        "assistant"
+      ]
+    }
+  },
   {
     "version": 1,
     "context_length": 4096,
@@ -497,7 +557,6 @@
       "en"
     ],
     "model_ability": [
-      "embed",
       "chat"
     ],
     "model_description": "Llama-2-Chat is a fine-tuned version of the Llama-2 LLM, specializing in chatting.",
@@ -613,6 +672,9 @@
       "inter_message_sep": " </s><s>",
       "stop_token_ids": [
         2
+      ],
+      "stop": [
+        "</s>"
       ]
     }
   },
@@ -624,7 +686,6 @@
       "en"
     ],
     "model_ability": [
-      "embed",
       "generate"
     ],
     "model_description": "Llama-2 is the second generation of Llama, open-source and trained on a larger amount of data.",
@@ -738,7 +799,6 @@
       "en"
     ],
     "model_ability": [
-      "embed",
       "generate"
     ],
     "model_description": "Opt is an open-source, decoder-only, Transformer based LLM that was designed to replicate GPT-3.",
@@ -764,7 +824,6 @@
       "en"
     ],
     "model_ability": [
-      "embed",
       "generate"
     ],
     "model_description": "Falcon is an open-source Transformer based LLM trained on the RefinedWeb dataset.",
@@ -801,7 +860,6 @@
       "en"
     ],
     "model_ability": [
-      "embed",
       "chat"
     ],
     "model_description": "Falcon-instruct is a fine-tuned version of the Falcon LLM, specializing in chatting.",
@@ -865,7 +923,6 @@
       "en"
     ],
     "model_ability": [
-      "embed",
       "generate"
     ],
     "model_description": "Starcoderplus is an open-source LLM trained by fine-tuning Starcoder on RedefinedWeb and StarCoderData datasets.",
@@ -892,7 +949,6 @@
       "en"
     ],
     "model_ability": [
-      "embed",
       "chat"
     ],
     "model_description": "Starchat-beta is a fine-tuned version of the Starcoderplus LLM, specializing in coding assistance.",
@@ -932,7 +988,6 @@
       "zh"
     ],
     "model_ability": [
-      "embed",
       "chat"
     ],
     "model_description": "Qwen-chat is a fine-tuned version of the Qwen LLM trained with alignment techniques, specializing in chatting.",
@@ -1021,7 +1076,6 @@
       "zh"
     ],
     "model_ability": [
-      "embed",
       "generate"
     ],
     "model_description": "InternLM is a Transformer-based LLM that is trained on both Chinese and English data, focusing on practical scenarios.",
@@ -1048,7 +1102,6 @@
       "zh"
     ],
     "model_ability": [
-      "embed",
       "chat"
     ],
     "model_description": "Internlm-chat is a fine-tuned version of the Internlm LLM, specializing in chatting.",
@@ -1092,7 +1145,6 @@
       "zh"
     ],
     "model_ability": [
-      "embed",
       "generate"
     ],
     "model_description": "Pre-trained on over 2.3T Tokens containing high-quality English, Chinese, and code data.",
@@ -1119,7 +1171,6 @@
       "zh"
     ],
     "model_ability": [
-      "embed",
       "chat"
     ],
     "model_description": "Pre-trained on over 2.3T Tokens containing high-quality English, Chinese, and code data. The Chat version has undergone SFT and RLHF training.",
@@ -1162,7 +1213,6 @@
       "en"
     ],
     "model_ability": [
-      "embed",
       "chat"
     ],
     "model_description": "Vicuna is an open-source LLM trained by fine-tuning LLaMA on data collected from ShareGPT.",
@@ -1209,7 +1259,6 @@
       "en"
     ],
     "model_ability": [
-      "embed",
       "chat"
     ],
     "model_description": "Vicuna-v1.5-16k is a special version of Vicuna-v1.5, with a context window of 16k tokens instead of 4k.",
@@ -1256,7 +1305,6 @@
       "en"
     ],
     "model_ability": [
-      "embed",
       "chat"
     ],
     "model_description": "WizardMath is an open-source LLM trained by fine-tuning Llama2 with Evol-Instruct, specializing in math.",
@@ -1636,6 +1684,9 @@
       "inter_message_sep": " </s><s>",
       "stop_token_ids": [
         2
+      ],
+      "stop": [
+        "</s>"
       ]
     }
   },
@@ -1648,7 +1699,6 @@
       "zh"
     ],
     "model_ability": [
-      "embed",
       "chat"
     ],
     "model_description": "Baichuan2-chat is a fine-tuned version of the Baichuan LLM, specializing in chatting.",
@@ -1700,7 +1750,6 @@
       "zh"
     ],
     "model_ability": [
-      "embed",
       "generate"
     ],
     "model_description": "Baichuan2 is an open-source Transformer based LLM that is trained on both Chinese and English data.",
@@ -1947,6 +1996,9 @@
       "inter_message_sep": " </s><s>",
       "stop_token_ids": [
         2
+      ],
+      "stop": [
+        "</s>"
       ]
     }
   },
@@ -1958,7 +2010,6 @@
       "en"
     ],
     "model_ability": [
-      "generate",
       "chat"
     ],
     "model_specs": [

xinference/model/llm/llm_family.py

@@ -23,7 +23,12 @@ from pydantic import BaseModel, Field
 from typing_extensions import Annotated, Literal
 
 from ...constants import XINFERENCE_CACHE_DIR, XINFERENCE_MODEL_DIR
-from ..utils import download_from_modelscope, retry_download, symlink_local_file
+from ..utils import (
+    download_from_modelscope,
+    retry_download,
+    symlink_local_file,
+    valid_model_revision,
+)
 from . import LLM
 
 logger = logging.getLogger(__name__)
@@ -403,6 +408,7 @@ def _skip_download(
     cache_dir: str,
     model_format: str,
     model_hub: str,
+    model_revision: Optional[str],
     quantization: Optional[str] = None,
 ) -> bool:
     if model_format == "pytorch":
@@ -414,7 +420,7 @@ def _skip_download(
                 cache_dir, model_format, "modelscope", quantization
             ),
         }
-        if os.path.exists(model_hub_to_meta_path[model_hub]):
+        if valid_model_revision(model_hub_to_meta_path[model_hub], model_revision):
             logger.info(f"Cache {cache_dir} exists")
             return True
         else:
@@ -439,7 +445,9 @@ def _generate_meta_file(
     llm_spec: "LLMSpecV1",
     quantization: Optional[str] = None,
 ):
-    assert not os.path.exists(meta_path), f"meta path {meta_path} should not exists"
+    assert not valid_model_revision(
+        meta_path, llm_spec.model_revision
+    ), f"meta file {meta_path} should not be valid"
     with open(meta_path, "w") as f:
         import json
 
@@ -462,7 +470,11 @@ def cache_from_modelscope(
 
     cache_dir = _get_cache_dir(llm_family, llm_spec)
     if _skip_download(
-        cache_dir, llm_spec.model_format, llm_spec.model_hub, quantization
+        cache_dir,
+        llm_spec.model_format,
+        llm_spec.model_hub,
+        llm_spec.model_revision,
+        quantization,
     ):
         return cache_dir
 
@@ -519,7 +531,11 @@ def cache_from_huggingface(
 
     cache_dir = _get_cache_dir(llm_family, llm_spec)
     if _skip_download(
-        cache_dir, llm_spec.model_format, llm_spec.model_hub, quantization
+        cache_dir,
+        llm_spec.model_format,
+        llm_spec.model_hub,
+        llm_spec.model_revision,
+        quantization,
     ):
         return cache_dir
 

xinference/model/llm/llm_family_modelscope.json

@@ -7,7 +7,6 @@
       "en"
     ],
     "model_ability": [
-      "embed",
       "chat"
     ],
     "model_description": "Llama-2-Chat is a fine-tuned version of the Llama-2 LLM, specializing in chatting.",
@@ -119,7 +118,6 @@
       "zh"
     ],
     "model_ability": [
-      "embed",
       "chat"
     ],
     "model_description": "Baichuan2-chat is a fine-tuned version of the Baichuan LLM, specializing in chatting.",
@@ -173,7 +171,6 @@
       "zh"
     ],
     "model_ability": [
-      "embed",
       "generate"
     ],
     "model_description": "Baichuan2 is an open-source Transformer based LLM that is trained on both Chinese and English data.",
@@ -213,7 +210,6 @@
       "zh"
     ],
     "model_ability": [
-      "embed",
       "chat"
     ],
     "model_description": "ChatGLM2 is the second generation of ChatGLM, still open-source and trained on Chinese and English data.",
@@ -250,7 +246,6 @@
       "zh"
     ],
     "model_ability": [
-      "embed",
       "chat"
     ],
     "model_description": "ChatGLM2-32k is a special version of ChatGLM2, with a context window of 32k tokens instead of 8k.",
@@ -278,6 +273,76 @@
       "intra_message_sep": "\n\n"
     }
   },
+  {
+    "version": 1,
+    "context_length": 8192,
+    "model_name": "chatglm3",
+    "model_lang": [
+      "en",
+      "zh"
+    ],
+    "model_ability": [
+      "chat"
+    ],
+    "model_description": "ChatGLM3 is the third generation of ChatGLM, still open-source and trained on Chinese and English data.",
+    "model_specs": [
+      {
+        "model_format": "pytorch",
+        "model_size_in_billions": 6,
+        "quantizations": [
+          "4-bit",
+          "8-bit",
+          "none"
+        ],
+        "model_hub": "modelscope",
+        "model_id": "ZhipuAI/chatglm3-6b",
+        "model_revision": "v1.0.0"
+      }
+    ],
+    "prompt_style": {
+      "style_name": "CHATGLM3",
+      "system_prompt": "",
+      "roles": [
+        "user",
+        "assistant"
+      ]
+    }
+  },
+  {
+    "version": 1,
+    "context_length": 32768,
+    "model_name": "chatglm3-32k",
+    "model_lang": [
+      "en",
+      "zh"
+    ],
+    "model_ability": [
+      "chat"
+    ],
+    "model_description": "ChatGLM3 is the third generation of ChatGLM, still open-source and trained on Chinese and English data.",
+    "model_specs": [
+      {
+        "model_format": "pytorch",
+        "model_size_in_billions": 6,
+        "quantizations": [
+          "4-bit",
+          "8-bit",
+          "none"
+        ],
+        "model_hub": "modelscope",
+        "model_id": "ZhipuAI/chatglm3-6b-32k",
+        "model_revision": "v1.0.0"
+      }
+    ],
+    "prompt_style": {
+      "style_name": "CHATGLM3",
+      "system_prompt": "",
+      "roles": [
+        "user",
+        "assistant"
+      ]
+    }
+  },
   {
     "version": 1,
     "context_length": 8192,
@@ -287,7 +352,6 @@
       "zh"
     ],
     "model_ability": [
-      "embed",
       "generate"
     ],
     "model_description": "InternLM is a Transformer-based LLM that is trained on both Chinese and English data, focusing on practical scenarios.",
@@ -315,7 +379,6 @@
       "zh"
     ],
     "model_ability": [
-      "embed",
       "chat"
     ],
     "model_description": "Internlm-chat is a fine-tuned version of the Internlm LLM, specializing in chatting.",
@@ -360,7 +423,6 @@
       "zh"
     ],
     "model_ability": [
-      "embed",
       "generate"
     ],
     "model_description": "Pre-trained on over 2.3T Tokens containing high-quality English, Chinese, and code data.",
@@ -388,7 +450,6 @@
       "zh"
     ],
     "model_ability": [
-      "embed",
       "chat"
     ],
     "model_description": "Pre-trained on over 2.3T Tokens containing high-quality English, Chinese, and code data. The Chat version has undergone SFT and RLHF training.",
@@ -432,7 +493,6 @@
       "en"
     ],
     "model_ability": [
-      "generate",
       "chat"
     ],
     "model_specs": [