xinference 1.5.0.post2__py3-none-any.whl → 1.6.0__py3-none-any.whl

xinference/model/llm/llm_family.py

@@ -72,8 +72,10 @@ class LlamaCppLLMSpecV1(BaseModel):
     model_hub: str = "huggingface"
     model_uri: Optional[str]
     model_revision: Optional[str]
+    # for MOE model, illustrates the activated model size
+    activated_size_in_billions: Optional[Union[str, int]]
 
-    @validator("model_size_in_billions", pre=False)
+    @validator("model_size_in_billions", "activated_size_in_billions", pre=False)
     def validate_model_size_with_radix(cls, v: object) -> object:
         if isinstance(v, str):
             if (
@@ -94,8 +96,10 @@ class PytorchLLMSpecV1(BaseModel):
     model_hub: str = "huggingface"
     model_uri: Optional[str]
     model_revision: Optional[str]
+    # for MOE model, illustrates the activated model size
+    activated_size_in_billions: Optional[Union[str, int]]
 
-    @validator("model_size_in_billions", pre=False)
+    @validator("model_size_in_billions", "activated_size_in_billions", pre=False)
     def validate_model_size_with_radix(cls, v: object) -> object:
         if isinstance(v, str):
             if (
@@ -116,8 +120,10 @@ class MLXLLMSpecV1(BaseModel):
     model_hub: str = "huggingface"
     model_uri: Optional[str]
     model_revision: Optional[str]
+    # for MOE model, illustrates the activated model size
+    activated_size_in_billions: Optional[Union[str, int]]
 
-    @validator("model_size_in_billions", pre=False)
+    @validator("model_size_in_billions", "activated_size_in_billions", pre=False)
     def validate_model_size_with_radix(cls, v: object) -> object:
         if isinstance(v, str):
             if (
@@ -136,7 +142,15 @@ class LLMFamilyV1(BaseModel):
     model_lang: List[str]
     model_ability: List[
         Literal[
-            "embed", "generate", "chat", "tools", "vision", "audio", "omni", "reasoning"
+            "embed",
+            "generate",
+            "chat",
+            "tools",
+            "vision",
+            "audio",
+            "omni",
+            "reasoning",
+            "hybrid",
         ]
     ]
     model_description: Optional[str]
@@ -370,6 +384,53 @@ def cache_from_uri(
         raise ValueError(f"Unsupported URL scheme: {src_scheme}")
 
 
+def cache_model_tokenizer_and_config(
+    llm_family: LLMFamilyV1,
+    llm_spec: "LLMSpecV1",
+) -> str:
+    """
+    Download model config.json and tokenizers only
+    """
+    cache_dir = _get_cache_dir_for_model_mem(llm_family, llm_spec, "tokenizer_config")
+    os.makedirs(cache_dir, exist_ok=True)
+    if llm_spec.model_hub == "huggingface":
+        from huggingface_hub import snapshot_download
+
+        download_dir = retry_download(
+            snapshot_download,
+            llm_family.model_name,
+            {
+                "model_size": llm_spec.model_size_in_billions,
+                "model_format": llm_spec.model_format,
+            },
+            llm_spec.model_id,
+            revision=llm_spec.model_revision,
+            allow_patterns=["tokenizer*", "config.json"],
+            local_dir=cache_dir,
+        )
+    elif llm_spec.model_hub == "modelscope":
+        from modelscope.hub.snapshot_download import snapshot_download
+
+        download_dir = retry_download(
+            snapshot_download,
+            llm_family.model_name,
+            {
+                "model_size": llm_spec.model_size_in_billions,
+                "model_format": llm_spec.model_format,
+            },
+            llm_spec.model_id,
+            revision=llm_spec.model_revision,
+            allow_patterns=["tokenizer*", "config.json"],
+            local_dir=cache_dir,
+        )
+    else:
+        raise NotImplementedError(
+            f"Does not support download config.json and "
+            f"tokenizer related files via {llm_spec.model_hub}"
+        )
+    return download_dir
+
+
 def cache_model_config(
     llm_family: LLMFamilyV1,
     llm_spec: "LLMSpecV1",
@@ -377,7 +438,7 @@ def cache_model_config(
     """Download model config.json into cache_dir,
     returns local filepath
     """
-    cache_dir = _get_cache_dir_for_model_mem(llm_family, llm_spec)
+    cache_dir = _get_cache_dir_for_model_mem(llm_family, llm_spec, "model_mem")
     config_file = os.path.join(cache_dir, "config.json")
     if not os.path.islink(config_file) and not os.path.exists(config_file):
         os.makedirs(cache_dir, exist_ok=True)
@@ -400,10 +461,13 @@ def cache_model_config(
 def _get_cache_dir_for_model_mem(
     llm_family: LLMFamilyV1,
     llm_spec: "LLMSpecV1",
+    category: str,
     create_if_not_exist=True,
 ):
     """
-    For cal-model-mem only. (might called from supervisor / cli)
+    Get file dir for special usage, like `cal-model-mem` and download partial files for
+
+    e.g. for cal-model-mem, (might called from supervisor / cli)
     Temporary use separate dir from worker's cache_dir, due to issue of different style of symlink.
     """
     quant_suffix = ""
@@ -418,7 +482,7 @@ def _get_cache_dir_for_model_mem(
     if quant_suffix:
         cache_dir_name += f"-{quant_suffix}"
     cache_dir = os.path.realpath(
-        os.path.join(XINFERENCE_CACHE_DIR, "model_mem", cache_dir_name)
+        os.path.join(XINFERENCE_CACHE_DIR, category, cache_dir_name)
     )
     if create_if_not_exist and not os.path.exists(cache_dir):
         os.makedirs(cache_dir, exist_ok=True)

xinference/model/llm/llm_family_csghub.json

@@ -17,8 +17,6 @@
         "model_format": "pytorch",
         "model_size_in_billions": "0_5",
         "quantizations": [
-          "4-bit",
-          "8-bit",
           "none"
         ],
         "model_id": "Qwen/Qwen2-0.5B-Instruct",
@@ -54,35 +52,5 @@
       "<|im_start|>",
       "<|im_end|>"
     ]
-  },
-  {
-    "version": 1,
-    "context_length": 32768,
-    "model_name": "csg-wukong-chat-v0.1",
-    "model_lang": [
-      "en"
-    ],
-    "model_ability": [
-      "chat"
-    ],
-    "model_description": "csg-wukong-1B is a 1 billion-parameter small language model(SLM) pretrained on 1T tokens.",
-    "model_specs": [
-      {
-        "model_format": "pytorch",
-        "model_size_in_billions": 1,
-        "quantizations": [
-          "none"
-        ],
-        "model_id": "OpenCSG/csg-wukong-1B-chat-v0.1",
-        "model_hub": "csghub"
-      }
-    ],
-    "chat_template": "{% for item in messages %}{% if loop.first and item['role'] == 'system' %}{{ item['content'] + '\n' }}{% elif loop.first %}{{ '<|system|>\nYou are a creative super artificial intelligence assistant, possessing all the knowledge of humankind. Your name is csg-wukong, developed by OpenCSG. You need to understand and infer the true intentions of users based on the topics discussed in the chat history, and respond to user questions correctly as required. You enjoy responding to users with accurate and insightful answers. Please pay attention to the appropriate style and format when replying, try to avoid repetitive words and sentences, and keep your responses as concise and profound as possible. You carefully consider the context of the discussion when replying to users. When the user says \"continue,\" please proceed with the continuation of the previous assistant\\'s response.</s>\n' }}{% endif %}{% if item['role'] == 'user' %}{{ '<|user|>\n' + item['content'] + '</s>\n' }}{% elif item['role'] == 'assistant' %}{{ '<|assistant|>\n' + item['content'] + '</s>\n' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% endif %}",
-    "stop_token_ids": [
-      2
-    ],
-    "stop": [
-      "</s>"
-    ]
   }
 ]