xinference 1.7.0__py3-none-any.whl → 1.7.0.post1__py3-none-any.whl

xinference/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-06-13T18:51:07+0800",
+ "date": "2025-06-14T01:13:55+0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "a362dba7334ef08c758bbc4a3d4904fe53cefe78",
- "version": "1.7.0"
+ "full-revisionid": "da2040e54c18c80ae88c64608de0081fa6df54c4",
+ "version": "1.7.0.post1"
 }
 '''  # END VERSION_JSON
 

xinference/model/llm/llm_family.json

@@ -6160,7 +6160,7 @@
         "quantizations": [
           "none"
         ],
-        "model_id": "openbmb/MiniCPM4-0.5B"
+        "model_id": "JunHowie/MiniCPM4-0.5B"
       },
       {
         "model_format": "pytorch",
@@ -6168,7 +6168,7 @@
         "quantizations": [
           "none"
         ],
-        "model_id": "openbmb/MiniCPM4-8B"
+        "model_id": "JunHowie/MiniCPM4-8B"
       },
       {
         "model_format": "mlx",

xinference/model/llm/llm_family_modelscope.json

@@ -4295,7 +4295,7 @@
         "quantizations": [
           "none"
         ],
-        "model_id": "OpenBMB/MiniCPM4-0.5B",
+        "model_id": "JunHowie/MiniCPM4-0.5B",
         "model_hub": "modelscope"
       },
       {
@@ -4304,7 +4304,7 @@
         "quantizations": [
           "none"
         ],
-        "model_id": "OpenBMB/MiniCPM4-8B",
+        "model_id": "JunHowie/MiniCPM4-8B",
         "model_hub": "modelscope"
       },
       {

xinference/model/rerank/core.py

@@ -14,6 +14,7 @@
 
 import gc
 import importlib
+import importlib.util
 import logging
 import os
 import threading
@@ -251,12 +252,21 @@ class RerankModel:
             tokenizer = AutoTokenizer.from_pretrained(
                 self._model_path, padding_side="left"
             )
+            flash_attn_installed = importlib.util.find_spec("flash_attn") is not None
+            model_kwargs = {"device_map": "auto"}
+            if flash_attn_installed:
+                model_kwargs["attn_implementation"] = "flash_attention_2"
+                model_kwargs["torch_dtype"] = torch.float16
             model = self._model = AutoModelForCausalLM.from_pretrained(
-                self._model_path
+                self._model_path, **model_kwargs
             ).eval()
             max_length = getattr(self._model_spec, "max_tokens")
 
-            prefix = '<|im_start|>system\nJudge whether the Document meets the requirements based on the Query and the Instruct provided. Note that the answer can only be "yes" or "no".<|im_end|>\n<|im_start|>user\n'
+            prefix = (
+                "<|im_start|>system\nJudge whether the Document meets the requirements based on the Query "
+                'and the Instruct provided. Note that the answer can only be "yes" or "no".'
+                "<|im_end|>\n<|im_start|>user\n"
+            )
             suffix = "<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n"
             prefix_tokens = tokenizer.encode(prefix, add_special_tokens=False)
             suffix_tokens = tokenizer.encode(suffix, add_special_tokens=False)

{xinference-1.7.0.dist-info → xinference-1.7.0.post1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xinference
-Version: 1.7.0
+Version: 1.7.0.post1
 Summary: Model Serving Made Easy
 Home-page: https://github.com/xorbitsai/inference
 Author: Qin Xuye

{xinference-1.7.0.dist-info → xinference-1.7.0.post1.dist-info}/RECORD

@@ -1,6 +1,6 @@
 xinference/__init__.py,sha256=nmTTrYbIpj964ZF6ojtgOM7E85JBOj1EyQbmYjbj1jw,915
 xinference/_compat.py,sha256=YF-lS6tX06zkFi2oFS0yq_LBn4hX_8u0Ft0vKxGALwA,4238
-xinference/_version.py,sha256=cTB6jB2VXLWZ_Heth-iG167tHE-bv24BvW3eaShba2k,497
+xinference/_version.py,sha256=FLTvXc3gsm40inNtWJwpd9lC2ODsO7OetP5N4-FyECo,503
 xinference/conftest.py,sha256=vETDpRBVIlWbWi7OTwf7og89U25KyYGyI7yPIB3O8N8,9564
 xinference/constants.py,sha256=-QqMRJo1MvTgnLmRZLF3DVj3PMXUzg1rV-XIsuN_nGI,4319
 xinference/device_utils.py,sha256=ELsqvnjvz9wYthTyQFzKSV4mZsaASz6hj_IsfMmfMWc,4447
@@ -96,10 +96,10 @@ xinference/model/image/stable_diffusion/core.py,sha256=sTceL97g4qXz9ukPGIBmPkRGg
 xinference/model/image/stable_diffusion/mlx.py,sha256=GZsozzGB04NfHAdU9MI6gwWE1t_A-s_Ddn_ic8DlkKQ,7476
 xinference/model/llm/__init__.py,sha256=Pm8hxP3E1m9-H-JaicIsUCvZ77uD0ClL0KvqcreO1aY,9137
 xinference/model/llm/core.py,sha256=S8Ys69u6NWRjjHksBlGG50RPwHIDSQsUBiK-wRunfj0,10236
-xinference/model/llm/llm_family.json,sha256=VU3khmH9X8_aSDhJ5Nfr7u_ZgEd0wUZvMCjXaCBvnWE,412320
+xinference/model/llm/llm_family.json,sha256=oL4aja5uZHaDetGKpEt8v0ZYG_GPQnpnMeGnA9Zyx10,412322
 xinference/model/llm/llm_family.py,sha256=Nex1zvegUH1TvGjy8yILYiehY2QFOziRASGBlAa7wPg,44510
 xinference/model/llm/llm_family_csghub.json,sha256=OYve65CgLVHdyTC3UMe7vqhFwC09OR7XhzDsVEpkJhY,6908
-xinference/model/llm/llm_family_modelscope.json,sha256=rOu1ouDLRMUNpuRQkB4QPRjaQdp5ScneXJAaUbVY24Y,357639
+xinference/model/llm/llm_family_modelscope.json,sha256=7Zs4rW07xpvWfT_3bE6cjDsn_xu0wiXGE7JJD6yUjws,357641
 xinference/model/llm/llm_family_openmind_hub.json,sha256=sAWzttUasckGzNdcBHhdxvXB893tBhnm__w_1hr6JXw,58243
 xinference/model/llm/memory.py,sha256=GLNmXBI-AtMbuaJfEf50fnhN4rdbOZjLyT6L_Vjqa5g,10206
 xinference/model/llm/reasoning_parser.py,sha256=L9BGzsQiTw8QY6BtQfdEGJmCL_RpnYR1MXBkczOS6SU,17184
@@ -151,7 +151,7 @@ xinference/model/llm/vllm/xavier/transfer.py,sha256=2IbaiAhRnJMwa9qsMa5bowngqfjx
 xinference/model/llm/vllm/xavier/test/__init__.py,sha256=CyLLkbImZouAk4lePIgKXT4WQoqyauIEwdqea5IOUVU,581
 xinference/model/llm/vllm/xavier/test/test_xavier.py,sha256=lSfo4caaLdWRh0diJ1jd_sY85GV-_9BT40ySs_nirJQ,4695
 xinference/model/rerank/__init__.py,sha256=yxxv4OOHCl3UUOLENP45Eghf_dEsRaCkUUcW5bth_NE,3141
-xinference/model/rerank/core.py,sha256=j5AP4f1KFkDUUPLcjywKIHe4jfysJFlgpW1N1fEckNk,18996
+xinference/model/rerank/core.py,sha256=mHgPvnhkD9CO8Lp_k5rUI9OmmrTKe2NPeKL1pHDOk9A,19411
 xinference/model/rerank/custom.py,sha256=wPKF3bHbGap9dHz9yYvXMXhozh4hRzS78RQijqvaRq8,3846
 xinference/model/rerank/model_spec.json,sha256=GC0aD3vcMMBaPoYn74AL4FlkgDugX3z56eWNBfqqRhk,2705
 xinference/model/rerank/model_spec_modelscope.json,sha256=lQ48VuQwokc564yKjYqCEPspW5EJryIJXtuo6crXKlQ,2211
@@ -15778,9 +15778,9 @@ xinference/web/ui/src/locales/en.json,sha256=R7JpAE_lXcF4j1ryOghE0_B8Y2Q0zIrax_v
 xinference/web/ui/src/locales/ja.json,sha256=nh-gFRtTpZAmTJNBa00cYfv6ftpR2FcI4ZmQZ0Y-ezc,11454
 xinference/web/ui/src/locales/ko.json,sha256=MJOPJFagJdjgLDb5g3kaSBEWmH5zF03QeLWhayGo_iQ,10333
 xinference/web/ui/src/locales/zh.json,sha256=7NRqta9ipMdoe19Zdp0-MGqbiiTxDoRopM1o2EOWb54,9424
-xinference-1.7.0.dist-info/licenses/LICENSE,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
-xinference-1.7.0.dist-info/METADATA,sha256=iIVl2mTLWs8_vkEP5uDGS96ulOZEU2UdjW7uZBSoI8k,26050
-xinference-1.7.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-xinference-1.7.0.dist-info/entry_points.txt,sha256=-lDyyzqWMFQF0Rgm7VxBNz0V-bMBMQLRR3pvQ-Y8XTY,226
-xinference-1.7.0.dist-info/top_level.txt,sha256=L1rQt7pl6m8tmKXpWVHzP-GtmzAxp663rXxGE7qnK00,11
-xinference-1.7.0.dist-info/RECORD,,
+xinference-1.7.0.post1.dist-info/licenses/LICENSE,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
+xinference-1.7.0.post1.dist-info/METADATA,sha256=DAz2hgr8ToiguCmFIiTH13uFar15QjhyyQ7LCj9FFxo,26056
+xinference-1.7.0.post1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+xinference-1.7.0.post1.dist-info/entry_points.txt,sha256=-lDyyzqWMFQF0Rgm7VxBNz0V-bMBMQLRR3pvQ-Y8XTY,226
+xinference-1.7.0.post1.dist-info/top_level.txt,sha256=L1rQt7pl6m8tmKXpWVHzP-GtmzAxp663rXxGE7qnK00,11
+xinference-1.7.0.post1.dist-info/RECORD,,