xinference 0.14.4.post1__py3-none-any.whl → 0.15.1__py3-none-any.whl

xinference/core/image_interface.py

@@ -73,13 +73,17 @@ class ImageInterface:
         return interface
 
     def text2image_interface(self) -> "gr.Blocks":
+        from ..model.image.stable_diffusion.core import SAMPLING_METHODS
+
         def text_generate_image(
             prompt: str,
             n: int,
             size_width: int,
             size_height: int,
+            guidance_scale: int,
             num_inference_steps: int,
             negative_prompt: Optional[str] = None,
+            sampler_name: Optional[str] = None,
         ) -> PIL.Image.Image:
             from ..client import RESTfulClient
 
@@ -89,16 +93,20 @@ class ImageInterface:
             assert isinstance(model, RESTfulImageModelHandle)
 
             size = f"{int(size_width)}*{int(size_height)}"
+            guidance_scale = None if guidance_scale == -1 else guidance_scale  # type: ignore
             num_inference_steps = (
                 None if num_inference_steps == -1 else num_inference_steps  # type: ignore
             )
+            sampler_name = None if sampler_name == "default" else sampler_name
 
             response = model.text_to_image(
                 prompt=prompt,
                 n=n,
                 size=size,
                 num_inference_steps=num_inference_steps,
+                guidance_scale=guidance_scale,
                 negative_prompt=negative_prompt,
+                sampler_name=sampler_name,
                 response_format="b64_json",
             )
 
@@ -132,9 +140,16 @@ class ImageInterface:
                     n = gr.Number(label="Number of Images", value=1)
                     size_width = gr.Number(label="Width", value=1024)
                     size_height = gr.Number(label="Height", value=1024)
+                with gr.Row():
+                    guidance_scale = gr.Number(label="Guidance scale", value=-1)
                     num_inference_steps = gr.Number(
                         label="Inference Step Number", value=-1
                     )
+                    sampler_name = gr.Dropdown(
+                        choices=SAMPLING_METHODS,
+                        value="default",
+                        label="Sampling method",
+                    )
 
                 with gr.Column():
                     image_output = gr.Gallery()
@@ -146,8 +161,10 @@ class ImageInterface:
                     n,
                     size_width,
                     size_height,
+                    guidance_scale,
                     num_inference_steps,
                     negative_prompt,
+                    sampler_name,
                 ],
                 outputs=image_output,
             )
@@ -155,6 +172,8 @@ class ImageInterface:
         return text2image_vl_interface
 
     def image2image_interface(self) -> "gr.Blocks":
+        from ..model.image.stable_diffusion.core import SAMPLING_METHODS
+
         def image_generate_image(
             prompt: str,
             negative_prompt: str,
@@ -164,6 +183,7 @@ class ImageInterface:
             size_height: int,
             num_inference_steps: int,
             padding_image_to_multiple: int,
+            sampler_name: Optional[str] = None,
         ) -> PIL.Image.Image:
             from ..client import RESTfulClient
 
@@ -180,6 +200,7 @@ class ImageInterface:
                 None if num_inference_steps == -1 else num_inference_steps  # type: ignore
             )
             padding_image_to_multiple = None if padding_image_to_multiple == -1 else padding_image_to_multiple  # type: ignore
+            sampler_name = None if sampler_name == "default" else sampler_name
 
             bio = io.BytesIO()
             image.save(bio, format="png")
@@ -193,6 +214,7 @@ class ImageInterface:
                 response_format="b64_json",
                 num_inference_steps=num_inference_steps,
                 padding_image_to_multiple=padding_image_to_multiple,
+                sampler_name=sampler_name,
             )
 
             images = []
@@ -233,6 +255,11 @@ class ImageInterface:
                     padding_image_to_multiple = gr.Number(
                         label="Padding image to multiple", value=-1
                     )
+                    sampler_name = gr.Dropdown(
+                        choices=SAMPLING_METHODS,
+                        value="default",
+                        label="Sampling method",
+                    )
 
                 with gr.Row():
                     with gr.Column(scale=1):
@@ -251,6 +278,7 @@ class ImageInterface:
                     size_height,
                     num_inference_steps,
                     padding_image_to_multiple,
+                    sampler_name,
                 ],
                 outputs=output_gallery,
             )

xinference/core/model.py

@@ -19,6 +19,7 @@ import json
 import os
 import time
 import types
+import uuid
 import weakref
 from asyncio.queues import Queue
 from asyncio.tasks import wait_for
@@ -65,7 +66,12 @@ except ImportError:
     OutOfMemoryError = _OutOfMemoryError
 
 
-XINFERENCE_BATCHING_ALLOWED_VISION_MODELS = ["qwen-vl-chat", "cogvlm2", "glm-4v"]
+XINFERENCE_BATCHING_ALLOWED_VISION_MODELS = [
+    "qwen-vl-chat",
+    "cogvlm2",
+    "glm-4v",
+    "MiniCPM-V-2.6",
+]
 
 
 def request_limit(fn):
@@ -265,7 +271,7 @@ class ModelActor(xo.StatelessActor):
 
         if self._worker_ref is None:
             self._worker_ref = await xo.actor_ref(
-                address=self._worker_address, uid=WorkerActor.uid()
+                address=self._worker_address, uid=WorkerActor.default_uid()
             )
         return self._worker_ref
 
@@ -434,23 +440,35 @@ class ModelActor(xo.StatelessActor):
             assert output_type == "binary", f"Unknown output type '{output_type}'"
             return ret
 
-    @log_async(logger=logger)
     @request_limit
     @xo.generator
+    @log_async(logger=logger)
     async def generate(self, prompt: str, *args, **kwargs):
         if self.allow_batching():
+            # not support request_id
+            kwargs.pop("request_id", None)
             return await self.handle_batching_request(
                 prompt, "generate", *args, **kwargs
             )
         else:
             kwargs.pop("raw_params", None)
             if hasattr(self._model, "generate"):
+                # not support request_id
+                kwargs.pop("request_id", None)
                 return await self._call_wrapper_json(
                     self._model.generate, prompt, *args, **kwargs
                 )
             if hasattr(self._model, "async_generate"):
+                if "request_id" not in kwargs:
+                    kwargs["request_id"] = str(uuid.uuid1())
+                else:
+                    # model only accept string
+                    kwargs["request_id"] = str(kwargs["request_id"])
                 return await self._call_wrapper_json(
-                    self._model.async_generate, prompt, *args, **kwargs
+                    self._model.async_generate,
+                    prompt,
+                    *args,
+                    **kwargs,
                 )
             raise AttributeError(f"Model {self._model.model_spec} is not for generate.")
 
@@ -481,22 +499,27 @@ class ModelActor(xo.StatelessActor):
                 yield res
 
     @staticmethod
-    def _get_stream_from_args(ability: str, *args) -> bool:
-        if ability == "chat":
-            assert args[2] is None or isinstance(args[2], dict)
-            return False if args[2] is None else args[2].get("stream", False)
-        else:
-            assert args[0] is None or isinstance(args[0], dict)
-            return False if args[0] is None else args[0].get("stream", False)
+    def _get_stream_from_args(*args) -> bool:
+        assert args[0] is None or isinstance(args[0], dict)
+        return False if args[0] is None else args[0].get("stream", False)
 
-    async def handle_batching_request(self, prompt: str, ability: str, *args, **kwargs):
-        stream = self._get_stream_from_args(ability, *args)
+    async def handle_batching_request(
+        self, prompt_or_messages: Union[str, List[Dict]], call_ability, *args, **kwargs
+    ):
+        """
+        The input parameter `prompt_or_messages`:
+        - when the model_ability is `generate`, it's `prompt`, which is str type.
+        - when the model_ability is `chat`, it's `messages`, which is List[Dict] type.
+        """
+        stream = self._get_stream_from_args(*args)
         assert self._scheduler_ref is not None
         if stream:
             assert self._scheduler_ref is not None
             queue: Queue[Any] = Queue()
             ret = self._queue_consumer(queue)
-            await self._scheduler_ref.add_request(prompt, queue, *args, **kwargs)
+            await self._scheduler_ref.add_request(
+                prompt_or_messages, queue, call_ability, *args, **kwargs
+            )
             gen = self._to_async_gen("json", ret)
             self._current_generator = weakref.ref(gen)
             return gen
@@ -505,7 +528,9 @@ class ModelActor(xo.StatelessActor):
 
             assert self._loop is not None
             future = ConcurrentFuture()
-            await self._scheduler_ref.add_request(prompt, future, *args, **kwargs)
+            await self._scheduler_ref.add_request(
+                prompt_or_messages, future, call_ability, *args, **kwargs
+            )
             fut = asyncio.wrap_future(future, loop=self._loop)
             result = await fut
             if result == XINFERENCE_NON_STREAMING_ABORT_FLAG:
@@ -514,27 +539,36 @@ class ModelActor(xo.StatelessActor):
                 )
             return await asyncio.to_thread(json_dumps, result)
 
-    @log_async(logger=logger)
     @request_limit
     @xo.generator
-    async def chat(self, prompt: str, *args, **kwargs):
+    @log_async(logger=logger)
+    async def chat(self, messages: List[Dict], *args, **kwargs):
         start_time = time.time()
         response = None
         try:
             if self.allow_batching():
+                # not support request_id
+                kwargs.pop("request_id", None)
                 return await self.handle_batching_request(
-                    prompt, "chat", *args, **kwargs
+                    messages, "chat", *args, **kwargs
                 )
             else:
                 kwargs.pop("raw_params", None)
                 if hasattr(self._model, "chat"):
+                    # not support request_id
+                    kwargs.pop("request_id", None)
                     response = await self._call_wrapper_json(
-                        self._model.chat, prompt, *args, **kwargs
+                        self._model.chat, messages, *args, **kwargs
                     )
                     return response
                 if hasattr(self._model, "async_chat"):
+                    if "request_id" not in kwargs:
+                        kwargs["request_id"] = str(uuid.uuid1())
+                    else:
+                        # model only accept string
+                        kwargs["request_id"] = str(kwargs["request_id"])
                     response = await self._call_wrapper_json(
-                        self._model.async_chat, prompt, *args, **kwargs
+                        self._model.async_chat, messages, *args, **kwargs
                     )
                     return response
                 raise AttributeError(f"Model {self._model.model_spec} is not for chat.")
@@ -565,9 +599,10 @@ class ModelActor(xo.StatelessActor):
             return await self._scheduler_ref.abort_request(request_id)
         return AbortRequestMessage.NO_OP.name
 
-    @log_async(logger=logger)
     @request_limit
+    @log_async(logger=logger)
     async def create_embedding(self, input: Union[str, List[str]], *args, **kwargs):
+        kwargs.pop("request_id", None)
         if hasattr(self._model, "create_embedding"):
             return await self._call_wrapper_json(
                 self._model.create_embedding, input, *args, **kwargs
@@ -577,8 +612,8 @@ class ModelActor(xo.StatelessActor):
             f"Model {self._model.model_spec} is not for creating embedding."
         )
 
-    @log_async(logger=logger)
     @request_limit
+    @log_async(logger=logger)
     async def rerank(
         self,
         documents: List[str],
@@ -590,6 +625,7 @@ class ModelActor(xo.StatelessActor):
         *args,
         **kwargs,
     ):
+        kwargs.pop("request_id", None)
         if hasattr(self._model, "rerank"):
             return await self._call_wrapper_json(
                 self._model.rerank,
@@ -604,8 +640,8 @@ class ModelActor(xo.StatelessActor):
             )
         raise AttributeError(f"Model {self._model.model_spec} is not for reranking.")
 
-    @log_async(logger=logger, args_formatter=lambda _, kwargs: kwargs.pop("audio"))
     @request_limit
+    @log_async(logger=logger, ignore_kwargs=["audio"])
     async def transcriptions(
         self,
         audio: bytes,
@@ -614,7 +650,9 @@ class ModelActor(xo.StatelessActor):
         response_format: str = "json",
         temperature: float = 0,
         timestamp_granularities: Optional[List[str]] = None,
+        **kwargs,
     ):
+        kwargs.pop("request_id", None)
         if hasattr(self._model, "transcriptions"):
             return await self._call_wrapper_json(
                 self._model.transcriptions,
@@ -629,8 +667,8 @@ class ModelActor(xo.StatelessActor):
             f"Model {self._model.model_spec} is not for creating transcriptions."
         )
 
-    @log_async(logger=logger, args_formatter=lambda _, kwargs: kwargs.pop("audio"))
     @request_limit
+    @log_async(logger=logger, ignore_kwargs=["audio"])
     async def translations(
         self,
         audio: bytes,
@@ -639,7 +677,9 @@ class ModelActor(xo.StatelessActor):
         response_format: str = "json",
         temperature: float = 0,
         timestamp_granularities: Optional[List[str]] = None,
+        **kwargs,
     ):
+        kwargs.pop("request_id", None)
         if hasattr(self._model, "translations"):
             return await self._call_wrapper_json(
                 self._model.translations,
@@ -654,12 +694,9 @@ class ModelActor(xo.StatelessActor):
             f"Model {self._model.model_spec} is not for creating translations."
         )
 
-    @log_async(
-        logger=logger,
-        args_formatter=lambda _, kwargs: kwargs.pop("prompt_speech", None),
-    )
     @request_limit
     @xo.generator
+    @log_async(logger=logger, ignore_kwargs=["prompt_speech"])
     async def speech(
         self,
         input: str,
@@ -669,6 +706,7 @@ class ModelActor(xo.StatelessActor):
         stream: bool = False,
         **kwargs,
     ):
+        kwargs.pop("request_id", None)
         if hasattr(self._model, "speech"):
             return await self._call_wrapper_binary(
                 self._model.speech,
@@ -683,8 +721,8 @@ class ModelActor(xo.StatelessActor):
             f"Model {self._model.model_spec} is not for creating speech."
         )
 
-    @log_async(logger=logger)
     @request_limit
+    @log_async(logger=logger)
     async def text_to_image(
         self,
         prompt: str,
@@ -694,6 +732,7 @@ class ModelActor(xo.StatelessActor):
         *args,
         **kwargs,
     ):
+        kwargs.pop("request_id", None)
         if hasattr(self._model, "text_to_image"):
             return await self._call_wrapper_json(
                 self._model.text_to_image,
@@ -708,6 +747,24 @@ class ModelActor(xo.StatelessActor):
             f"Model {self._model.model_spec} is not for creating image."
         )
 
+    @request_limit
+    @log_async(logger=logger)
+    async def txt2img(
+        self,
+        **kwargs,
+    ):
+        kwargs.pop("request_id", None)
+        if hasattr(self._model, "txt2img"):
+            return await self._call_wrapper_json(
+                self._model.txt2img,
+                **kwargs,
+            )
+        raise AttributeError(f"Model {self._model.model_spec} is not for txt2img.")
+
+    @log_async(
+        logger=logger,
+        ignore_kwargs=["image"],
+    )
     async def image_to_image(
         self,
         image: "PIL.Image",
@@ -719,6 +776,7 @@ class ModelActor(xo.StatelessActor):
         *args,
         **kwargs,
     ):
+        kwargs.pop("request_id", None)
         if hasattr(self._model, "image_to_image"):
             return await self._call_wrapper_json(
                 self._model.image_to_image,
@@ -735,6 +793,24 @@ class ModelActor(xo.StatelessActor):
             f"Model {self._model.model_spec} is not for creating image."
         )
 
+    @request_limit
+    @log_async(logger=logger)
+    async def img2img(
+        self,
+        **kwargs,
+    ):
+        kwargs.pop("request_id", None)
+        if hasattr(self._model, "img2img"):
+            return await self._call_wrapper_json(
+                self._model.img2img,
+                **kwargs,
+            )
+        raise AttributeError(f"Model {self._model.model_spec} is not for img2img.")
+
+    @log_async(
+        logger=logger,
+        ignore_kwargs=["image"],
+    )
     async def inpainting(
         self,
         image: "PIL.Image",
@@ -747,6 +823,7 @@ class ModelActor(xo.StatelessActor):
         *args,
         **kwargs,
     ):
+        kwargs.pop("request_id", None)
         if hasattr(self._model, "inpainting"):
             return await self._call_wrapper_json(
                 self._model.inpainting,
@@ -764,12 +841,13 @@ class ModelActor(xo.StatelessActor):
             f"Model {self._model.model_spec} is not for creating image."
         )
 
-    @log_async(logger=logger)
     @request_limit
+    @log_async(logger=logger, ignore_kwargs=["image"])
     async def infer(
         self,
         **kwargs,
     ):
+        kwargs.pop("request_id", None)
         if hasattr(self._model, "infer"):
             return await self._call_wrapper_json(
                 self._model.infer,
@@ -779,8 +857,8 @@ class ModelActor(xo.StatelessActor):
             f"Model {self._model.model_spec} is not for flexible infer."
         )
 
-    @log_async(logger=logger)
     @request_limit
+    @log_async(logger=logger)
     async def text_to_video(
         self,
         prompt: str,
@@ -788,6 +866,7 @@ class ModelActor(xo.StatelessActor):
         *args,
         **kwargs,
     ):
+        kwargs.pop("request_id", None)
         if hasattr(self._model, "text_to_video"):
             return await self._call_wrapper_json(
                 self._model.text_to_video,

xinference/core/scheduler.py

@@ -18,7 +18,7 @@ import logging
 import uuid
 from collections import deque
 from enum import Enum
-from typing import List, Optional, Set, Tuple
+from typing import Dict, List, Optional, Set, Tuple, Union
 
 import xoscar as xo
 
@@ -37,13 +37,24 @@ class AbortRequestMessage(Enum):
 
 
 class InferenceRequest:
-    def __init__(self, prompt, future_or_queue, is_prefill, *args, **kwargs):
-        # original prompt
-        self._prompt = prompt
+    def __init__(
+        self,
+        prompt_or_messages,
+        future_or_queue,
+        is_prefill,
+        call_ability,
+        *args,
+        **kwargs,
+    ):
+        # original prompt, prompt(str) for generate model and messages(List[Dict]) for chat model
+        self._prompt = prompt_or_messages
         # full prompt that contains chat history and applies chat template
         self._full_prompt = None
         # whether the current request is in the prefill phase
         self._is_prefill = is_prefill
+        # the ability that the user calls this model for, that is `generate` / `chat` for now,
+        # which is for results formatting
+        self._call_ability = call_ability
         # full prompt tokens
         self._prompt_tokens = None
         # all new generated tokens during decode phase
@@ -88,38 +99,22 @@ class InferenceRequest:
         self._check_args()
 
     def _check_args(self):
-        # chat
-        if len(self._inference_args) == 3:
-            # system prompt
-            assert self._inference_args[0] is None or isinstance(
-                self._inference_args[0], str
-            )
-            # chat history
-            assert self._inference_args[1] is None or isinstance(
-                self._inference_args[1], list
-            )
-            # generate config
-            assert self._inference_args[2] is None or isinstance(
-                self._inference_args[2], dict
-            )
-        else:  # generate
-            assert len(self._inference_args) == 1
-            # generate config
-            assert self._inference_args[0] is None or isinstance(
-                self._inference_args[0], dict
-            )
+        assert len(self._inference_args) == 1
+        # generate config
+        assert self._inference_args[0] is None or isinstance(
+            self._inference_args[0], dict
+        )
 
     @property
     def prompt(self):
+        """
+        prompt for generate model and messages for chat model
+        """
         return self._prompt
 
     @property
-    def system_prompt(self):
-        return self._inference_args[0]
-
-    @property
-    def chat_history(self):
-        return self._inference_args[1]
+    def call_ability(self):
+        return self._call_ability
 
     @property
     def full_prompt(self):
@@ -162,11 +157,7 @@ class InferenceRequest:
 
     @property
     def generate_config(self):
-        return (
-            self._inference_args[2]
-            if len(self._inference_args) == 3
-            else self._inference_args[0]
-        )
+        return self._inference_args[0]
 
     @property
     def sanitized_generate_config(self):
@@ -423,8 +414,17 @@ class SchedulerActor(xo.StatelessActor):
 
         self._empty_cache()
 
-    async def add_request(self, prompt: str, future_or_queue, *args, **kwargs):
-        req = InferenceRequest(prompt, future_or_queue, True, *args, **kwargs)
+    async def add_request(
+        self,
+        prompt_or_messages: Union[str, List[Dict]],
+        future_or_queue,
+        call_ability,
+        *args,
+        **kwargs,
+    ):
+        req = InferenceRequest(
+            prompt_or_messages, future_or_queue, True, call_ability, *args, **kwargs
+        )
         rid = req.request_id
         if rid is not None:
             if rid in self._id_to_req:

xinference/core/status_guard.py

@@ -51,7 +51,7 @@ class StatusGuardActor(xo.StatelessActor):
         self._model_uid_to_info: Dict[str, InstanceInfo] = {}  # type: ignore
 
     @classmethod
-    def uid(cls) -> str:
+    def default_uid(cls) -> str:
         return "status_guard"
 
     @staticmethod

xinference/core/supervisor.py

@@ -105,7 +105,7 @@ class SupervisorActor(xo.StatelessActor):
         self._lock = asyncio.Lock()
 
     @classmethod
-    def uid(cls) -> str:
+    def default_uid(cls) -> str:
         return "supervisor"
 
     def _get_worker_ref_by_ip(
@@ -135,12 +135,12 @@ class SupervisorActor(xo.StatelessActor):
         self._status_guard_ref: xo.ActorRefType[  # type: ignore
             "StatusGuardActor"
         ] = await xo.create_actor(
-            StatusGuardActor, address=self.address, uid=StatusGuardActor.uid()
+            StatusGuardActor, address=self.address, uid=StatusGuardActor.default_uid()
         )
         self._cache_tracker_ref: xo.ActorRefType[  # type: ignore
             "CacheTrackerActor"
         ] = await xo.create_actor(
-            CacheTrackerActor, address=self.address, uid=CacheTrackerActor.uid()
+            CacheTrackerActor, address=self.address, uid=CacheTrackerActor.default_uid()
         )
 
         from .event import EventCollectorActor
@@ -148,7 +148,9 @@ class SupervisorActor(xo.StatelessActor):
         self._event_collector_ref: xo.ActorRefType[  # type: ignore
             EventCollectorActor
         ] = await xo.create_actor(
-            EventCollectorActor, address=self.address, uid=EventCollectorActor.uid()
+            EventCollectorActor,
+            address=self.address,
+            uid=EventCollectorActor.default_uid(),
         )
 
         from ..model.audio import (
@@ -308,14 +310,12 @@ class SupervisorActor(xo.StatelessActor):
     async def get_builtin_prompts() -> Dict[str, Any]:
         from ..model.llm.llm_family import BUILTIN_LLM_PROMPT_STYLE
 
-        data = {}
-        for k, v in BUILTIN_LLM_PROMPT_STYLE.items():
-            data[k] = v.dict()
-        return data
+        return {k: v for k, v in BUILTIN_LLM_PROMPT_STYLE.items()}
 
     @staticmethod
     async def get_builtin_families() -> Dict[str, List[str]]:
         from ..model.llm.llm_family import (
+            BUILTIN_LLM_FAMILIES,
             BUILTIN_LLM_MODEL_CHAT_FAMILIES,
             BUILTIN_LLM_MODEL_GENERATE_FAMILIES,
             BUILTIN_LLM_MODEL_TOOL_CALL_FAMILIES,
@@ -325,6 +325,11 @@ class SupervisorActor(xo.StatelessActor):
             "chat": list(BUILTIN_LLM_MODEL_CHAT_FAMILIES),
             "generate": list(BUILTIN_LLM_MODEL_GENERATE_FAMILIES),
             "tools": list(BUILTIN_LLM_MODEL_TOOL_CALL_FAMILIES),
+            "vision": [
+                family.model_name
+                for family in BUILTIN_LLM_FAMILIES
+                if "vision" in family.model_ability
+            ],
         }
 
     async def get_devices_count(self) -> int:
@@ -1028,7 +1033,7 @@ class SupervisorActor(xo.StatelessActor):
         else:
             task = asyncio.create_task(_launch_model())
             ASYNC_LAUNCH_TASKS[model_uid] = task
-            task.add_done_callback(lambda _: callback_for_async_launch(model_uid))
+            task.add_done_callback(lambda _: callback_for_async_launch(model_uid))  # type: ignore
         return model_uid
 
     async def get_instance_info(
@@ -1233,7 +1238,9 @@ class SupervisorActor(xo.StatelessActor):
             worker_address not in self._worker_address_to_worker
         ), f"Worker {worker_address} exists"
 
-        worker_ref = await xo.actor_ref(address=worker_address, uid=WorkerActor.uid())
+        worker_ref = await xo.actor_ref(
+            address=worker_address, uid=WorkerActor.default_uid()
+        )
         self._worker_address_to_worker[worker_address] = worker_ref
         logger.debug("Worker %s has been added successfully", worker_address)