xinference 0.14.4.post1__py3-none-any.whl → 0.15.0__py3-none-any.whl

xinference/_compat.py

@@ -11,6 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from typing import Dict, Iterable, List, Literal, Optional, Union
+
 from pydantic.version import VERSION as PYDANTIC_VERSION
 
 PYDANTIC_V2 = PYDANTIC_VERSION.startswith("2.")
@@ -50,3 +52,52 @@ else:
     from pydantic.parse import load_str_bytes  # noqa: F401
     from pydantic.types import StrBytes  # noqa: F401
     from pydantic.utils import ROOT_KEY  # noqa: F401
+
+from openai.types.chat.chat_completion_named_tool_choice_param import (
+    ChatCompletionNamedToolChoiceParam,
+)
+from openai.types.chat.chat_completion_stream_options_param import (
+    ChatCompletionStreamOptionsParam,
+)
+from openai.types.chat.chat_completion_tool_param import ChatCompletionToolParam
+
+OpenAIChatCompletionStreamOptionsParam = create_model_from_typeddict(
+    ChatCompletionStreamOptionsParam
+)
+OpenAIChatCompletionToolParam = create_model_from_typeddict(ChatCompletionToolParam)
+OpenAIChatCompletionNamedToolChoiceParam = create_model_from_typeddict(
+    ChatCompletionNamedToolChoiceParam
+)
+
+
+class CreateChatCompletionOpenAI(BaseModel):
+    """
+    Comes from source code: https://github.com/openai/openai-python/blob/main/src/openai/types/chat/completion_create_params.py
+    """
+
+    messages: List[Dict]
+    model: str
+    frequency_penalty: Optional[float]
+    logit_bias: Optional[Dict[str, int]]
+    logprobs: Optional[bool]
+    max_tokens: Optional[int]
+    n: Optional[int]
+    parallel_tool_calls: Optional[bool]
+    presence_penalty: Optional[float]
+    # we do not support this
+    # response_format: ResponseFormat
+    seed: Optional[int]
+    service_tier: Optional[Literal["auto", "default"]]
+    stop: Union[Optional[str], List[str]]
+    stream_options: Optional[OpenAIChatCompletionStreamOptionsParam]  # type: ignore
+    temperature: Optional[float]
+    tool_choice: Optional[  # type: ignore
+        Union[
+            Literal["none", "auto", "required"],
+            OpenAIChatCompletionNamedToolChoiceParam,
+        ]
+    ]
+    tools: Optional[Iterable[OpenAIChatCompletionToolParam]]  # type: ignore
+    top_logprobs: Optional[int]
+    top_p: Optional[float]
+    user: Optional[str]

xinference/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2024-09-03T15:42:58+0800",
+ "date": "2024-09-06T16:29:42+0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "b1b7c44e6f0ad934eb8366d531c87f29cfa239a7",
- "version": "0.14.4.post1"
+ "full-revisionid": "e2618be96293f112709c9ceed639a3443455a0e7",
+ "version": "0.15.0"
 }
 '''  # END VERSION_JSON
 

xinference/api/restful_api.py

@@ -57,9 +57,7 @@ from ..core.event import Event, EventCollectorActor, EventType
 from ..core.supervisor import SupervisorActor
 from ..core.utils import json_dumps
 from ..types import (
-    SPECIAL_TOOL_PROMPT,
     ChatCompletion,
-    ChatCompletionMessage,
     Completion,
     CreateChatCompletion,
     CreateCompletion,
@@ -199,14 +197,14 @@ class RESTfulAPI:
     async def _get_supervisor_ref(self) -> xo.ActorRefType[SupervisorActor]:
         if self._supervisor_ref is None:
             self._supervisor_ref = await xo.actor_ref(
-                address=self._supervisor_address, uid=SupervisorActor.uid()
+                address=self._supervisor_address, uid=SupervisorActor.default_uid()
             )
         return self._supervisor_ref
 
     async def _get_event_collector_ref(self) -> xo.ActorRefType[EventCollectorActor]:
         if self._event_collector_ref is None:
             self._event_collector_ref = await xo.actor_ref(
-                address=self._supervisor_address, uid=EventCollectorActor.uid()
+                address=self._supervisor_address, uid=EventCollectorActor.default_uid()
             )
         return self._event_collector_ref
 
@@ -1627,33 +1625,7 @@ class RESTfulAPI:
                 status_code=400, detail="Invalid input. Please specify the prompt."
             )
 
-        system_messages: List["ChatCompletionMessage"] = []
-        system_messages_contents = []
-        non_system_messages = []
-        for msg in messages:
-            assert (
-                msg.get("content") != SPECIAL_TOOL_PROMPT
-            ), f"Invalid message content {SPECIAL_TOOL_PROMPT}"
-            if msg["role"] == "system":
-                system_messages_contents.append(msg["content"])
-            else:
-                non_system_messages.append(msg)
-        system_messages.append(
-            {"role": "system", "content": ". ".join(system_messages_contents)}
-        )
-
         has_tool_message = messages[-1].get("role") == "tool"
-        if has_tool_message:
-            prompt = SPECIAL_TOOL_PROMPT
-            system_prompt = system_messages[0]["content"] if system_messages else None
-            chat_history = non_system_messages  # exclude the prompt
-        else:
-            prompt = None
-            if non_system_messages:
-                prompt = non_system_messages[-1]["content"]
-            system_prompt = system_messages[0]["content"] if system_messages else None
-            chat_history = non_system_messages[:-1]  # exclude the prompt
-
         model_uid = body.model
 
         try:
@@ -1681,9 +1653,7 @@ class RESTfulAPI:
         from ..model.llm.utils import GLM4_TOOL_CALL_FAMILY, QWEN_TOOL_CALL_FAMILY
 
         model_family = desc.get("model_family", "")
-        function_call_models = (
-            ["gorilla-openfunctions-v1"] + QWEN_TOOL_CALL_FAMILY + GLM4_TOOL_CALL_FAMILY
-        )
+        function_call_models = QWEN_TOOL_CALL_FAMILY + GLM4_TOOL_CALL_FAMILY
 
         if model_family not in function_call_models:
             if body.tools:
@@ -1716,9 +1686,7 @@ class RESTfulAPI:
                 try:
                     try:
                         iterator = await model.chat(
-                            prompt,
-                            system_prompt,
-                            chat_history,
+                            messages,
                             kwargs,
                             raw_params=raw_kwargs,
                         )
@@ -1750,9 +1718,7 @@ class RESTfulAPI:
         else:
             try:
                 data = await model.chat(
-                    prompt,
-                    system_prompt,
-                    chat_history,
+                    messages,
                     kwargs,
                     raw_params=raw_kwargs,
                 )

xinference/client/restful/restful_client.py

@@ -13,7 +13,6 @@
 # limitations under the License.
 import json
 import typing
-import warnings
 from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Union
 
 import requests
@@ -470,9 +469,7 @@ class RESTfulGenerateModelHandle(RESTfulModelHandle):
 class RESTfulChatModelHandle(RESTfulGenerateModelHandle):
     def chat(
         self,
-        prompt: str,
-        system_prompt: Optional[str] = None,
-        chat_history: Optional[List["ChatCompletionMessage"]] = None,
+        messages: List[Dict],
         tools: Optional[List[Dict]] = None,
         generate_config: Optional[
             Union["LlamaCppGenerateConfig", "PytorchGenerateConfig"]
@@ -483,11 +480,7 @@ class RESTfulChatModelHandle(RESTfulGenerateModelHandle):
 
         Parameters
         ----------
-        prompt: str
-            The user's input.
-        system_prompt: Optional[str]
-            The system context provide to Model prior to any chats.
-        chat_history: Optional[List["ChatCompletionMessage"]]
+        messages: List[Dict]
             A list of messages comprising the conversation so far.
         tools: Optional[List[Dict]]
             A tool list.
@@ -509,25 +502,11 @@ class RESTfulChatModelHandle(RESTfulGenerateModelHandle):
             Report the failure to generate the chat from the server. Detailed information provided in error message.
 
         """
-        warnings.warn(
-            "The parameters `prompt`, `system_prompt` and `chat_history` will be deprecated in version v0.15.0, "
-            "and will be replaced by the parameter `messages`, "
-            "similar to the OpenAI API: https://platform.openai.com/docs/guides/chat-completions/getting-started",
-            category=DeprecationWarning,
-            stacklevel=2,
-        )
-
         url = f"{self._base_url}/v1/chat/completions"
 
-        if chat_history is None:
-            chat_history = []
-
-        chat_history = handle_system_prompts(chat_history, system_prompt)
-        chat_history.append({"role": "user", "content": prompt})  # type: ignore
-
         request_body: Dict[str, Any] = {
             "model": self._model_uid,
-            "messages": chat_history,
+            "messages": messages,
         }
         if tools is not None:
             request_body["tools"] = tools

xinference/conftest.py

@@ -144,7 +144,7 @@ async def _start_test_cluster(
             address=f"test://{address}", logging_conf=logging_conf
         )
         await xo.create_actor(
-            SupervisorActor, address=address, uid=SupervisorActor.uid()
+            SupervisorActor, address=address, uid=SupervisorActor.default_uid()
         )
         await start_worker_components(
             address=address,

xinference/constants.py

@@ -38,6 +38,10 @@ def get_xinference_home() -> str:
         # if user has already set `XINFERENCE_HOME` env, change huggingface and modelscope default download path
         os.environ["HUGGINGFACE_HUB_CACHE"] = os.path.join(home_path, "huggingface")
         os.environ["MODELSCOPE_CACHE"] = os.path.join(home_path, "modelscope")
+    # In multi-tenant mode,
+    # gradio's temporary files are stored in their respective home directories,
+    # to prevent insufficient permissions
+    os.environ["GRADIO_TEMP_DIR"] = os.path.join(home_path, "tmp", "gradio")
     return home_path
 
 
@@ -59,6 +63,7 @@ XINFERENCE_DEFAULT_ENDPOINT_PORT = 9997
 XINFERENCE_DEFAULT_LOG_FILE_NAME = "xinference.log"
 XINFERENCE_LOG_MAX_BYTES = 100 * 1024 * 1024
 XINFERENCE_LOG_BACKUP_COUNT = 30
+XINFERENCE_LOG_ARG_MAX_LENGTH = 100
 XINFERENCE_HEALTH_CHECK_FAILURE_THRESHOLD = int(
     os.environ.get(XINFERENCE_ENV_HEALTH_CHECK_FAILURE_THRESHOLD, 5)
 )

xinference/core/cache_tracker.py

@@ -25,7 +25,7 @@ class CacheTrackerActor(xo.Actor):
         self._model_name_to_version_info: Dict[str, List[Dict]] = {}  # type: ignore
 
     @classmethod
-    def uid(cls) -> str:
+    def default_uid(cls) -> str:
         return "cache_tracker"
 
     @staticmethod

xinference/core/chat_interface.py

@@ -16,7 +16,7 @@ import base64
 import logging
 import os
 from io import BytesIO
-from typing import Generator, List, Optional
+from typing import Dict, Generator, List, Optional
 
 import gradio as gr
 import PIL.Image
@@ -27,7 +27,6 @@ from ..client.restful.restful_client import (
     RESTfulChatModelHandle,
     RESTfulGenerateModelHandle,
 )
-from ..types import ChatCompletionMessage
 
 logger = logging.getLogger(__name__)
 
@@ -96,11 +95,11 @@ class GradioInterface:
                 flat_list += row
             return flat_list
 
-        def to_chat(lst: List[str]) -> List[ChatCompletionMessage]:
+        def to_chat(lst: List[str]) -> List[Dict]:
             res = []
             for i in range(len(lst)):
                 role = "assistant" if i % 2 == 1 else "user"
-                res.append(ChatCompletionMessage(role=role, content=lst[i]))
+                res.append(dict(role=role, content=lst[i]))
             return res
 
         def generate_wrapper(
@@ -116,11 +115,12 @@ class GradioInterface:
             client._set_token(self._access_token)
             model = client.get_model(self.model_uid)
             assert isinstance(model, RESTfulChatModelHandle)
+            messages = to_chat(flatten(history))
+            messages.append(dict(role="user", content=message))
 
             response_content = ""
             for chunk in model.chat(
-                prompt=message,
-                chat_history=to_chat(flatten(history)),
+                messages,
                 generate_config={
                     "max_tokens": int(max_tokens),
                     "temperature": temperature,
@@ -191,15 +191,10 @@ class GradioInterface:
             model = client.get_model(self.model_uid)
             assert isinstance(model, RESTfulChatModelHandle)
 
-            prompt = history[-1]
-            assert prompt["role"] == "user"
-            prompt = prompt["content"]
-            # multimodal chat does not support stream.
             if stream:
                 response_content = ""
                 for chunk in model.chat(
-                    prompt=prompt,
-                    chat_history=history[:-1],
+                    messages=history,
                     generate_config={
                         "max_tokens": max_tokens,
                         "temperature": temperature,
@@ -224,8 +219,7 @@ class GradioInterface:
                 yield history, bot
             else:
                 response = model.chat(
-                    prompt=prompt,
-                    chat_history=history[:-1],
+                    messages=history,
                     generate_config={
                         "max_tokens": max_tokens,
                         "temperature": temperature,

xinference/core/event.py

@@ -41,7 +41,7 @@ class EventCollectorActor(xo.StatelessActor):
         )
 
     @classmethod
-    def uid(cls) -> str:
+    def default_uid(cls) -> str:
         return "event_collector"
 
     def get_model_events(self, model_uid: str) -> List[Dict]:

xinference/core/model.py

@@ -19,6 +19,7 @@ import json
 import os
 import time
 import types
+import uuid
 import weakref
 from asyncio.queues import Queue
 from asyncio.tasks import wait_for
@@ -65,7 +66,12 @@ except ImportError:
     OutOfMemoryError = _OutOfMemoryError
 
 
-XINFERENCE_BATCHING_ALLOWED_VISION_MODELS = ["qwen-vl-chat", "cogvlm2", "glm-4v"]
+XINFERENCE_BATCHING_ALLOWED_VISION_MODELS = [
+    "qwen-vl-chat",
+    "cogvlm2",
+    "glm-4v",
+    "MiniCPM-V-2.6",
+]
 
 
 def request_limit(fn):
@@ -265,7 +271,7 @@ class ModelActor(xo.StatelessActor):
 
         if self._worker_ref is None:
             self._worker_ref = await xo.actor_ref(
-                address=self._worker_address, uid=WorkerActor.uid()
+                address=self._worker_address, uid=WorkerActor.default_uid()
             )
         return self._worker_ref
 
@@ -434,23 +440,35 @@ class ModelActor(xo.StatelessActor):
             assert output_type == "binary", f"Unknown output type '{output_type}'"
             return ret
 
-    @log_async(logger=logger)
     @request_limit
     @xo.generator
+    @log_async(logger=logger)
     async def generate(self, prompt: str, *args, **kwargs):
         if self.allow_batching():
+            # not support request_id
+            kwargs.pop("request_id", None)
             return await self.handle_batching_request(
                 prompt, "generate", *args, **kwargs
             )
         else:
             kwargs.pop("raw_params", None)
             if hasattr(self._model, "generate"):
+                # not support request_id
+                kwargs.pop("request_id", None)
                 return await self._call_wrapper_json(
                     self._model.generate, prompt, *args, **kwargs
                 )
             if hasattr(self._model, "async_generate"):
+                if "request_id" not in kwargs:
+                    kwargs["request_id"] = str(uuid.uuid1())
+                else:
+                    # model only accept string
+                    kwargs["request_id"] = str(kwargs["request_id"])
                 return await self._call_wrapper_json(
-                    self._model.async_generate, prompt, *args, **kwargs
+                    self._model.async_generate,
+                    prompt,
+                    *args,
+                    **kwargs,
                 )
             raise AttributeError(f"Model {self._model.model_spec} is not for generate.")
 
@@ -481,22 +499,27 @@ class ModelActor(xo.StatelessActor):
                 yield res
 
     @staticmethod
-    def _get_stream_from_args(ability: str, *args) -> bool:
-        if ability == "chat":
-            assert args[2] is None or isinstance(args[2], dict)
-            return False if args[2] is None else args[2].get("stream", False)
-        else:
-            assert args[0] is None or isinstance(args[0], dict)
-            return False if args[0] is None else args[0].get("stream", False)
+    def _get_stream_from_args(*args) -> bool:
+        assert args[0] is None or isinstance(args[0], dict)
+        return False if args[0] is None else args[0].get("stream", False)
 
-    async def handle_batching_request(self, prompt: str, ability: str, *args, **kwargs):
-        stream = self._get_stream_from_args(ability, *args)
+    async def handle_batching_request(
+        self, prompt_or_messages: Union[str, List[Dict]], call_ability, *args, **kwargs
+    ):
+        """
+        The input parameter `prompt_or_messages`:
+        - when the model_ability is `generate`, it's `prompt`, which is str type.
+        - when the model_ability is `chat`, it's `messages`, which is List[Dict] type.
+        """
+        stream = self._get_stream_from_args(*args)
         assert self._scheduler_ref is not None
         if stream:
             assert self._scheduler_ref is not None
             queue: Queue[Any] = Queue()
             ret = self._queue_consumer(queue)
-            await self._scheduler_ref.add_request(prompt, queue, *args, **kwargs)
+            await self._scheduler_ref.add_request(
+                prompt_or_messages, queue, call_ability, *args, **kwargs
+            )
             gen = self._to_async_gen("json", ret)
             self._current_generator = weakref.ref(gen)
             return gen
@@ -505,7 +528,9 @@ class ModelActor(xo.StatelessActor):
 
             assert self._loop is not None
             future = ConcurrentFuture()
-            await self._scheduler_ref.add_request(prompt, future, *args, **kwargs)
+            await self._scheduler_ref.add_request(
+                prompt_or_messages, future, call_ability, *args, **kwargs
+            )
             fut = asyncio.wrap_future(future, loop=self._loop)
             result = await fut
             if result == XINFERENCE_NON_STREAMING_ABORT_FLAG:
@@ -514,27 +539,36 @@ class ModelActor(xo.StatelessActor):
                 )
             return await asyncio.to_thread(json_dumps, result)
 
-    @log_async(logger=logger)
     @request_limit
     @xo.generator
-    async def chat(self, prompt: str, *args, **kwargs):
+    @log_async(logger=logger)
+    async def chat(self, messages: List[Dict], *args, **kwargs):
         start_time = time.time()
         response = None
         try:
             if self.allow_batching():
+                # not support request_id
+                kwargs.pop("request_id", None)
                 return await self.handle_batching_request(
-                    prompt, "chat", *args, **kwargs
+                    messages, "chat", *args, **kwargs
                 )
             else:
                 kwargs.pop("raw_params", None)
                 if hasattr(self._model, "chat"):
+                    # not support request_id
+                    kwargs.pop("request_id", None)
                     response = await self._call_wrapper_json(
-                        self._model.chat, prompt, *args, **kwargs
+                        self._model.chat, messages, *args, **kwargs
                     )
                     return response
                 if hasattr(self._model, "async_chat"):
+                    if "request_id" not in kwargs:
+                        kwargs["request_id"] = str(uuid.uuid1())
+                    else:
+                        # model only accept string
+                        kwargs["request_id"] = str(kwargs["request_id"])
                     response = await self._call_wrapper_json(
-                        self._model.async_chat, prompt, *args, **kwargs
+                        self._model.async_chat, messages, *args, **kwargs
                     )
                     return response
                 raise AttributeError(f"Model {self._model.model_spec} is not for chat.")
@@ -565,9 +599,10 @@ class ModelActor(xo.StatelessActor):
             return await self._scheduler_ref.abort_request(request_id)
         return AbortRequestMessage.NO_OP.name
 
-    @log_async(logger=logger)
     @request_limit
+    @log_async(logger=logger)
     async def create_embedding(self, input: Union[str, List[str]], *args, **kwargs):
+        kwargs.pop("request_id", None)
         if hasattr(self._model, "create_embedding"):
             return await self._call_wrapper_json(
                 self._model.create_embedding, input, *args, **kwargs
@@ -577,8 +612,8 @@ class ModelActor(xo.StatelessActor):
             f"Model {self._model.model_spec} is not for creating embedding."
         )
 
-    @log_async(logger=logger)
     @request_limit
+    @log_async(logger=logger)
     async def rerank(
         self,
         documents: List[str],
@@ -590,6 +625,7 @@ class ModelActor(xo.StatelessActor):
         *args,
         **kwargs,
     ):
+        kwargs.pop("request_id", None)
         if hasattr(self._model, "rerank"):
             return await self._call_wrapper_json(
                 self._model.rerank,
@@ -604,8 +640,8 @@ class ModelActor(xo.StatelessActor):
             )
         raise AttributeError(f"Model {self._model.model_spec} is not for reranking.")
 
-    @log_async(logger=logger, args_formatter=lambda _, kwargs: kwargs.pop("audio"))
     @request_limit
+    @log_async(logger=logger, ignore_kwargs=["audio"])
     async def transcriptions(
         self,
         audio: bytes,
@@ -614,7 +650,9 @@ class ModelActor(xo.StatelessActor):
         response_format: str = "json",
         temperature: float = 0,
         timestamp_granularities: Optional[List[str]] = None,
+        **kwargs,
     ):
+        kwargs.pop("request_id", None)
         if hasattr(self._model, "transcriptions"):
             return await self._call_wrapper_json(
                 self._model.transcriptions,
@@ -629,8 +667,8 @@ class ModelActor(xo.StatelessActor):
             f"Model {self._model.model_spec} is not for creating transcriptions."
         )
 
-    @log_async(logger=logger, args_formatter=lambda _, kwargs: kwargs.pop("audio"))
     @request_limit
+    @log_async(logger=logger, ignore_kwargs=["audio"])
     async def translations(
         self,
         audio: bytes,
@@ -639,7 +677,9 @@ class ModelActor(xo.StatelessActor):
         response_format: str = "json",
         temperature: float = 0,
         timestamp_granularities: Optional[List[str]] = None,
+        **kwargs,
     ):
+        kwargs.pop("request_id", None)
         if hasattr(self._model, "translations"):
             return await self._call_wrapper_json(
                 self._model.translations,
@@ -654,12 +694,9 @@ class ModelActor(xo.StatelessActor):
             f"Model {self._model.model_spec} is not for creating translations."
         )
 
-    @log_async(
-        logger=logger,
-        args_formatter=lambda _, kwargs: kwargs.pop("prompt_speech", None),
-    )
     @request_limit
     @xo.generator
+    @log_async(logger=logger, ignore_kwargs=["prompt_speech"])
     async def speech(
         self,
         input: str,
@@ -669,6 +706,7 @@ class ModelActor(xo.StatelessActor):
         stream: bool = False,
         **kwargs,
     ):
+        kwargs.pop("request_id", None)
         if hasattr(self._model, "speech"):
             return await self._call_wrapper_binary(
                 self._model.speech,
@@ -683,8 +721,8 @@ class ModelActor(xo.StatelessActor):
             f"Model {self._model.model_spec} is not for creating speech."
         )
 
-    @log_async(logger=logger)
     @request_limit
+    @log_async(logger=logger)
     async def text_to_image(
         self,
         prompt: str,
@@ -694,6 +732,7 @@ class ModelActor(xo.StatelessActor):
         *args,
         **kwargs,
     ):
+        kwargs.pop("request_id", None)
         if hasattr(self._model, "text_to_image"):
             return await self._call_wrapper_json(
                 self._model.text_to_image,
@@ -708,6 +747,10 @@ class ModelActor(xo.StatelessActor):
             f"Model {self._model.model_spec} is not for creating image."
         )
 
+    @log_async(
+        logger=logger,
+        ignore_kwargs=["image"],
+    )
     async def image_to_image(
         self,
         image: "PIL.Image",
@@ -719,6 +762,7 @@ class ModelActor(xo.StatelessActor):
         *args,
         **kwargs,
     ):
+        kwargs.pop("request_id", None)
         if hasattr(self._model, "image_to_image"):
             return await self._call_wrapper_json(
                 self._model.image_to_image,
@@ -735,6 +779,10 @@ class ModelActor(xo.StatelessActor):
             f"Model {self._model.model_spec} is not for creating image."
         )
 
+    @log_async(
+        logger=logger,
+        ignore_kwargs=["image"],
+    )
     async def inpainting(
         self,
         image: "PIL.Image",
@@ -747,6 +795,7 @@ class ModelActor(xo.StatelessActor):
         *args,
         **kwargs,
     ):
+        kwargs.pop("request_id", None)
         if hasattr(self._model, "inpainting"):
             return await self._call_wrapper_json(
                 self._model.inpainting,
@@ -764,12 +813,13 @@ class ModelActor(xo.StatelessActor):
             f"Model {self._model.model_spec} is not for creating image."
         )
 
-    @log_async(logger=logger)
     @request_limit
+    @log_async(logger=logger, ignore_kwargs=["image"])
     async def infer(
         self,
         **kwargs,
     ):
+        kwargs.pop("request_id", None)
         if hasattr(self._model, "infer"):
             return await self._call_wrapper_json(
                 self._model.infer,
@@ -779,8 +829,8 @@ class ModelActor(xo.StatelessActor):
             f"Model {self._model.model_spec} is not for flexible infer."
         )
 
-    @log_async(logger=logger)
     @request_limit
+    @log_async(logger=logger)
     async def text_to_video(
         self,
         prompt: str,
@@ -788,6 +838,7 @@ class ModelActor(xo.StatelessActor):
         *args,
         **kwargs,
     ):
+        kwargs.pop("request_id", None)
         if hasattr(self._model, "text_to_video"):
             return await self._call_wrapper_json(
                 self._model.text_to_video,