xinference 0.14.4.post1__py3-none-any.whl → 0.15.1__py3-none-any.whl

xinference/_compat.py

@@ -11,6 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from typing import Dict, Iterable, List, Literal, Optional, Union
+
 from pydantic.version import VERSION as PYDANTIC_VERSION
 
 PYDANTIC_V2 = PYDANTIC_VERSION.startswith("2.")
@@ -50,3 +52,52 @@ else:
     from pydantic.parse import load_str_bytes  # noqa: F401
     from pydantic.types import StrBytes  # noqa: F401
     from pydantic.utils import ROOT_KEY  # noqa: F401
+
+from openai.types.chat.chat_completion_named_tool_choice_param import (
+    ChatCompletionNamedToolChoiceParam,
+)
+from openai.types.chat.chat_completion_stream_options_param import (
+    ChatCompletionStreamOptionsParam,
+)
+from openai.types.chat.chat_completion_tool_param import ChatCompletionToolParam
+
+OpenAIChatCompletionStreamOptionsParam = create_model_from_typeddict(
+    ChatCompletionStreamOptionsParam
+)
+OpenAIChatCompletionToolParam = create_model_from_typeddict(ChatCompletionToolParam)
+OpenAIChatCompletionNamedToolChoiceParam = create_model_from_typeddict(
+    ChatCompletionNamedToolChoiceParam
+)
+
+
+class CreateChatCompletionOpenAI(BaseModel):
+    """
+    Comes from source code: https://github.com/openai/openai-python/blob/main/src/openai/types/chat/completion_create_params.py
+    """
+
+    messages: List[Dict]
+    model: str
+    frequency_penalty: Optional[float]
+    logit_bias: Optional[Dict[str, int]]
+    logprobs: Optional[bool]
+    max_tokens: Optional[int]
+    n: Optional[int]
+    parallel_tool_calls: Optional[bool]
+    presence_penalty: Optional[float]
+    # we do not support this
+    # response_format: ResponseFormat
+    seed: Optional[int]
+    service_tier: Optional[Literal["auto", "default"]]
+    stop: Union[Optional[str], List[str]]
+    stream_options: Optional[OpenAIChatCompletionStreamOptionsParam]  # type: ignore
+    temperature: Optional[float]
+    tool_choice: Optional[  # type: ignore
+        Union[
+            Literal["none", "auto", "required"],
+            OpenAIChatCompletionNamedToolChoiceParam,
+        ]
+    ]
+    tools: Optional[Iterable[OpenAIChatCompletionToolParam]]  # type: ignore
+    top_logprobs: Optional[int]
+    top_p: Optional[float]
+    user: Optional[str]

xinference/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2024-09-03T15:42:58+0800",
+ "date": "2024-09-14T13:22:13+0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "b1b7c44e6f0ad934eb8366d531c87f29cfa239a7",
- "version": "0.14.4.post1"
+ "full-revisionid": "961d355102007e3cd7963a353105b2422a31d4fd",
+ "version": "0.15.1"
 }
 '''  # END VERSION_JSON
 

xinference/api/restful_api.py

@@ -57,14 +57,13 @@ from ..core.event import Event, EventCollectorActor, EventType
 from ..core.supervisor import SupervisorActor
 from ..core.utils import json_dumps
 from ..types import (
-    SPECIAL_TOOL_PROMPT,
     ChatCompletion,
-    ChatCompletionMessage,
     Completion,
     CreateChatCompletion,
     CreateCompletion,
     ImageList,
     PeftModelConfig,
+    SDAPIResult,
     VideoList,
     max_tokens_field,
 )
@@ -124,6 +123,43 @@ class TextToImageRequest(BaseModel):
     user: Optional[str] = None
 
 
+class SDAPIOptionsRequest(BaseModel):
+    sd_model_checkpoint: Optional[str] = None
+
+
+class SDAPITxt2imgRequst(BaseModel):
+    model: Optional[str]
+    prompt: Optional[str] = ""
+    negative_prompt: Optional[str] = ""
+    steps: Optional[int] = None
+    seed: Optional[int] = -1
+    cfg_scale: Optional[float] = 7.0
+    override_settings: Optional[dict] = {}
+    width: Optional[int] = 512
+    height: Optional[int] = 512
+    sampler_name: Optional[str] = None
+    denoising_strength: Optional[float] = None
+    kwargs: Optional[str] = None
+    user: Optional[str] = None
+
+
+class SDAPIImg2imgRequst(BaseModel):
+    model: Optional[str]
+    init_images: Optional[list]
+    prompt: Optional[str] = ""
+    negative_prompt: Optional[str] = ""
+    steps: Optional[int] = None
+    seed: Optional[int] = -1
+    cfg_scale: Optional[float] = 7.0
+    override_settings: Optional[dict] = {}
+    width: Optional[int] = 512
+    height: Optional[int] = 512
+    sampler_name: Optional[str] = None
+    denoising_strength: Optional[float] = None
+    kwargs: Optional[str] = None
+    user: Optional[str] = None
+
+
 class TextToVideoRequest(BaseModel):
     model: str
     prompt: Union[str, List[str]] = Field(description="The input to embed.")
@@ -165,7 +201,7 @@ class BuildGradioImageInterfaceRequest(BaseModel):
     model_name: str
     model_family: str
     model_id: str
-    controlnet: Union[None, List[Dict[str, Union[str, None]]]]
+    controlnet: Union[None, List[Dict[str, Union[str, dict, None]]]]
     model_revision: str
     model_ability: List[str]
 
@@ -199,14 +235,14 @@ class RESTfulAPI:
     async def _get_supervisor_ref(self) -> xo.ActorRefType[SupervisorActor]:
         if self._supervisor_ref is None:
             self._supervisor_ref = await xo.actor_ref(
-                address=self._supervisor_address, uid=SupervisorActor.uid()
+                address=self._supervisor_address, uid=SupervisorActor.default_uid()
             )
         return self._supervisor_ref
 
     async def _get_event_collector_ref(self) -> xo.ActorRefType[EventCollectorActor]:
         if self._event_collector_ref is None:
             self._event_collector_ref = await xo.actor_ref(
-                address=self._supervisor_address, uid=EventCollectorActor.uid()
+                address=self._supervisor_address, uid=EventCollectorActor.default_uid()
             )
         return self._event_collector_ref
 
@@ -521,6 +557,59 @@ class RESTfulAPI:
                 else None
             ),
         )
+        # SD WebUI API
+        self._router.add_api_route(
+            "/sdapi/v1/options",
+            self.sdapi_options,
+            methods=["POST"],
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:read"])]
+                if self.is_authenticated()
+                else None
+            ),
+        )
+        self._router.add_api_route(
+            "/sdapi/v1/sd-models",
+            self.sdapi_sd_models,
+            methods=["GET"],
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:read"])]
+                if self.is_authenticated()
+                else None
+            ),
+        )
+        self._router.add_api_route(
+            "/sdapi/v1/samplers",
+            self.sdapi_samplers,
+            methods=["GET"],
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:read"])]
+                if self.is_authenticated()
+                else None
+            ),
+        )
+        self._router.add_api_route(
+            "/sdapi/v1/txt2img",
+            self.sdapi_txt2img,
+            methods=["POST"],
+            response_model=SDAPIResult,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:read"])]
+                if self.is_authenticated()
+                else None
+            ),
+        )
+        self._router.add_api_route(
+            "/sdapi/v1/img2img",
+            self.sdapi_img2img,
+            methods=["POST"],
+            response_model=SDAPIResult,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:read"])]
+                if self.is_authenticated()
+                else None
+            ),
+        )
         self._router.add_api_route(
             "/v1/video/generations",
             self.create_videos,
@@ -1431,6 +1520,118 @@ class RESTfulAPI:
             await self._report_error_event(model_uid, str(e))
             raise HTTPException(status_code=500, detail=str(e))
 
+    async def sdapi_options(self, request: Request) -> Response:
+        body = SDAPIOptionsRequest.parse_obj(await request.json())
+        model_uid = body.sd_model_checkpoint
+
+        try:
+            if not model_uid:
+                raise ValueError("Unknown model")
+            await (await self._get_supervisor_ref()).get_model(model_uid)
+            return Response()
+        except ValueError as ve:
+            logger.error(str(ve), exc_info=True)
+            await self._report_error_event(model_uid, str(ve))
+            raise HTTPException(status_code=400, detail=str(ve))
+        except Exception as e:
+            logger.error(e, exc_info=True)
+            await self._report_error_event(model_uid, str(e))
+            raise HTTPException(status_code=500, detail=str(e))
+
+    async def sdapi_sd_models(self, request: Request) -> Response:
+        try:
+            models = await (await self._get_supervisor_ref()).list_models()
+            sd_models = []
+            for model_name, info in models.items():
+                if info["model_type"] != "image":
+                    continue
+                sd_models.append({"model_name": model_name, "config": None})
+            return JSONResponse(content=sd_models)
+        except Exception as e:
+            logger.error(e, exc_info=True)
+            raise HTTPException(status_code=500, detail=str(e))
+
+    async def sdapi_samplers(self, request: Request) -> Response:
+        try:
+            from ..model.image.stable_diffusion.core import SAMPLING_METHODS
+
+            samplers = [
+                {"name": sample_method, "alias": [], "options": {}}
+                for sample_method in SAMPLING_METHODS
+            ]
+            return JSONResponse(content=samplers)
+        except Exception as e:
+            logger.error(e, exc_info=True)
+            raise HTTPException(status_code=500, detail=str(e))
+
+    async def sdapi_txt2img(self, request: Request) -> Response:
+        body = SDAPITxt2imgRequst.parse_obj(await request.json())
+        model_uid = body.model or body.override_settings.get("sd_model_checkpoint")
+
+        try:
+            if not model_uid:
+                raise ValueError("Unknown model")
+            model = await (await self._get_supervisor_ref()).get_model(model_uid)
+        except ValueError as ve:
+            logger.error(str(ve), exc_info=True)
+            await self._report_error_event(model_uid, str(ve))
+            raise HTTPException(status_code=400, detail=str(ve))
+        except Exception as e:
+            logger.error(e, exc_info=True)
+            await self._report_error_event(model_uid, str(e))
+            raise HTTPException(status_code=500, detail=str(e))
+
+        try:
+            kwargs = dict(body)
+            kwargs.update(json.loads(body.kwargs) if body.kwargs else {})
+            image_list = await model.txt2img(
+                **kwargs,
+            )
+            return Response(content=image_list, media_type="application/json")
+        except RuntimeError as re:
+            logger.error(re, exc_info=True)
+            await self._report_error_event(model_uid, str(re))
+            self.handle_request_limit_error(re)
+            raise HTTPException(status_code=400, detail=str(re))
+        except Exception as e:
+            logger.error(e, exc_info=True)
+            await self._report_error_event(model_uid, str(e))
+            raise HTTPException(status_code=500, detail=str(e))
+
+    async def sdapi_img2img(self, request: Request) -> Response:
+        body = SDAPIImg2imgRequst.parse_obj(await request.json())
+        model_uid = body.model or body.override_settings.get("sd_model_checkpoint")
+
+        try:
+            if not model_uid:
+                raise ValueError("Unknown model")
+            model = await (await self._get_supervisor_ref()).get_model(model_uid)
+        except ValueError as ve:
+            logger.error(str(ve), exc_info=True)
+            await self._report_error_event(model_uid, str(ve))
+            raise HTTPException(status_code=400, detail=str(ve))
+        except Exception as e:
+            logger.error(e, exc_info=True)
+            await self._report_error_event(model_uid, str(e))
+            raise HTTPException(status_code=500, detail=str(e))
+
+        try:
+            kwargs = dict(body)
+            kwargs.update(json.loads(body.kwargs) if body.kwargs else {})
+            image_list = await model.img2img(
+                **kwargs,
+            )
+            return Response(content=image_list, media_type="application/json")
+        except RuntimeError as re:
+            logger.error(re, exc_info=True)
+            await self._report_error_event(model_uid, str(re))
+            self.handle_request_limit_error(re)
+            raise HTTPException(status_code=400, detail=str(re))
+        except Exception as e:
+            logger.error(e, exc_info=True)
+            await self._report_error_event(model_uid, str(e))
+            raise HTTPException(status_code=500, detail=str(e))
+
     async def create_variations(
         self,
         model: str = Form(...),
@@ -1627,33 +1828,7 @@ class RESTfulAPI:
                 status_code=400, detail="Invalid input. Please specify the prompt."
             )
 
-        system_messages: List["ChatCompletionMessage"] = []
-        system_messages_contents = []
-        non_system_messages = []
-        for msg in messages:
-            assert (
-                msg.get("content") != SPECIAL_TOOL_PROMPT
-            ), f"Invalid message content {SPECIAL_TOOL_PROMPT}"
-            if msg["role"] == "system":
-                system_messages_contents.append(msg["content"])
-            else:
-                non_system_messages.append(msg)
-        system_messages.append(
-            {"role": "system", "content": ". ".join(system_messages_contents)}
-        )
-
         has_tool_message = messages[-1].get("role") == "tool"
-        if has_tool_message:
-            prompt = SPECIAL_TOOL_PROMPT
-            system_prompt = system_messages[0]["content"] if system_messages else None
-            chat_history = non_system_messages  # exclude the prompt
-        else:
-            prompt = None
-            if non_system_messages:
-                prompt = non_system_messages[-1]["content"]
-            system_prompt = system_messages[0]["content"] if system_messages else None
-            chat_history = non_system_messages[:-1]  # exclude the prompt
-
         model_uid = body.model
 
         try:
@@ -1681,9 +1856,7 @@ class RESTfulAPI:
         from ..model.llm.utils import GLM4_TOOL_CALL_FAMILY, QWEN_TOOL_CALL_FAMILY
 
         model_family = desc.get("model_family", "")
-        function_call_models = (
-            ["gorilla-openfunctions-v1"] + QWEN_TOOL_CALL_FAMILY + GLM4_TOOL_CALL_FAMILY
-        )
+        function_call_models = QWEN_TOOL_CALL_FAMILY + GLM4_TOOL_CALL_FAMILY
 
         if model_family not in function_call_models:
             if body.tools:
@@ -1716,9 +1889,7 @@ class RESTfulAPI:
                 try:
                     try:
                         iterator = await model.chat(
-                            prompt,
-                            system_prompt,
-                            chat_history,
+                            messages,
                             kwargs,
                             raw_params=raw_kwargs,
                         )
@@ -1750,9 +1921,7 @@ class RESTfulAPI:
         else:
             try:
                 data = await model.chat(
-                    prompt,
-                    system_prompt,
-                    chat_history,
+                    messages,
                     kwargs,
                     raw_params=raw_kwargs,
                 )

xinference/client/restful/restful_client.py

@@ -13,7 +13,6 @@
 # limitations under the License.
 import json
 import typing
-import warnings
 from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Union
 
 import requests
@@ -470,9 +469,7 @@ class RESTfulGenerateModelHandle(RESTfulModelHandle):
 class RESTfulChatModelHandle(RESTfulGenerateModelHandle):
     def chat(
         self,
-        prompt: str,
-        system_prompt: Optional[str] = None,
-        chat_history: Optional[List["ChatCompletionMessage"]] = None,
+        messages: List[Dict],
         tools: Optional[List[Dict]] = None,
         generate_config: Optional[
             Union["LlamaCppGenerateConfig", "PytorchGenerateConfig"]
@@ -483,11 +480,7 @@ class RESTfulChatModelHandle(RESTfulGenerateModelHandle):
 
         Parameters
         ----------
-        prompt: str
-            The user's input.
-        system_prompt: Optional[str]
-            The system context provide to Model prior to any chats.
-        chat_history: Optional[List["ChatCompletionMessage"]]
+        messages: List[Dict]
             A list of messages comprising the conversation so far.
         tools: Optional[List[Dict]]
             A tool list.
@@ -509,25 +502,11 @@ class RESTfulChatModelHandle(RESTfulGenerateModelHandle):
             Report the failure to generate the chat from the server. Detailed information provided in error message.
 
         """
-        warnings.warn(
-            "The parameters `prompt`, `system_prompt` and `chat_history` will be deprecated in version v0.15.0, "
-            "and will be replaced by the parameter `messages`, "
-            "similar to the OpenAI API: https://platform.openai.com/docs/guides/chat-completions/getting-started",
-            category=DeprecationWarning,
-            stacklevel=2,
-        )
-
         url = f"{self._base_url}/v1/chat/completions"
 
-        if chat_history is None:
-            chat_history = []
-
-        chat_history = handle_system_prompts(chat_history, system_prompt)
-        chat_history.append({"role": "user", "content": prompt})  # type: ignore
-
         request_body: Dict[str, Any] = {
             "model": self._model_uid,
-            "messages": chat_history,
+            "messages": messages,
         }
         if tools is not None:
             request_body["tools"] = tools
@@ -730,10 +709,12 @@ class RESTfulAudioModelHandle(RESTfulModelHandle):
                 )
             )
             response = requests.post(
-                url, data=params, files=files, headers=self.auth_headers
+                url, data=params, files=files, headers=self.auth_headers, stream=stream
             )
         else:
-            response = requests.post(url, json=params, headers=self.auth_headers)
+            response = requests.post(
+                url, json=params, headers=self.auth_headers, stream=stream
+            )
         if response.status_code != 200:
             raise RuntimeError(
                 f"Failed to speech the text, detail: {_get_error_string(response)}"

xinference/conftest.py

@@ -144,7 +144,7 @@ async def _start_test_cluster(
             address=f"test://{address}", logging_conf=logging_conf
         )
         await xo.create_actor(
-            SupervisorActor, address=address, uid=SupervisorActor.uid()
+            SupervisorActor, address=address, uid=SupervisorActor.default_uid()
         )
         await start_worker_components(
             address=address,

xinference/constants.py

@@ -38,6 +38,10 @@ def get_xinference_home() -> str:
         # if user has already set `XINFERENCE_HOME` env, change huggingface and modelscope default download path
         os.environ["HUGGINGFACE_HUB_CACHE"] = os.path.join(home_path, "huggingface")
         os.environ["MODELSCOPE_CACHE"] = os.path.join(home_path, "modelscope")
+    # In multi-tenant mode,
+    # gradio's temporary files are stored in their respective home directories,
+    # to prevent insufficient permissions
+    os.environ["GRADIO_TEMP_DIR"] = os.path.join(home_path, "tmp", "gradio")
     return home_path
 
 
@@ -59,6 +63,7 @@ XINFERENCE_DEFAULT_ENDPOINT_PORT = 9997
 XINFERENCE_DEFAULT_LOG_FILE_NAME = "xinference.log"
 XINFERENCE_LOG_MAX_BYTES = 100 * 1024 * 1024
 XINFERENCE_LOG_BACKUP_COUNT = 30
+XINFERENCE_LOG_ARG_MAX_LENGTH = 100
 XINFERENCE_HEALTH_CHECK_FAILURE_THRESHOLD = int(
     os.environ.get(XINFERENCE_ENV_HEALTH_CHECK_FAILURE_THRESHOLD, 5)
 )

xinference/core/cache_tracker.py

@@ -25,7 +25,7 @@ class CacheTrackerActor(xo.Actor):
         self._model_name_to_version_info: Dict[str, List[Dict]] = {}  # type: ignore
 
     @classmethod
-    def uid(cls) -> str:
+    def default_uid(cls) -> str:
         return "cache_tracker"
 
     @staticmethod

xinference/core/chat_interface.py

@@ -16,7 +16,7 @@ import base64
 import logging
 import os
 from io import BytesIO
-from typing import Generator, List, Optional
+from typing import Dict, Generator, List, Optional
 
 import gradio as gr
 import PIL.Image
@@ -27,7 +27,6 @@ from ..client.restful.restful_client import (
     RESTfulChatModelHandle,
     RESTfulGenerateModelHandle,
 )
-from ..types import ChatCompletionMessage
 
 logger = logging.getLogger(__name__)
 
@@ -96,11 +95,11 @@ class GradioInterface:
                 flat_list += row
             return flat_list
 
-        def to_chat(lst: List[str]) -> List[ChatCompletionMessage]:
+        def to_chat(lst: List[str]) -> List[Dict]:
             res = []
             for i in range(len(lst)):
                 role = "assistant" if i % 2 == 1 else "user"
-                res.append(ChatCompletionMessage(role=role, content=lst[i]))
+                res.append(dict(role=role, content=lst[i]))
             return res
 
         def generate_wrapper(
@@ -116,11 +115,12 @@ class GradioInterface:
             client._set_token(self._access_token)
             model = client.get_model(self.model_uid)
             assert isinstance(model, RESTfulChatModelHandle)
+            messages = to_chat(flatten(history))
+            messages.append(dict(role="user", content=message))
 
             response_content = ""
             for chunk in model.chat(
-                prompt=message,
-                chat_history=to_chat(flatten(history)),
+                messages,
                 generate_config={
                     "max_tokens": int(max_tokens),
                     "temperature": temperature,
@@ -191,15 +191,10 @@ class GradioInterface:
             model = client.get_model(self.model_uid)
             assert isinstance(model, RESTfulChatModelHandle)
 
-            prompt = history[-1]
-            assert prompt["role"] == "user"
-            prompt = prompt["content"]
-            # multimodal chat does not support stream.
             if stream:
                 response_content = ""
                 for chunk in model.chat(
-                    prompt=prompt,
-                    chat_history=history[:-1],
+                    messages=history,
                     generate_config={
                         "max_tokens": max_tokens,
                         "temperature": temperature,
@@ -224,8 +219,7 @@ class GradioInterface:
                 yield history, bot
             else:
                 response = model.chat(
-                    prompt=prompt,
-                    chat_history=history[:-1],
+                    messages=history,
                     generate_config={
                         "max_tokens": max_tokens,
                         "temperature": temperature,

xinference/core/event.py

@@ -41,7 +41,7 @@ class EventCollectorActor(xo.StatelessActor):
         )
 
     @classmethod
-    def uid(cls) -> str:
+    def default_uid(cls) -> str:
         return "event_collector"
 
     def get_model_events(self, model_uid: str) -> List[Dict]: