xinference 0.15.0__py3-none-any.whl → 0.15.1__py3-none-any.whl

xinference/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2024-09-06T16:29:42+0800",
+ "date": "2024-09-14T13:22:13+0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "e2618be96293f112709c9ceed639a3443455a0e7",
- "version": "0.15.0"
+ "full-revisionid": "961d355102007e3cd7963a353105b2422a31d4fd",
+ "version": "0.15.1"
 }
 '''  # END VERSION_JSON
 

xinference/api/restful_api.py

@@ -63,6 +63,7 @@ from ..types import (
     CreateCompletion,
     ImageList,
     PeftModelConfig,
+    SDAPIResult,
     VideoList,
     max_tokens_field,
 )
@@ -122,6 +123,43 @@ class TextToImageRequest(BaseModel):
     user: Optional[str] = None
 
 
+class SDAPIOptionsRequest(BaseModel):
+    sd_model_checkpoint: Optional[str] = None
+
+
+class SDAPITxt2imgRequst(BaseModel):
+    model: Optional[str]
+    prompt: Optional[str] = ""
+    negative_prompt: Optional[str] = ""
+    steps: Optional[int] = None
+    seed: Optional[int] = -1
+    cfg_scale: Optional[float] = 7.0
+    override_settings: Optional[dict] = {}
+    width: Optional[int] = 512
+    height: Optional[int] = 512
+    sampler_name: Optional[str] = None
+    denoising_strength: Optional[float] = None
+    kwargs: Optional[str] = None
+    user: Optional[str] = None
+
+
+class SDAPIImg2imgRequst(BaseModel):
+    model: Optional[str]
+    init_images: Optional[list]
+    prompt: Optional[str] = ""
+    negative_prompt: Optional[str] = ""
+    steps: Optional[int] = None
+    seed: Optional[int] = -1
+    cfg_scale: Optional[float] = 7.0
+    override_settings: Optional[dict] = {}
+    width: Optional[int] = 512
+    height: Optional[int] = 512
+    sampler_name: Optional[str] = None
+    denoising_strength: Optional[float] = None
+    kwargs: Optional[str] = None
+    user: Optional[str] = None
+
+
 class TextToVideoRequest(BaseModel):
     model: str
     prompt: Union[str, List[str]] = Field(description="The input to embed.")
@@ -163,7 +201,7 @@ class BuildGradioImageInterfaceRequest(BaseModel):
     model_name: str
     model_family: str
     model_id: str
-    controlnet: Union[None, List[Dict[str, Union[str, None]]]]
+    controlnet: Union[None, List[Dict[str, Union[str, dict, None]]]]
     model_revision: str
     model_ability: List[str]
 
@@ -519,6 +557,59 @@ class RESTfulAPI:
                 else None
             ),
         )
+        # SD WebUI API
+        self._router.add_api_route(
+            "/sdapi/v1/options",
+            self.sdapi_options,
+            methods=["POST"],
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:read"])]
+                if self.is_authenticated()
+                else None
+            ),
+        )
+        self._router.add_api_route(
+            "/sdapi/v1/sd-models",
+            self.sdapi_sd_models,
+            methods=["GET"],
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:read"])]
+                if self.is_authenticated()
+                else None
+            ),
+        )
+        self._router.add_api_route(
+            "/sdapi/v1/samplers",
+            self.sdapi_samplers,
+            methods=["GET"],
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:read"])]
+                if self.is_authenticated()
+                else None
+            ),
+        )
+        self._router.add_api_route(
+            "/sdapi/v1/txt2img",
+            self.sdapi_txt2img,
+            methods=["POST"],
+            response_model=SDAPIResult,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:read"])]
+                if self.is_authenticated()
+                else None
+            ),
+        )
+        self._router.add_api_route(
+            "/sdapi/v1/img2img",
+            self.sdapi_img2img,
+            methods=["POST"],
+            response_model=SDAPIResult,
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:read"])]
+                if self.is_authenticated()
+                else None
+            ),
+        )
         self._router.add_api_route(
             "/v1/video/generations",
             self.create_videos,
@@ -1429,6 +1520,118 @@ class RESTfulAPI:
             await self._report_error_event(model_uid, str(e))
             raise HTTPException(status_code=500, detail=str(e))
 
+    async def sdapi_options(self, request: Request) -> Response:
+        body = SDAPIOptionsRequest.parse_obj(await request.json())
+        model_uid = body.sd_model_checkpoint
+
+        try:
+            if not model_uid:
+                raise ValueError("Unknown model")
+            await (await self._get_supervisor_ref()).get_model(model_uid)
+            return Response()
+        except ValueError as ve:
+            logger.error(str(ve), exc_info=True)
+            await self._report_error_event(model_uid, str(ve))
+            raise HTTPException(status_code=400, detail=str(ve))
+        except Exception as e:
+            logger.error(e, exc_info=True)
+            await self._report_error_event(model_uid, str(e))
+            raise HTTPException(status_code=500, detail=str(e))
+
+    async def sdapi_sd_models(self, request: Request) -> Response:
+        try:
+            models = await (await self._get_supervisor_ref()).list_models()
+            sd_models = []
+            for model_name, info in models.items():
+                if info["model_type"] != "image":
+                    continue
+                sd_models.append({"model_name": model_name, "config": None})
+            return JSONResponse(content=sd_models)
+        except Exception as e:
+            logger.error(e, exc_info=True)
+            raise HTTPException(status_code=500, detail=str(e))
+
+    async def sdapi_samplers(self, request: Request) -> Response:
+        try:
+            from ..model.image.stable_diffusion.core import SAMPLING_METHODS
+
+            samplers = [
+                {"name": sample_method, "alias": [], "options": {}}
+                for sample_method in SAMPLING_METHODS
+            ]
+            return JSONResponse(content=samplers)
+        except Exception as e:
+            logger.error(e, exc_info=True)
+            raise HTTPException(status_code=500, detail=str(e))
+
+    async def sdapi_txt2img(self, request: Request) -> Response:
+        body = SDAPITxt2imgRequst.parse_obj(await request.json())
+        model_uid = body.model or body.override_settings.get("sd_model_checkpoint")
+
+        try:
+            if not model_uid:
+                raise ValueError("Unknown model")
+            model = await (await self._get_supervisor_ref()).get_model(model_uid)
+        except ValueError as ve:
+            logger.error(str(ve), exc_info=True)
+            await self._report_error_event(model_uid, str(ve))
+            raise HTTPException(status_code=400, detail=str(ve))
+        except Exception as e:
+            logger.error(e, exc_info=True)
+            await self._report_error_event(model_uid, str(e))
+            raise HTTPException(status_code=500, detail=str(e))
+
+        try:
+            kwargs = dict(body)
+            kwargs.update(json.loads(body.kwargs) if body.kwargs else {})
+            image_list = await model.txt2img(
+                **kwargs,
+            )
+            return Response(content=image_list, media_type="application/json")
+        except RuntimeError as re:
+            logger.error(re, exc_info=True)
+            await self._report_error_event(model_uid, str(re))
+            self.handle_request_limit_error(re)
+            raise HTTPException(status_code=400, detail=str(re))
+        except Exception as e:
+            logger.error(e, exc_info=True)
+            await self._report_error_event(model_uid, str(e))
+            raise HTTPException(status_code=500, detail=str(e))
+
+    async def sdapi_img2img(self, request: Request) -> Response:
+        body = SDAPIImg2imgRequst.parse_obj(await request.json())
+        model_uid = body.model or body.override_settings.get("sd_model_checkpoint")
+
+        try:
+            if not model_uid:
+                raise ValueError("Unknown model")
+            model = await (await self._get_supervisor_ref()).get_model(model_uid)
+        except ValueError as ve:
+            logger.error(str(ve), exc_info=True)
+            await self._report_error_event(model_uid, str(ve))
+            raise HTTPException(status_code=400, detail=str(ve))
+        except Exception as e:
+            logger.error(e, exc_info=True)
+            await self._report_error_event(model_uid, str(e))
+            raise HTTPException(status_code=500, detail=str(e))
+
+        try:
+            kwargs = dict(body)
+            kwargs.update(json.loads(body.kwargs) if body.kwargs else {})
+            image_list = await model.img2img(
+                **kwargs,
+            )
+            return Response(content=image_list, media_type="application/json")
+        except RuntimeError as re:
+            logger.error(re, exc_info=True)
+            await self._report_error_event(model_uid, str(re))
+            self.handle_request_limit_error(re)
+            raise HTTPException(status_code=400, detail=str(re))
+        except Exception as e:
+            logger.error(e, exc_info=True)
+            await self._report_error_event(model_uid, str(e))
+            raise HTTPException(status_code=500, detail=str(e))
+
     async def create_variations(
         self,
         model: str = Form(...),

xinference/client/restful/restful_client.py

@@ -709,10 +709,12 @@ class RESTfulAudioModelHandle(RESTfulModelHandle):
                 )
             )
             response = requests.post(
-                url, data=params, files=files, headers=self.auth_headers
+                url, data=params, files=files, headers=self.auth_headers, stream=stream
             )
         else:
-            response = requests.post(url, json=params, headers=self.auth_headers)
+            response = requests.post(
+                url, json=params, headers=self.auth_headers, stream=stream
+            )
         if response.status_code != 200:
             raise RuntimeError(
                 f"Failed to speech the text, detail: {_get_error_string(response)}"

xinference/core/image_interface.py

@@ -73,13 +73,17 @@ class ImageInterface:
         return interface
 
     def text2image_interface(self) -> "gr.Blocks":
+        from ..model.image.stable_diffusion.core import SAMPLING_METHODS
+
         def text_generate_image(
             prompt: str,
             n: int,
             size_width: int,
             size_height: int,
+            guidance_scale: int,
             num_inference_steps: int,
             negative_prompt: Optional[str] = None,
+            sampler_name: Optional[str] = None,
         ) -> PIL.Image.Image:
             from ..client import RESTfulClient
 
@@ -89,16 +93,20 @@ class ImageInterface:
             assert isinstance(model, RESTfulImageModelHandle)
 
             size = f"{int(size_width)}*{int(size_height)}"
+            guidance_scale = None if guidance_scale == -1 else guidance_scale  # type: ignore
             num_inference_steps = (
                 None if num_inference_steps == -1 else num_inference_steps  # type: ignore
             )
+            sampler_name = None if sampler_name == "default" else sampler_name
 
             response = model.text_to_image(
                 prompt=prompt,
                 n=n,
                 size=size,
                 num_inference_steps=num_inference_steps,
+                guidance_scale=guidance_scale,
                 negative_prompt=negative_prompt,
+                sampler_name=sampler_name,
                 response_format="b64_json",
             )
 
@@ -132,9 +140,16 @@ class ImageInterface:
                     n = gr.Number(label="Number of Images", value=1)
                     size_width = gr.Number(label="Width", value=1024)
                     size_height = gr.Number(label="Height", value=1024)
+                with gr.Row():
+                    guidance_scale = gr.Number(label="Guidance scale", value=-1)
                     num_inference_steps = gr.Number(
                         label="Inference Step Number", value=-1
                     )
+                    sampler_name = gr.Dropdown(
+                        choices=SAMPLING_METHODS,
+                        value="default",
+                        label="Sampling method",
+                    )
 
                 with gr.Column():
                     image_output = gr.Gallery()
@@ -146,8 +161,10 @@ class ImageInterface:
                     n,
                     size_width,
                     size_height,
+                    guidance_scale,
                     num_inference_steps,
                     negative_prompt,
+                    sampler_name,
                 ],
                 outputs=image_output,
             )
@@ -155,6 +172,8 @@ class ImageInterface:
         return text2image_vl_interface
 
     def image2image_interface(self) -> "gr.Blocks":
+        from ..model.image.stable_diffusion.core import SAMPLING_METHODS
+
         def image_generate_image(
             prompt: str,
             negative_prompt: str,
@@ -164,6 +183,7 @@ class ImageInterface:
             size_height: int,
             num_inference_steps: int,
             padding_image_to_multiple: int,
+            sampler_name: Optional[str] = None,
         ) -> PIL.Image.Image:
             from ..client import RESTfulClient
 
@@ -180,6 +200,7 @@ class ImageInterface:
                 None if num_inference_steps == -1 else num_inference_steps  # type: ignore
             )
             padding_image_to_multiple = None if padding_image_to_multiple == -1 else padding_image_to_multiple  # type: ignore
+            sampler_name = None if sampler_name == "default" else sampler_name
 
             bio = io.BytesIO()
             image.save(bio, format="png")
@@ -193,6 +214,7 @@ class ImageInterface:
                 response_format="b64_json",
                 num_inference_steps=num_inference_steps,
                 padding_image_to_multiple=padding_image_to_multiple,
+                sampler_name=sampler_name,
             )
 
             images = []
@@ -233,6 +255,11 @@ class ImageInterface:
                     padding_image_to_multiple = gr.Number(
                         label="Padding image to multiple", value=-1
                     )
+                    sampler_name = gr.Dropdown(
+                        choices=SAMPLING_METHODS,
+                        value="default",
+                        label="Sampling method",
+                    )
 
                 with gr.Row():
                     with gr.Column(scale=1):
@@ -251,6 +278,7 @@ class ImageInterface:
                     size_height,
                     num_inference_steps,
                     padding_image_to_multiple,
+                    sampler_name,
                 ],
                 outputs=output_gallery,
             )

xinference/core/model.py

@@ -747,6 +747,20 @@ class ModelActor(xo.StatelessActor):
             f"Model {self._model.model_spec} is not for creating image."
         )
 
+    @request_limit
+    @log_async(logger=logger)
+    async def txt2img(
+        self,
+        **kwargs,
+    ):
+        kwargs.pop("request_id", None)
+        if hasattr(self._model, "txt2img"):
+            return await self._call_wrapper_json(
+                self._model.txt2img,
+                **kwargs,
+            )
+        raise AttributeError(f"Model {self._model.model_spec} is not for txt2img.")
+
     @log_async(
         logger=logger,
         ignore_kwargs=["image"],
@@ -779,6 +793,20 @@ class ModelActor(xo.StatelessActor):
             f"Model {self._model.model_spec} is not for creating image."
         )
 
+    @request_limit
+    @log_async(logger=logger)
+    async def img2img(
+        self,
+        **kwargs,
+    ):
+        kwargs.pop("request_id", None)
+        if hasattr(self._model, "img2img"):
+            return await self._call_wrapper_json(
+                self._model.img2img,
+                **kwargs,
+            )
+        raise AttributeError(f"Model {self._model.model_spec} is not for img2img.")
+
     @log_async(
         logger=logger,
         ignore_kwargs=["image"],

xinference/core/supervisor.py

@@ -315,6 +315,7 @@ class SupervisorActor(xo.StatelessActor):
     @staticmethod
     async def get_builtin_families() -> Dict[str, List[str]]:
         from ..model.llm.llm_family import (
+            BUILTIN_LLM_FAMILIES,
             BUILTIN_LLM_MODEL_CHAT_FAMILIES,
             BUILTIN_LLM_MODEL_GENERATE_FAMILIES,
             BUILTIN_LLM_MODEL_TOOL_CALL_FAMILIES,
@@ -324,6 +325,11 @@ class SupervisorActor(xo.StatelessActor):
             "chat": list(BUILTIN_LLM_MODEL_CHAT_FAMILIES),
             "generate": list(BUILTIN_LLM_MODEL_GENERATE_FAMILIES),
             "tools": list(BUILTIN_LLM_MODEL_TOOL_CALL_FAMILIES),
+            "vision": [
+                family.model_name
+                for family in BUILTIN_LLM_FAMILIES
+                if "vision" in family.model_ability
+            ],
         }
 
     async def get_devices_count(self) -> int:

xinference/model/audio/fish_speech.py

@@ -92,7 +92,7 @@ class FishSpeechModel:
 
         checkpoint_path = os.path.join(
             self._model_path,
-            "firefly-gan-vq-fsq-4x1024-42hz-generator.pth",
+            "firefly-gan-vq-fsq-8x1024-21hz-generator.pth",
         )
         self._model = load_decoder_model(
             config_name="firefly_gan_vq",
@@ -159,11 +159,11 @@ class FishSpeechModel:
         segments = []
 
         while True:
-            result: WrappedGenerateResponse = response_queue.get()
+            result: WrappedGenerateResponse = response_queue.get()  # type: ignore
             if result.status == "error":
                 raise Exception(str(result.response))
 
-            result: GenerateResponse = result.response
+            result: GenerateResponse = result.response  # type: ignore
             if result.action == "next":
                 break
 
@@ -213,12 +213,12 @@ class FishSpeechModel:
                 text=input,
                 enable_reference_audio=False,
                 reference_audio=None,
-                reference_text="",
-                max_new_tokens=0,
-                chunk_length=100,
-                top_p=0.7,
-                repetition_penalty=1.2,
-                temperature=0.7,
+                reference_text=kwargs.get("reference_text", ""),
+                max_new_tokens=kwargs.get("max_new_tokens", 1024),
+                chunk_length=kwargs.get("chunk_length", 200),
+                top_p=kwargs.get("top_p", 0.7),
+                repetition_penalty=kwargs.get("repetition_penalty", 1.2),
+                temperature=kwargs.get("temperature", 0.7),
             )
         )
         sample_rate, audio = result[0][1]

xinference/model/audio/model_spec.json

@@ -126,32 +126,32 @@
   {
     "model_name": "CosyVoice-300M",
     "model_family": "CosyVoice",
-    "model_id": "model-scope/CosyVoice-300M",
-    "model_revision": "ca4e036d2db2aa4731cc1747859a68044b6a4694",
+    "model_id": "FunAudioLLM/CosyVoice-300M",
+    "model_revision": "39c4e13d46bd4dfb840d214547623e5fcd2428e2",
     "model_ability": "audio-to-audio",
     "multilingual": true
   },
   {
     "model_name": "CosyVoice-300M-SFT",
     "model_family": "CosyVoice",
-    "model_id": "model-scope/CosyVoice-300M-SFT",
-    "model_revision": "ab918940c6c134b1fc1f069246e67bad6b66abcb",
+    "model_id": "FunAudioLLM/CosyVoice-300M-SFT",
+    "model_revision": "096a5cff8d497fabb3dec2756a200f3688457a1b",
     "model_ability": "text-to-audio",
     "multilingual": true
   },
   {
     "model_name": "CosyVoice-300M-Instruct",
     "model_family": "CosyVoice",
-    "model_id": "model-scope/CosyVoice-300M-Instruct",
-    "model_revision": "fb5f676733139f35670bed9b59a77d476b1aa898",
+    "model_id": "FunAudioLLM/CosyVoice-300M-Instruct",
+    "model_revision": "ba5265d9a3169c1fedce145122c9dd4bc24e062c",
     "model_ability": "text-to-audio",
     "multilingual": true
   },
   {
-    "model_name": "FishSpeech-1.2-SFT",
+    "model_name": "FishSpeech-1.4",
     "model_family": "FishAudio",
-    "model_id": "fishaudio/fish-speech-1.2-sft",
-    "model_revision": "180288e21ec5c50cfc564023a22f789e4b88a0e0",
+    "model_id": "fishaudio/fish-speech-1.4",
+    "model_revision": "3c49651b8e583b6b13f55e375432e0d57e1aa84d",
     "model_ability": "text-to-audio",
     "multilingual": true
   }

xinference/model/audio/whisper.py

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
+import os
+from glob import glob
 from typing import TYPE_CHECKING, Dict, List, Optional, Union
 
 from ...device_utils import (
@@ -56,12 +58,13 @@ class WhisperModel:
                 raise ValueError(f"Device {self._device} is not available!")
 
         torch_dtype = get_device_preferred_dtype(self._device)
+        use_safetensors = any(glob(os.path.join(self._model_path, "*.safetensors")))
 
         model = AutoModelForSpeechSeq2Seq.from_pretrained(
             self._model_path,
             torch_dtype=torch_dtype,
             low_cpu_mem_usage=True,
-            use_safetensors=True,
+            use_safetensors=use_safetensors,
         )
         model.to(self._device)
 

xinference/model/image/core.py

@@ -47,6 +47,7 @@ class ImageModelFamilyV1(CacheableModelSpec):
     model_hub: str = "huggingface"
     model_ability: Optional[List[str]]
     controlnet: Optional[List["ImageModelFamilyV1"]]
+    default_generate_config: Optional[dict] = {}
 
 
 class ImageModelDescription(ModelDescription):
@@ -238,7 +239,7 @@ def create_image_model_instance(
         lora_model_paths=lora_model,
         lora_load_kwargs=lora_load_kwargs,
         lora_fuse_kwargs=lora_fuse_kwargs,
-        abilities=model_spec.model_ability,
+        model_spec=model_spec,
         **kwargs,
     )
     model_description = ImageModelDescription(

xinference/model/image/model_spec.json

@@ -5,7 +5,9 @@
     "model_id": "black-forest-labs/FLUX.1-schnell",
     "model_revision": "768d12a373ed5cc9ef9a9dea7504dc09fcc14842",
     "model_ability": [
-      "text2image"
+      "text2image",
+      "image2image",
+      "inpainting"
     ]
   },
   {
@@ -14,7 +16,9 @@
     "model_id": "black-forest-labs/FLUX.1-dev",
     "model_revision": "01aa605f2c300568dd6515476f04565a954fcb59",
     "model_ability": [
-      "text2image"
+      "text2image",
+      "image2image",
+      "inpainting"
     ]
   },
   {
@@ -35,7 +39,11 @@
     "model_revision": "1681ed09e0cff58eeb41e878a49893228b78b94c",
     "model_ability": [
       "text2image"
-    ]
+    ],
+    "default_generate_config": {
+      "guidance_scale": 0.0,
+      "num_inference_steps": 1
+    }
   },
   {
     "model_name": "sdxl-turbo",
@@ -44,7 +52,11 @@
     "model_revision": "f4b0486b498f84668e828044de1d0c8ba486e05b",
     "model_ability": [
       "text2image"
-    ]
+    ],
+    "default_generate_config": {
+      "guidance_scale": 0.0,
+      "num_inference_steps": 1
+    }
   },
   {
     "model_name": "stable-diffusion-v1.5",

xinference/model/image/model_spec_modelscope.json

@@ -6,7 +6,9 @@
     "model_id": "AI-ModelScope/FLUX.1-schnell",
     "model_revision": "master",
     "model_ability": [
-      "text2image"
+      "text2image",
+      "image2image",
+      "inpainting"
     ]
   },
   {
@@ -16,7 +18,9 @@
     "model_id": "AI-ModelScope/FLUX.1-dev",
     "model_revision": "master",
     "model_ability": [
-      "text2image"
+      "text2image",
+      "image2image",
+      "inpainting"
     ]
   },
   {
@@ -39,7 +43,11 @@
     "model_revision": "master",
     "model_ability": [
       "text2image"
-    ]
+    ],
+    "default_generate_config": {
+      "guidance_scale": 0.0,
+      "num_inference_steps": 1
+    }
   },
   {
     "model_name": "sdxl-turbo",
@@ -49,7 +57,11 @@
     "model_revision": "master",
     "model_ability": [
       "text2image"
-    ]
+    ],
+    "default_generate_config": {
+      "guidance_scale": 0.0,
+      "num_inference_steps": 1
+    }
   },
   {
     "model_name": "stable-diffusion-v1.5",