xinference 1.0.1__py3-none-any.whl → 1.1.1__py3-none-any.whl

xinference/_compat.py

@@ -72,6 +72,7 @@ OpenAIChatCompletionToolParam = create_model_from_typeddict(ChatCompletionToolPa
 OpenAIChatCompletionNamedToolChoiceParam = create_model_from_typeddict(
     ChatCompletionNamedToolChoiceParam
 )
+from openai._types import Body
 
 
 class JSONSchema(BaseModel):
@@ -120,4 +121,5 @@ class CreateChatCompletionOpenAI(BaseModel):
     tools: Optional[Iterable[OpenAIChatCompletionToolParam]]  # type: ignore
     top_logprobs: Optional[int]
     top_p: Optional[float]
+    extra_body: Optional[Body]
     user: Optional[str]

xinference/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2024-11-29T16:57:04+0800",
+ "date": "2024-12-27T18:14:37+0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "eb8ddd431f5c5fcb2216e25e0d43745f8455d9b9",
- "version": "1.0.1"
+ "full-revisionid": "d3428697115cc4666b38b32925ba28bdc1a21957",
+ "version": "1.1.1"
 }
 '''  # END VERSION_JSON
 

xinference/api/restful_api.py

@@ -94,9 +94,9 @@ class CreateCompletionRequest(CreateCompletion):
 
 class CreateEmbeddingRequest(BaseModel):
     model: str
-    input: Union[str, List[str], List[int], List[List[int]]] = Field(
-        description="The input to embed."
-    )
+    input: Union[
+        str, List[str], List[int], List[List[int]], Dict[str, str], List[Dict[str, str]]
+    ] = Field(description="The input to embed.")
     user: Optional[str] = None
 
     class Config:
@@ -2044,7 +2044,6 @@ class RESTfulAPI(CancelMixin):
                 )
         if body.tools and body.stream:
             is_vllm = await model.is_vllm_backend()
-
             if not (
                 (is_vllm and model_family in QWEN_TOOL_CALL_FAMILY)
                 or (not is_vllm and model_family in GLM4_TOOL_CALL_FAMILY)
@@ -2054,7 +2053,8 @@ class RESTfulAPI(CancelMixin):
                     detail="Streaming support for tool calls is available only when using "
                     "Qwen models with vLLM backend or GLM4-chat models without vLLM backend.",
                 )
-
+        if "skip_special_tokens" in raw_kwargs and await model.is_vllm_backend():
+            kwargs["skip_special_tokens"] = raw_kwargs["skip_special_tokens"]
         if body.stream:
 
             async def stream_results():
@@ -2346,7 +2346,8 @@ class RESTfulAPI(CancelMixin):
     @staticmethod
     def extract_guided_params(raw_body: dict) -> dict:
         kwargs = {}
-        if raw_body.get("guided_json") is not None:
+        raw_extra_body: dict = raw_body.get("extra_body")  # type: ignore
+        if raw_body.get("guided_json"):
             kwargs["guided_json"] = raw_body.get("guided_json")
         if raw_body.get("guided_regex") is not None:
             kwargs["guided_regex"] = raw_body.get("guided_regex")
@@ -2362,6 +2363,27 @@ class RESTfulAPI(CancelMixin):
             kwargs["guided_whitespace_pattern"] = raw_body.get(
                 "guided_whitespace_pattern"
             )
+        # Parse OpenAI extra_body
+        if raw_extra_body is not None:
+            if raw_extra_body.get("guided_json"):
+                kwargs["guided_json"] = raw_extra_body.get("guided_json")
+            if raw_extra_body.get("guided_regex") is not None:
+                kwargs["guided_regex"] = raw_extra_body.get("guided_regex")
+            if raw_extra_body.get("guided_choice") is not None:
+                kwargs["guided_choice"] = raw_extra_body.get("guided_choice")
+            if raw_extra_body.get("guided_grammar") is not None:
+                kwargs["guided_grammar"] = raw_extra_body.get("guided_grammar")
+            if raw_extra_body.get("guided_json_object") is not None:
+                kwargs["guided_json_object"] = raw_extra_body.get("guided_json_object")
+            if raw_extra_body.get("guided_decoding_backend") is not None:
+                kwargs["guided_decoding_backend"] = raw_extra_body.get(
+                    "guided_decoding_backend"
+                )
+            if raw_extra_body.get("guided_whitespace_pattern") is not None:
+                kwargs["guided_whitespace_pattern"] = raw_extra_body.get(
+                    "guided_whitespace_pattern"
+                )
+
         return kwargs
 
 

xinference/core/utils.py

@@ -62,12 +62,16 @@ def log_async(
 
         @wraps(func)
         async def wrapped(*args, **kwargs):
-            try:
-                bound_args = sig.bind_partial(*args, **kwargs)
-                arguments = bound_args.arguments
-            except TypeError:
-                arguments = {}
-            request_id_str = arguments.get("request_id", "")
+            request_id_str = kwargs.get("request_id")
+            if not request_id_str:
+                # sometimes `request_id` not in kwargs
+                # we try to bind the arguments
+                try:
+                    bound_args = sig.bind_partial(*args, **kwargs)
+                    arguments = bound_args.arguments
+                except TypeError:
+                    arguments = {}
+                request_id_str = arguments.get("request_id", "")
             if not request_id_str:
                 request_id_str = uuid.uuid1()
                 if func_name == "text_to_image":

xinference/deploy/cmdline.py

@@ -846,7 +846,9 @@ def model_launch(
     kwargs = {}
     for i in range(0, len(ctx.args), 2):
         if not ctx.args[i].startswith("--"):
-            raise ValueError("You must specify extra kwargs with `--` prefix.")
+            raise ValueError(
+                f"You must specify extra kwargs with `--` prefix. There is an error in parameter passing that is {ctx.args[i]}."
+            )
         kwargs[ctx.args[i][2:]] = handle_click_args_type(ctx.args[i + 1])
     print(f"Launch model name: {model_name} with kwargs: {kwargs}", file=sys.stderr)
 

xinference/deploy/test/test_cmdline.py

@@ -23,6 +23,7 @@ from ..cmdline import (
     list_model_registrations,
     model_chat,
     model_generate,
+    model_launch,
     model_list,
     model_terminate,
     register_model,
@@ -311,3 +312,58 @@ def test_remove_cache(setup):
 
     assert result.exit_code == 0
     assert "Cache directory qwen1.5-chat has been deleted."
+
+
+def test_launch_error_in_passing_parameters():
+    runner = CliRunner()
+
+    # Known parameter but not provided with value.
+    result = runner.invoke(
+        model_launch,
+        [
+            "--model-engine",
+            "transformers",
+            "--model-name",
+            "qwen2.5-instruct",
+            "--model-uid",
+            "-s",
+            "0.5",
+            "-f",
+            "gptq",
+            "-q",
+            "INT4",
+            "111",
+            "-l",
+        ],
+    )
+    assert result.exit_code == 1
+    assert (
+        "You must specify extra kwargs with `--` prefix. There is an error in parameter passing that is 0.5."
+        in str(result)
+    )
+
+    # Unknown parameter
+    result = runner.invoke(
+        model_launch,
+        [
+            "--model-engine",
+            "transformers",
+            "--model-name",
+            "qwen2.5-instruct",
+            "--model-uid",
+            "123",
+            "-s",
+            "0.5",
+            "-f",
+            "gptq",
+            "-q",
+            "INT4",
+            "-l",
+            "111",
+        ],
+    )
+    assert result.exit_code == 1
+    assert (
+        "You must specify extra kwargs with `--` prefix. There is an error in parameter passing that is -l."
+        in str(result)
+    )

xinference/isolation.py

@@ -37,6 +37,30 @@ class Isolation:
         asyncio.set_event_loop(self._loop)
         self._stopped = asyncio.Event()
         self._loop.run_until_complete(self._stopped.wait())
+        self._cancel_all_tasks(self._loop)
+
+    @staticmethod
+    def _cancel_all_tasks(loop):
+        to_cancel = asyncio.all_tasks(loop)
+        if not to_cancel:
+            return
+
+        for task in to_cancel:
+            task.cancel()
+
+        loop.run_until_complete(asyncio.gather(*to_cancel, return_exceptions=True))
+
+        for task in to_cancel:
+            if task.cancelled():
+                continue
+            if task.exception() is not None:
+                loop.call_exception_handler(
+                    {
+                        "message": "unhandled exception during asyncio.run() shutdown",
+                        "exception": task.exception(),
+                        "task": task,
+                    }
+                )
 
     def start(self):
         if self._threaded:

xinference/model/audio/core.py

@@ -21,6 +21,8 @@ from ..core import CacheableModelSpec, ModelDescription
 from ..utils import valid_model_revision
 from .chattts import ChatTTSModel
 from .cosyvoice import CosyVoiceModel
+from .f5tts import F5TTSModel
+from .f5tts_mlx import F5TTSMLXModel
 from .fish_speech import FishSpeechModel
 from .funasr import FunASRModel
 from .whisper import WhisperModel
@@ -169,6 +171,8 @@ def create_audio_model_instance(
         ChatTTSModel,
         CosyVoiceModel,
         FishSpeechModel,
+        F5TTSModel,
+        F5TTSMLXModel,
     ],
     AudioModelDescription,
 ]:
@@ -182,6 +186,8 @@ def create_audio_model_instance(
         ChatTTSModel,
         CosyVoiceModel,
         FishSpeechModel,
+        F5TTSModel,
+        F5TTSMLXModel,
     ]
     if model_spec.model_family == "whisper":
         if not model_spec.engine:
@@ -196,6 +202,10 @@ def create_audio_model_instance(
         model = CosyVoiceModel(model_uid, model_path, model_spec, **kwargs)
     elif model_spec.model_family == "FishAudio":
         model = FishSpeechModel(model_uid, model_path, model_spec, **kwargs)
+    elif model_spec.model_family == "F5-TTS":
+        model = F5TTSModel(model_uid, model_path, model_spec, **kwargs)
+    elif model_spec.model_family == "F5-TTS-MLX":
+        model = F5TTSMLXModel(model_uid, model_path, model_spec, **kwargs)
     else:
         raise Exception(f"Unsupported audio model family: {model_spec.model_family}")
     model_description = AudioModelDescription(

xinference/model/audio/cosyvoice.py

@@ -39,6 +39,7 @@ class CosyVoiceModel:
         self._device = device
         self._model = None
         self._kwargs = kwargs
+        self._is_cosyvoice2 = False
 
     @property
     def model_ability(self):
@@ -51,7 +52,14 @@ class CosyVoiceModel:
         # The yaml config loaded from model has hard-coded the import paths. please refer to: load_hyperpyyaml
         sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../thirdparty"))
 
-        from cosyvoice.cli.cosyvoice import CosyVoice
+        if "CosyVoice2" in self._model_spec.model_name:
+            from cosyvoice.cli.cosyvoice import CosyVoice2 as CosyVoice
+
+            self._is_cosyvoice2 = True
+        else:
+            from cosyvoice.cli.cosyvoice import CosyVoice
+
+            self._is_cosyvoice2 = False
 
         self._model = CosyVoice(
             self._model_path, load_jit=self._kwargs.get("load_jit", False)
@@ -78,12 +86,22 @@ class CosyVoiceModel:
                 output = self._model.inference_zero_shot(
                     input, prompt_text, prompt_speech_16k, stream=stream
                 )
+            elif instruct_text:
+                assert self._is_cosyvoice2
+                logger.info("CosyVoice inference_instruct")
+                output = self._model.inference_instruct2(
+                    input,
+                    instruct_text=instruct_text,
+                    prompt_speech_16k=prompt_speech_16k,
+                    stream=stream,
+                )
             else:
                 logger.info("CosyVoice inference_cross_lingual")
                 output = self._model.inference_cross_lingual(
                     input, prompt_speech_16k, stream=stream
                 )
         else:
+            assert not self._is_cosyvoice2
             available_speakers = self._model.list_avaliable_spks()
             if not voice:
                 voice = available_speakers[0]
@@ -106,7 +124,9 @@ class CosyVoiceModel:
         def _generator_stream():
             with BytesIO() as out:
                 writer = torchaudio.io.StreamWriter(out, format=response_format)
-                writer.add_audio_stream(sample_rate=22050, num_channels=1)
+                writer.add_audio_stream(
+                    sample_rate=self._model.sample_rate, num_channels=1
+                )
                 i = 0
                 last_pos = 0
                 with writer.open():
@@ -125,7 +145,7 @@ class CosyVoiceModel:
             chunks = [o["tts_speech"] for o in output]
             t = torch.cat(chunks, dim=1)
             with BytesIO() as out:
-                torchaudio.save(out, t, 22050, format=response_format)
+                torchaudio.save(out, t, self._model.sample_rate, format=response_format)
                 return out.getvalue()
 
         return _generator_stream() if stream else _generator_block()
@@ -163,6 +183,8 @@ class CosyVoiceModel:
             assert (
                 prompt_text is None
             ), "CosyVoice Instruct model does not support prompt_text"
+        elif self._is_cosyvoice2:
+            assert prompt_speech is not None, "CosyVoice2 requires prompt_speech"
         else:
             # inference_zero_shot
             # inference_cross_lingual

xinference/model/audio/f5tts.py

@@ -0,0 +1,200 @@
+# Copyright 2022-2023 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import io
+import logging
+import os
+import re
+from io import BytesIO
+from typing import TYPE_CHECKING, Optional, Union
+
+if TYPE_CHECKING:
+    from .core import AudioModelFamilyV1
+
+logger = logging.getLogger(__name__)
+
+
+class F5TTSModel:
+    def __init__(
+        self,
+        model_uid: str,
+        model_path: str,
+        model_spec: "AudioModelFamilyV1",
+        device: Optional[str] = None,
+        **kwargs,
+    ):
+        self._model_uid = model_uid
+        self._model_path = model_path
+        self._model_spec = model_spec
+        self._device = device
+        self._model = None
+        self._vocoder = None
+        self._kwargs = kwargs
+
+    @property
+    def model_ability(self):
+        return self._model_spec.model_ability
+
+    def load(self):
+        import os
+        import sys
+
+        # The yaml config loaded from model has hard-coded the import paths. please refer to: load_hyperpyyaml
+        sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../thirdparty"))
+
+        from f5_tts.infer.utils_infer import load_model, load_vocoder
+        from f5_tts.model import DiT
+
+        vocoder_name = self._kwargs.get("vocoder_name", "vocos")
+        vocoder_path = self._kwargs.get("vocoder_path")
+
+        if vocoder_name not in ["vocos", "bigvgan"]:
+            raise Exception(f"Unsupported vocoder name: {vocoder_name}")
+
+        if vocoder_path is not None:
+            self._vocoder = load_vocoder(
+                vocoder_name=vocoder_name, is_local=True, local_path=vocoder_path
+            )
+        else:
+            self._vocoder = load_vocoder(vocoder_name=vocoder_name, is_local=False)
+
+        model_cls = DiT
+        model_cfg = dict(
+            dim=1024, depth=22, heads=16, ff_mult=2, text_dim=512, conv_layers=4
+        )
+        if vocoder_name == "vocos":
+            exp_name = "F5TTS_Base"
+            ckpt_step = 1200000
+        elif vocoder_name == "bigvgan":
+            exp_name = "F5TTS_Base_bigvgan"
+            ckpt_step = 1250000
+        else:
+            assert False
+        ckpt_file = os.path.join(
+            self._model_path, exp_name, f"model_{ckpt_step}.safetensors"
+        )
+        logger.info(f"Loading %s...", ckpt_file)
+        self._model = load_model(
+            model_cls, model_cfg, ckpt_file, mel_spec_type=vocoder_name
+        )
+
+    def _infer(self, ref_audio, ref_text, text_gen, model_obj, mel_spec_type, speed):
+        import numpy as np
+        from f5_tts.infer.utils_infer import infer_process, preprocess_ref_audio_text
+
+        config = {}
+        main_voice = {"ref_audio": ref_audio, "ref_text": ref_text}
+        if "voices" not in config:
+            voices = {"main": main_voice}
+        else:
+            voices = config["voices"]
+            voices["main"] = main_voice
+        for voice in voices:
+            (
+                voices[voice]["ref_audio"],
+                voices[voice]["ref_text"],
+            ) = preprocess_ref_audio_text(
+                voices[voice]["ref_audio"], voices[voice]["ref_text"]
+            )
+            logger.info("Voice:", voice)
+            logger.info("Ref_audio:", voices[voice]["ref_audio"])
+            logger.info("Ref_text:", voices[voice]["ref_text"])
+
+        final_sample_rate = None
+        generated_audio_segments = []
+        reg1 = r"(?=\[\w+\])"
+        chunks = re.split(reg1, text_gen)
+        reg2 = r"\[(\w+)\]"
+        for text in chunks:
+            if not text.strip():
+                continue
+            match = re.match(reg2, text)
+            if match:
+                voice = match[1]
+            else:
+                logger.info("No voice tag found, using main.")
+                voice = "main"
+            if voice not in voices:
+                logger.info(f"Voice {voice} not found, using main.")
+                voice = "main"
+            text = re.sub(reg2, "", text)
+            gen_text = text.strip()
+            ref_audio = voices[voice]["ref_audio"]
+            ref_text = voices[voice]["ref_text"]
+            logger.info(f"Voice: {voice}")
+            audio, final_sample_rate, spectragram = infer_process(
+                ref_audio,
+                ref_text,
+                gen_text,
+                model_obj,
+                self._vocoder,
+                mel_spec_type=mel_spec_type,
+                speed=speed,
+            )
+            generated_audio_segments.append(audio)
+
+        if generated_audio_segments:
+            final_wave = np.concatenate(generated_audio_segments)
+            return final_sample_rate, final_wave
+        return None, None
+
+    def speech(
+        self,
+        input: str,
+        voice: str,
+        response_format: str = "mp3",
+        speed: float = 1.0,
+        stream: bool = False,
+        **kwargs,
+    ):
+        import f5_tts
+        import soundfile
+        import tomli
+
+        if stream:
+            raise Exception("F5-TTS does not support stream generation.")
+
+        prompt_speech: Optional[bytes] = kwargs.pop("prompt_speech", None)
+        prompt_text: Optional[str] = kwargs.pop("prompt_text", None)
+
+        ref_audio: Union[str, io.BytesIO]
+        if prompt_speech is None:
+            base = os.path.dirname(f5_tts.__file__)
+            config = os.path.join(base, "infer/examples/basic/basic.toml")
+            with open(config, "rb") as f:
+                config_dict = tomli.load(f)
+            ref_audio = os.path.join(base, config_dict["ref_audio"])
+            prompt_text = config_dict["ref_text"]
+        else:
+            ref_audio = io.BytesIO(prompt_speech)
+            if prompt_text is None:
+                raise ValueError("`prompt_text` cannot be empty")
+
+        assert self._model is not None
+        vocoder_name = self._kwargs.get("vocoder_name", "vocos")
+        sample_rate, wav = self._infer(
+            ref_audio=ref_audio,
+            ref_text=prompt_text,
+            text_gen=input,
+            model_obj=self._model,
+            mel_spec_type=vocoder_name,
+            speed=speed,
+        )
+
+        # Save the generated audio
+        with BytesIO() as out:
+            with soundfile.SoundFile(
+                out, "w", sample_rate, 1, format=response_format.upper()
+            ) as f:
+                f.write(wav)
+            return out.getvalue()