xinference 1.0.1__py3-none-any.whl → 1.1.0__py3-none-any.whl

xinference/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2024-11-29T16:57:04+0800",
+ "date": "2024-12-13T18:21:03+0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "eb8ddd431f5c5fcb2216e25e0d43745f8455d9b9",
- "version": "1.0.1"
+ "full-revisionid": "b132fca91f3e1b11b111f9b89f68a55e4b7605c6",
+ "version": "1.1.0"
 }
 '''  # END VERSION_JSON
 

xinference/api/restful_api.py

@@ -94,9 +94,9 @@ class CreateCompletionRequest(CreateCompletion):
 
 class CreateEmbeddingRequest(BaseModel):
     model: str
-    input: Union[str, List[str], List[int], List[List[int]]] = Field(
-        description="The input to embed."
-    )
+    input: Union[
+        str, List[str], List[int], List[List[int]], Dict[str, str], List[Dict[str, str]]
+    ] = Field(description="The input to embed.")
     user: Optional[str] = None
 
     class Config:
@@ -2044,7 +2044,6 @@ class RESTfulAPI(CancelMixin):
                 )
         if body.tools and body.stream:
             is_vllm = await model.is_vllm_backend()
-
             if not (
                 (is_vllm and model_family in QWEN_TOOL_CALL_FAMILY)
                 or (not is_vllm and model_family in GLM4_TOOL_CALL_FAMILY)
@@ -2054,7 +2053,8 @@ class RESTfulAPI(CancelMixin):
                     detail="Streaming support for tool calls is available only when using "
                     "Qwen models with vLLM backend or GLM4-chat models without vLLM backend.",
                 )
-
+        if "skip_special_tokens" in raw_kwargs and await model.is_vllm_backend():
+            kwargs["skip_special_tokens"] = raw_kwargs["skip_special_tokens"]
         if body.stream:
 
             async def stream_results():

xinference/core/model.py

@@ -78,6 +78,7 @@ XINFERENCE_BATCHING_ALLOWED_VISION_MODELS = [
 ]
 
 XINFERENCE_TEXT_TO_IMAGE_BATCHING_ALLOWED_MODELS = ["FLUX.1-dev", "FLUX.1-schnell"]
+XINFERENCE_BATCHING_BLACK_LIST = ["glm4-chat"]
 
 
 def request_limit(fn):
@@ -372,7 +373,11 @@ class ModelActor(xo.StatelessActor, CancelMixin):
                     f"Your model {self._model.model_family.model_name} with model family {self._model.model_family.model_family} is disqualified."
                 )
                 return False
-        return condition
+        return (
+            condition
+            and self._model.model_family.model_name
+            not in XINFERENCE_BATCHING_BLACK_LIST
+        )
 
     def allow_batching_for_text_to_image(self) -> bool:
         from ..model.image.stable_diffusion.core import DiffusionModel

xinference/deploy/cmdline.py

@@ -846,7 +846,9 @@ def model_launch(
     kwargs = {}
     for i in range(0, len(ctx.args), 2):
         if not ctx.args[i].startswith("--"):
-            raise ValueError("You must specify extra kwargs with `--` prefix.")
+            raise ValueError(
+                f"You must specify extra kwargs with `--` prefix. There is an error in parameter passing that is {ctx.args[i]}."
+            )
         kwargs[ctx.args[i][2:]] = handle_click_args_type(ctx.args[i + 1])
     print(f"Launch model name: {model_name} with kwargs: {kwargs}", file=sys.stderr)
 

xinference/deploy/test/test_cmdline.py

@@ -23,6 +23,7 @@ from ..cmdline import (
     list_model_registrations,
     model_chat,
     model_generate,
+    model_launch,
     model_list,
     model_terminate,
     register_model,
@@ -311,3 +312,58 @@ def test_remove_cache(setup):
 
     assert result.exit_code == 0
     assert "Cache directory qwen1.5-chat has been deleted."
+
+
+def test_launch_error_in_passing_parameters():
+    runner = CliRunner()
+
+    # Known parameter but not provided with value.
+    result = runner.invoke(
+        model_launch,
+        [
+            "--model-engine",
+            "transformers",
+            "--model-name",
+            "qwen2.5-instruct",
+            "--model-uid",
+            "-s",
+            "0.5",
+            "-f",
+            "gptq",
+            "-q",
+            "INT4",
+            "111",
+            "-l",
+        ],
+    )
+    assert result.exit_code == 1
+    assert (
+        "You must specify extra kwargs with `--` prefix. There is an error in parameter passing that is 0.5."
+        in str(result)
+    )
+
+    # Unknown parameter
+    result = runner.invoke(
+        model_launch,
+        [
+            "--model-engine",
+            "transformers",
+            "--model-name",
+            "qwen2.5-instruct",
+            "--model-uid",
+            "123",
+            "-s",
+            "0.5",
+            "-f",
+            "gptq",
+            "-q",
+            "INT4",
+            "-l",
+            "111",
+        ],
+    )
+    assert result.exit_code == 1
+    assert (
+        "You must specify extra kwargs with `--` prefix. There is an error in parameter passing that is -l."
+        in str(result)
+    )

xinference/isolation.py

@@ -37,6 +37,30 @@ class Isolation:
         asyncio.set_event_loop(self._loop)
         self._stopped = asyncio.Event()
         self._loop.run_until_complete(self._stopped.wait())
+        self._cancel_all_tasks(self._loop)
+
+    @staticmethod
+    def _cancel_all_tasks(loop):
+        to_cancel = asyncio.all_tasks(loop)
+        if not to_cancel:
+            return
+
+        for task in to_cancel:
+            task.cancel()
+
+        loop.run_until_complete(asyncio.gather(*to_cancel, return_exceptions=True))
+
+        for task in to_cancel:
+            if task.cancelled():
+                continue
+            if task.exception() is not None:
+                loop.call_exception_handler(
+                    {
+                        "message": "unhandled exception during asyncio.run() shutdown",
+                        "exception": task.exception(),
+                        "task": task,
+                    }
+                )
 
     def start(self):
         if self._threaded:

xinference/model/audio/core.py

@@ -21,6 +21,7 @@ from ..core import CacheableModelSpec, ModelDescription
 from ..utils import valid_model_revision
 from .chattts import ChatTTSModel
 from .cosyvoice import CosyVoiceModel
+from .f5tts import F5TTSModel
 from .fish_speech import FishSpeechModel
 from .funasr import FunASRModel
 from .whisper import WhisperModel
@@ -169,6 +170,7 @@ def create_audio_model_instance(
         ChatTTSModel,
         CosyVoiceModel,
         FishSpeechModel,
+        F5TTSModel,
     ],
     AudioModelDescription,
 ]:
@@ -182,6 +184,7 @@ def create_audio_model_instance(
         ChatTTSModel,
         CosyVoiceModel,
         FishSpeechModel,
+        F5TTSModel,
     ]
     if model_spec.model_family == "whisper":
         if not model_spec.engine:
@@ -196,6 +199,8 @@ def create_audio_model_instance(
         model = CosyVoiceModel(model_uid, model_path, model_spec, **kwargs)
     elif model_spec.model_family == "FishAudio":
         model = FishSpeechModel(model_uid, model_path, model_spec, **kwargs)
+    elif model_spec.model_family == "F5-TTS":
+        model = F5TTSModel(model_uid, model_path, model_spec, **kwargs)
     else:
         raise Exception(f"Unsupported audio model family: {model_spec.model_family}")
     model_description = AudioModelDescription(

xinference/model/audio/f5tts.py

@@ -0,0 +1,195 @@
+# Copyright 2022-2023 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+import re
+from io import BytesIO
+from typing import TYPE_CHECKING, Optional
+
+if TYPE_CHECKING:
+    from .core import AudioModelFamilyV1
+
+logger = logging.getLogger(__name__)
+
+
+class F5TTSModel:
+    def __init__(
+        self,
+        model_uid: str,
+        model_path: str,
+        model_spec: "AudioModelFamilyV1",
+        device: Optional[str] = None,
+        **kwargs,
+    ):
+        self._model_uid = model_uid
+        self._model_path = model_path
+        self._model_spec = model_spec
+        self._device = device
+        self._model = None
+        self._vocoder = None
+        self._kwargs = kwargs
+
+    @property
+    def model_ability(self):
+        return self._model_spec.model_ability
+
+    def load(self):
+        import os
+        import sys
+
+        # The yaml config loaded from model has hard-coded the import paths. please refer to: load_hyperpyyaml
+        sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../thirdparty"))
+
+        from f5_tts.infer.utils_infer import load_model, load_vocoder
+        from f5_tts.model import DiT
+
+        vocoder_name = self._kwargs.get("vocoder_name", "vocos")
+        vocoder_path = self._kwargs.get("vocoder_path")
+
+        if vocoder_name not in ["vocos", "bigvgan"]:
+            raise Exception(f"Unsupported vocoder name: {vocoder_name}")
+
+        if vocoder_path is not None:
+            self._vocoder = load_vocoder(
+                vocoder_name=vocoder_name, is_local=True, local_path=vocoder_path
+            )
+        else:
+            self._vocoder = load_vocoder(vocoder_name=vocoder_name, is_local=False)
+
+        model_cls = DiT
+        model_cfg = dict(
+            dim=1024, depth=22, heads=16, ff_mult=2, text_dim=512, conv_layers=4
+        )
+        if vocoder_name == "vocos":
+            exp_name = "F5TTS_Base"
+            ckpt_step = 1200000
+        elif vocoder_name == "bigvgan":
+            exp_name = "F5TTS_Base_bigvgan"
+            ckpt_step = 1250000
+        else:
+            assert False
+        ckpt_file = os.path.join(
+            self._model_path, exp_name, f"model_{ckpt_step}.safetensors"
+        )
+        logger.info(f"Loading %s...", ckpt_file)
+        self._model = load_model(
+            model_cls, model_cfg, ckpt_file, mel_spec_type=vocoder_name
+        )
+
+    def _infer(self, ref_audio, ref_text, text_gen, model_obj, mel_spec_type, speed):
+        import numpy as np
+        from f5_tts.infer.utils_infer import infer_process, preprocess_ref_audio_text
+
+        config = {}
+        main_voice = {"ref_audio": ref_audio, "ref_text": ref_text}
+        if "voices" not in config:
+            voices = {"main": main_voice}
+        else:
+            voices = config["voices"]
+            voices["main"] = main_voice
+        for voice in voices:
+            (
+                voices[voice]["ref_audio"],
+                voices[voice]["ref_text"],
+            ) = preprocess_ref_audio_text(
+                voices[voice]["ref_audio"], voices[voice]["ref_text"]
+            )
+            print("Voice:", voice)
+            print("Ref_audio:", voices[voice]["ref_audio"])
+            print("Ref_text:", voices[voice]["ref_text"])
+
+        final_sample_rate = None
+        generated_audio_segments = []
+        reg1 = r"(?=\[\w+\])"
+        chunks = re.split(reg1, text_gen)
+        reg2 = r"\[(\w+)\]"
+        for text in chunks:
+            if not text.strip():
+                continue
+            match = re.match(reg2, text)
+            if match:
+                voice = match[1]
+            else:
+                print("No voice tag found, using main.")
+                voice = "main"
+            if voice not in voices:
+                print(f"Voice {voice} not found, using main.")
+                voice = "main"
+            text = re.sub(reg2, "", text)
+            gen_text = text.strip()
+            ref_audio = voices[voice]["ref_audio"]
+            ref_text = voices[voice]["ref_text"]
+            print(f"Voice: {voice}")
+            audio, final_sample_rate, spectragram = infer_process(
+                ref_audio,
+                ref_text,
+                gen_text,
+                model_obj,
+                self._vocoder,
+                mel_spec_type=mel_spec_type,
+                speed=speed,
+            )
+            generated_audio_segments.append(audio)
+
+        if generated_audio_segments:
+            final_wave = np.concatenate(generated_audio_segments)
+            return final_sample_rate, final_wave
+        return None, None
+
+    def speech(
+        self,
+        input: str,
+        voice: str,
+        response_format: str = "mp3",
+        speed: float = 1.0,
+        stream: bool = False,
+        **kwargs,
+    ):
+        import f5_tts
+        import soundfile
+        import tomli
+
+        if stream:
+            raise Exception("F5-TTS does not support stream generation.")
+
+        prompt_speech: Optional[bytes] = kwargs.pop("prompt_speech", None)
+        prompt_text: Optional[str] = kwargs.pop("prompt_text", None)
+
+        if prompt_speech is None:
+            base = os.path.dirname(f5_tts.__file__)
+            config = os.path.join(base, "infer/examples/basic/basic.toml")
+            with open(config, "rb") as f:
+                config_dict = tomli.load(f)
+            prompt_speech = os.path.join(base, config_dict["ref_audio"])
+            prompt_text = config_dict["ref_text"]
+
+        assert self._model is not None
+        vocoder_name = self._kwargs.get("vocoder_name", "vocos")
+        sample_rate, wav = self._infer(
+            ref_audio=prompt_speech,
+            ref_text=prompt_text,
+            text_gen=input,
+            model_obj=self._model,
+            mel_spec_type=vocoder_name,
+            speed=speed,
+        )
+
+        # Save the generated audio
+        with BytesIO() as out:
+            with soundfile.SoundFile(
+                out, "w", sample_rate, 1, format=response_format.upper()
+            ) as f:
+                f.write(wav)
+            return out.getvalue()

xinference/model/audio/fish_speech.py

@@ -213,13 +213,14 @@ class FishSpeechModel:
         import torchaudio
 
         prompt_speech = kwargs.get("prompt_speech")
+        prompt_text = kwargs.get("prompt_text", kwargs.get("reference_text", ""))
         result = self._inference(
             text=input,
             enable_reference_audio=kwargs.get(
                 "enable_reference_audio", prompt_speech is not None
             ),
             reference_audio=prompt_speech,
-            reference_text=kwargs.get("reference_text", ""),
+            reference_text=prompt_text,
             max_new_tokens=kwargs.get("max_new_tokens", 1024),
             chunk_length=kwargs.get("chunk_length", 200),
             top_p=kwargs.get("top_p", 0.7),

xinference/model/audio/model_spec.json

@@ -242,5 +242,13 @@
     "model_revision": "069c573759936b35191d3380deb89183c0656f59",
     "model_ability": "text-to-audio",
     "multilingual": true
+  },
+  {
+    "model_name": "F5-TTS",
+    "model_family": "F5-TTS",
+    "model_id": "SWivid/F5-TTS",
+    "model_revision": "4dcc16f297f2ff98a17b3726b16f5de5a5e45672",
+    "model_ability": "text-to-audio",
+    "multilingual": true
   }
 ]

xinference/model/audio/model_spec_modelscope.json

@@ -73,5 +73,14 @@
     "model_revision": "master",
     "model_ability": "text-to-audio",
     "multilingual": true
+  },
+  {
+    "model_name": "F5-TTS",
+    "model_family": "F5-TTS",
+    "model_hub": "modelscope",
+    "model_id": "SWivid/F5-TTS_Emilia-ZH-EN",
+    "model_revision": "master",
+    "model_ability": "text-to-audio",
+    "multilingual": true
   }
 ]