xinference 0.16.3__py3-none-any.whl → 1.2.1__py3-none-any.whl

xinference/model/audio/melotts.py

@@ -0,0 +1,110 @@
+# Copyright 2022-2023 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+from io import BytesIO
+from typing import TYPE_CHECKING, Optional
+
+from ...device_utils import get_available_device, is_device_available
+
+if TYPE_CHECKING:
+    from .core import AudioModelFamilyV1
+
+logger = logging.getLogger(__name__)
+
+
+class MeloTTSModel:
+    def __init__(
+        self,
+        model_uid: str,
+        model_path: str,
+        model_spec: "AudioModelFamilyV1",
+        device: Optional[str] = None,
+        **kwargs,
+    ):
+        self._model_uid = model_uid
+        self._model_path = model_path
+        self._model_spec = model_spec
+        self._device = device
+        self._model = None
+        self._kwargs = kwargs
+
+    @property
+    def model_ability(self):
+        return self._model_spec.model_ability
+
+    def load(self):
+        if self._device is None:
+            self._device = get_available_device()
+        else:
+            if not is_device_available(self._device):
+                raise ValueError(f"Device {self._device} is not available!")
+
+        import os
+        import sys
+
+        import nltk
+
+        # English language requires download averaged_perceptron_tagger_eng
+        nltk.download("averaged_perceptron_tagger_eng")
+
+        # The yaml config loaded from model has hard-coded the import paths. please refer to: load_hyperpyyaml
+        sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../thirdparty"))
+
+        from melo.api import TTS
+
+        config_path = os.path.join(self._model_path, "config.json")
+        ckpt_path = os.path.join(self._model_path, "checkpoint.pth")
+        self._model = TTS(
+            language=self._model_spec.language,
+            device=self._device,
+            config_path=config_path,
+            ckpt_path=ckpt_path,
+        )
+
+    def speech(
+        self,
+        input: str,
+        voice: str,
+        response_format: str = "mp3",
+        speed: float = 1.0,
+        stream: bool = False,
+        **kwargs,
+    ):
+        import soundfile
+
+        if stream:
+            raise Exception("MeloTTS does not support stream mode.")
+        assert self._model is not None
+        speaker_ids = self._model.hps.data.spk2id
+        if not voice:
+            voice = next(iter(speaker_ids.keys()))
+            logger.info("Auto select speaker: %s", voice)
+        elif voice not in speaker_ids:
+            raise ValueError(
+                f"Invalid voice: {voice}, available speakers: {speaker_ids}"
+            )
+        audio = self._model.tts_to_file(
+            text=input, speaker_id=speaker_ids[voice], speed=speed, **kwargs
+        )
+        # Save the generated audio
+        with BytesIO() as out:
+            with soundfile.SoundFile(
+                out,
+                "w",
+                self._model.hps.data.sampling_rate,
+                1,
+                format=response_format.upper(),
+            ) as f:
+                f.write(audio)
+            return out.getvalue()

xinference/model/audio/model_spec.json

@@ -103,6 +103,86 @@
     "model_ability": "audio-to-text",
     "multilingual": false
   },
+  {
+    "model_name": "whisper-tiny-mlx",
+    "model_family": "whisper",
+    "model_id": "mlx-community/whisper-tiny",
+    "model_ability": "audio-to-text",
+    "multilingual": true,
+    "engine": "mlx"
+  },
+  {
+    "model_name": "whisper-tiny.en-mlx",
+    "model_family": "whisper",
+    "model_id": "mlx-community/whisper-tiny.en-mlx",
+    "model_ability": "audio-to-text",
+    "multilingual": false,
+    "engine": "mlx"
+  },
+  {
+    "model_name": "whisper-base-mlx",
+    "model_family": "whisper",
+    "model_id": "mlx-community/whisper-base-mlx",
+    "model_ability": "audio-to-text",
+    "multilingual": true,
+    "engine": "mlx"
+  },
+  {
+    "model_name": "whisper-base.en-mlx",
+    "model_family": "whisper",
+    "model_id": "mlx-community/whisper-base.en-mlx",
+    "model_ability": "audio-to-text",
+    "multilingual": false,
+    "engine": "mlx"
+  },
+  {
+    "model_name": "whisper-small-mlx",
+    "model_family": "whisper",
+    "model_id": "mlx-community/whisper-small-mlx",
+    "model_ability": "audio-to-text",
+    "multilingual": true,
+    "engine": "mlx"
+  },
+  {
+    "model_name": "whisper-small.en-mlx",
+    "model_family": "whisper",
+    "model_id": "mlx-community/whisper-small.en-mlx",
+    "model_ability": "audio-to-text",
+    "multilingual": false,
+    "engine": "mlx"
+  },
+  {
+    "model_name": "whisper-medium-mlx",
+    "model_family": "whisper",
+    "model_id": "mlx-community/whisper-medium-mlx",
+    "model_ability": "audio-to-text",
+    "multilingual": true,
+    "engine": "mlx"
+  },
+  {
+    "model_name": "whisper-medium.en-mlx",
+    "model_family": "whisper",
+    "model_id": "mlx-community/whisper-medium.en-mlx",
+    "model_ability": "audio-to-text",
+    "multilingual": false,
+    "engine": "mlx"
+  },
+  {
+    "model_name": "whisper-large-v3-mlx",
+    "model_family": "whisper",
+    "model_id": "mlx-community/whisper-large-v3-mlx",
+    "model_ability": "audio-to-text",
+    "multilingual": true,
+    "engine": "mlx"
+  },
+  {
+    "model_name": "whisper-large-v3-turbo-mlx",
+    "model_family": "whisper",
+    "model_id": "mlx-community/whisper-large-v3-turbo",
+    "model_ability": "audio-to-text",
+    "multilingual": true,
+    "engine": "mlx"
+  },
   {
     "model_name": "SenseVoiceSmall",
     "model_family": "funasr",
@@ -156,11 +236,107 @@
     "multilingual": true
   },
   {
-    "model_name": "FishSpeech-1.4",
+    "model_name": "CosyVoice2-0.5B",
+    "model_family": "CosyVoice",
+    "model_id": "mrfakename/CosyVoice2-0.5B",
+    "model_revision": "5676baabc8a76dc93ef60a88bbd2420deaa2f644",
+    "model_ability": "text-to-audio",
+    "multilingual": true
+  },
+  {
+    "model_name": "FishSpeech-1.5",
     "model_family": "FishAudio",
-    "model_id": "fishaudio/fish-speech-1.4",
-    "model_revision": "3c49651b8e583b6b13f55e375432e0d57e1aa84d",
+    "model_id": "fishaudio/fish-speech-1.5",
+    "model_revision": "268b6ec86243dd683bc78dab7e9a6cedf9191f2a",
     "model_ability": "text-to-audio",
     "multilingual": true
+  },
+  {
+    "model_name": "F5-TTS",
+    "model_family": "F5-TTS",
+    "model_id": "SWivid/F5-TTS",
+    "model_revision": "4dcc16f297f2ff98a17b3726b16f5de5a5e45672",
+    "model_ability": "text-to-audio",
+    "multilingual": true
+  },
+  {
+    "model_name": "F5-TTS-MLX",
+    "model_family": "F5-TTS-MLX",
+    "model_id": "lucasnewman/f5-tts-mlx",
+    "model_revision": "7642bb232e3fcacf92c51c786edebb8624da6b93",
+    "model_ability": "text-to-audio",
+    "multilingual": true
+  },
+  {
+    "model_name": "MeloTTS-English",
+    "model_family": "MeloTTS",
+    "model_id": "myshell-ai/MeloTTS-English",
+    "model_revision": "bb4fb7346d566d277ba8c8c7dbfdf6786139b8ef",
+    "model_ability": "text-to-audio",
+    "multilingual": false,
+    "language": "EN"
+  },
+  {
+    "model_name": "MeloTTS-English-v2",
+    "model_family": "MeloTTS",
+    "model_id": "myshell-ai/MeloTTS-English-v2",
+    "model_revision": "a53e3509c4ee4ff16d79272feb2474ff864e18f3",
+    "model_ability": "text-to-audio",
+    "multilingual": false,
+    "language": "EN"
+  },
+  {
+    "model_name": "MeloTTS-English-v3",
+    "model_family": "MeloTTS",
+    "model_id": "myshell-ai/MeloTTS-English-v3",
+    "model_revision": "f7c4a35392c0e9be24a755f1edb4c3f63040f759",
+    "model_ability": "text-to-audio",
+    "multilingual": false,
+    "language": "EN"
+  },
+  {
+    "model_name": "MeloTTS-French",
+    "model_family": "MeloTTS",
+    "model_id": "myshell-ai/MeloTTS-French",
+    "model_revision": "1e9bf590262392d8bffb679b0a3b0c16b0f9fdaf",
+    "model_ability": "text-to-audio",
+    "multilingual": false,
+    "language": "FR"
+  },
+  {
+    "model_name": "MeloTTS-Japanese",
+    "model_family": "MeloTTS",
+    "model_id": "myshell-ai/MeloTTS-Japanese",
+    "model_revision": "367f8795464b531b4e97c1515bddfc1243e60891",
+    "model_ability": "text-to-audio",
+    "multilingual": false,
+    "language": "JP"
+  },
+  {
+    "model_name": "MeloTTS-Spanish",
+    "model_family": "MeloTTS",
+    "model_id": "myshell-ai/MeloTTS-Spanish",
+    "model_revision": "dbb5496df39d11a66c1d5f5a9ca357c3c9fb95fb",
+    "model_ability": "text-to-audio",
+    "multilingual": false,
+    "language": "ES"
+  },
+  {
+    "model_name": "MeloTTS-Chinese",
+    "model_family": "MeloTTS",
+    "model_id": "myshell-ai/MeloTTS-Chinese",
+    "model_revision": "af5d207a364ea4208c6f589c89f57f88414bdd16",
+    "model_ability": "text-to-audio",
+    "multilingual": false,
+    "language": "ZH"
+  },
+  {
+    "model_name": "MeloTTS-Korean",
+    "model_family": "MeloTTS",
+    "model_id": "myshell-ai/MeloTTS-Korean",
+    "model_revision": "0207e5adfc90129a51b6b03d89be6d84360ed323",
+    "model_ability": "text-to-audio",
+    "multilingual": false,
+    "language": "KR"
   }
 ]

xinference/model/audio/model_spec_modelscope.json

@@ -17,6 +17,15 @@
     "model_ability": "audio-to-text",
     "multilingual": true
   },
+  {
+    "model_name": "Belle-whisper-large-v3-zh",
+    "model_family": "whisper",
+    "model_hub": "modelscope",
+    "model_id": "Xorbits/Belle-whisper-large-v3-zh",
+    "model_revision": "master",
+    "model_ability": "audio-to-text",
+    "multilingual": false
+  },
   {
     "model_name": "SenseVoiceSmall",
     "model_family": "funasr",
@@ -73,5 +82,23 @@
     "model_revision": "master",
     "model_ability": "text-to-audio",
     "multilingual": true
+  },
+  {
+    "model_name": "CosyVoice2-0.5B",
+    "model_family": "CosyVoice",
+    "model_hub": "modelscope",
+    "model_id": "iic/CosyVoice2-0.5B",
+    "model_revision": "master",
+    "model_ability": "text-to-audio",
+    "multilingual": true
+  },
+  {
+    "model_name": "F5-TTS",
+    "model_family": "F5-TTS",
+    "model_hub": "modelscope",
+    "model_id": "SWivid/F5-TTS_Emilia-ZH-EN",
+    "model_revision": "master",
+    "model_ability": "text-to-audio",
+    "multilingual": true
   }
 ]

xinference/model/audio/utils.py

@@ -11,8 +11,40 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+import io
+
+import numpy as np
+
 from .core import AudioModelFamilyV1
 
 
 def get_model_version(audio_model: AudioModelFamilyV1) -> str:
     return audio_model.model_name
+
+
+def ensure_sample_rate(
+    audio: np.ndarray, old_sample_rate: int, sample_rate: int
+) -> np.ndarray:
+    import soundfile as sf
+    from scipy.signal import resample
+
+    if old_sample_rate != sample_rate:
+        # Calculate the new data length
+        new_length = int(len(audio) * sample_rate / old_sample_rate)
+
+        # Resample the data
+        resampled_data = resample(audio, new_length)
+
+        # Use BytesIO to save the resampled data to memory
+        with io.BytesIO() as buffer:
+            # Write the resampled data to the memory buffer
+            sf.write(buffer, resampled_data, sample_rate, format="WAV")
+
+            # Reset the buffer position to the beginning
+            buffer.seek(0)
+
+            # Read the data from the memory buffer
+            audio, sr = sf.read(buffer, dtype="float32")
+
+    return audio

xinference/model/audio/whisper.py

@@ -13,9 +13,12 @@
 # limitations under the License.
 import logging
 import os
+import typing
 from glob import glob
 from typing import TYPE_CHECKING, Dict, List, Optional, Union
 
+from typing_extensions import TypedDict
+
 from ...device_utils import (
     get_available_device,
     get_device_preferred_dtype,
@@ -28,6 +31,13 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
+class WhisperModelConfig(TypedDict, total=False):
+    chunk_length_s: Optional[float]
+    stride_length_s: Optional[float]
+    return_timestamps: Optional[bool]
+    batch_size: Optional[int]
+
+
 class WhisperModel:
     def __init__(
         self,
@@ -35,6 +45,7 @@ class WhisperModel:
         model_path: str,
         model_spec: "AudioModelFamilyV1",
         device: Optional[str] = None,
+        max_new_tokens: Optional[int] = 128,
         **kwargs,
     ):
         self._model_uid = model_uid
@@ -42,7 +53,21 @@ class WhisperModel:
         self._model_spec = model_spec
         self._device = device
         self._model = None
-        self._kwargs = kwargs
+        self._max_new_tokens = max_new_tokens
+        self._model_config: WhisperModelConfig = self._sanitize_model_config(
+            typing.cast(WhisperModelConfig, kwargs)
+        )
+
+    def _sanitize_model_config(
+        self, model_config: Optional[WhisperModelConfig]
+    ) -> WhisperModelConfig:
+        if model_config is None:
+            model_config = WhisperModelConfig()
+        model_config.setdefault("chunk_length_s", 30)
+        model_config.setdefault("stride_length_s", None)
+        model_config.setdefault("return_timestamps", False)
+        model_config.setdefault("batch_size", 16)
+        return model_config
 
     @property
     def model_ability(self):
@@ -75,10 +100,10 @@ class WhisperModel:
             model=model,
             tokenizer=processor.tokenizer,
             feature_extractor=processor.feature_extractor,
-            max_new_tokens=128,
-            chunk_length_s=30,
-            batch_size=16,
-            return_timestamps=False,
+            chunk_length_s=self._model_config.get("chunk_length_s"),
+            stride_length_s=self._model_config.get("stride_length_s"),
+            return_timestamps=self._model_config.get("return_timestamps"),
+            batch_size=self._model_config.get("batch_size"),
             torch_dtype=torch_dtype,
             device=self._device,
         )
@@ -185,13 +210,13 @@ class WhisperModel:
             logger.warning(
                 "Prompt for whisper transcriptions will be ignored: %s", prompt
             )
+        generate_kwargs = {"max_new_tokens": self._max_new_tokens, "task": "transcribe"}
+        if language is not None:
+            generate_kwargs["language"] = language
+
         return self._call_model(
             audio=audio,
-            generate_kwargs=(
-                {"language": language, "task": "transcribe"}
-                if language is not None
-                else {"task": "transcribe"}
-            ),
+            generate_kwargs=generate_kwargs,
             response_format=response_format,
             temperature=temperature,
             timestamp_granularities=timestamp_granularities,

xinference/model/audio/whisper_mlx.py

@@ -0,0 +1,208 @@
+# Copyright 2022-2023 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import functools
+import itertools
+import logging
+import tempfile
+from typing import TYPE_CHECKING, List, Optional
+
+if TYPE_CHECKING:
+    from .core import AudioModelFamilyV1
+
+logger = logging.getLogger(__name__)
+
+
+class WhisperMLXModel:
+    def __init__(
+        self,
+        model_uid: str,
+        model_path: str,
+        model_spec: "AudioModelFamilyV1",
+        device: Optional[str] = None,
+        **kwargs,
+    ):
+        self._model_uid = model_uid
+        self._model_path = model_path
+        self._model_spec = model_spec
+        self._device = device
+        self._model = None
+        self._kwargs = kwargs
+        self._use_lighting = False
+
+    @property
+    def model_ability(self):
+        return self._model_spec.model_ability
+
+    def load(self):
+        use_lightning = self._kwargs.get("use_lightning", "auto")
+        if use_lightning not in ("auto", True, False, None):
+            raise ValueError("use_lightning can only be True, False, None or auto")
+
+        if use_lightning == "auto" or use_lightning is True:
+            try:
+                import mlx.core as mx
+                from lightning_whisper_mlx.transcribe import ModelHolder
+            except ImportError:
+                if use_lightning == "auto":
+                    use_lightning = False
+                else:
+                    error_message = "Failed to import module 'lightning_whisper_mlx'"
+                    installation_guide = [
+                        "Please make sure 'lightning_whisper_mlx' is installed.\n",
+                    ]
+
+                    raise ImportError(
+                        f"{error_message}\n\n{''.join(installation_guide)}"
+                    )
+            else:
+                use_lightning = True
+        if not use_lightning:
+            try:
+                import mlx.core as mx  # noqa: F811
+                from mlx_whisper.transcribe import ModelHolder  # noqa: F811
+            except ImportError:
+                error_message = "Failed to import module 'mlx_whisper'"
+                installation_guide = [
+                    "Please make sure 'mlx_whisper' is installed.\n",
+                ]
+
+                raise ImportError(f"{error_message}\n\n{''.join(installation_guide)}")
+            else:
+                use_lightning = False
+
+        logger.info(
+            "Loading MLX whisper from %s, use lightning: %s",
+            self._model_path,
+            use_lightning,
+        )
+        self._use_lighting = use_lightning
+        self._model = ModelHolder.get_model(self._model_path, mx.float16)
+
+    def transcriptions(
+        self,
+        audio: bytes,
+        language: Optional[str] = None,
+        prompt: Optional[str] = None,
+        response_format: str = "json",
+        temperature: float = 0,
+        timestamp_granularities: Optional[List[str]] = None,
+    ):
+        return self._call(
+            audio,
+            language=language,
+            prompt=prompt,
+            response_format=response_format,
+            temperature=temperature,
+            timestamp_granularities=timestamp_granularities,
+            task="transcribe",
+        )
+
+    def translations(
+        self,
+        audio: bytes,
+        language: Optional[str] = None,
+        prompt: Optional[str] = None,
+        response_format: str = "json",
+        temperature: float = 0,
+        timestamp_granularities: Optional[List[str]] = None,
+    ):
+        if not self._model_spec.multilingual:
+            raise RuntimeError(
+                f"Model {self._model_spec.model_name} is not suitable for translations."
+            )
+        return self._call(
+            audio,
+            language=language,
+            prompt=prompt,
+            response_format=response_format,
+            temperature=temperature,
+            timestamp_granularities=timestamp_granularities,
+            task="translate",
+        )
+
+    def _call(
+        self,
+        audio: bytes,
+        language: Optional[str] = None,
+        prompt: Optional[str] = None,
+        response_format: str = "json",
+        temperature: float = 0,
+        timestamp_granularities: Optional[List[str]] = None,
+        task: str = "transcribe",
+    ):
+        if self._use_lighting:
+            from lightning_whisper_mlx.transcribe import transcribe_audio
+
+            transcribe = functools.partial(
+                transcribe_audio, batch_size=self._kwargs.get("batch_size", 12)
+            )
+        else:
+            from mlx_whisper import transcribe  # type: ignore
+
+        with tempfile.NamedTemporaryFile(delete=True) as f:
+            f.write(audio)
+
+            kwargs = {"task": task}
+            if response_format == "verbose_json":
+                if timestamp_granularities == ["word"]:
+                    kwargs["word_timestamps"] = True  # type: ignore
+
+            result = transcribe(
+                f.name,
+                path_or_hf_repo=self._model_path,
+                language=language,
+                temperature=temperature,
+                initial_prompt=prompt,
+                **kwargs,
+            )
+            text = result["text"]
+            segments = result["segments"]
+            language = result["language"]
+
+            if response_format == "json":
+                return {"text": text}
+            elif response_format == "verbose_json":
+                if not timestamp_granularities or timestamp_granularities == [
+                    "segment"
+                ]:
+                    return {
+                        "task": task,
+                        "language": language,
+                        "duration": segments[-1]["end"] if segments else 0,
+                        "text": text,
+                        "segments": segments,
+                    }
+                else:
+                    assert timestamp_granularities == ["word"]
+
+                    def _extract_word(word: dict) -> dict:
+                        return {
+                            "start": word["start"].item(),
+                            "end": word["end"].item(),
+                            "word": word["word"],
+                        }
+
+                    words = [
+                        _extract_word(w)
+                        for w in itertools.chain(*[s["words"] for s in segments])
+                    ]
+                    return {
+                        "task": task,
+                        "language": language,
+                        "duration": words[-1]["end"] if words else 0,
+                        "text": text,
+                        "words": words,
+                    }
+            else:
+                raise ValueError(f"Unsupported response format: {response_format}")