xinference 0.14.4.post1__py3-none-any.whl → 0.15.1__py3-none-any.whl

xinference/core/utils.py

@@ -11,62 +11,120 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import copy
 import logging
 import os
 import random
 import string
-from typing import Dict, Generator, List, Tuple, Union
+import uuid
+from typing import Dict, Generator, List, Optional, Tuple, Union
 
 import orjson
 from pynvml import nvmlDeviceGetCount, nvmlInit, nvmlShutdown
 
 from .._compat import BaseModel
+from ..constants import XINFERENCE_LOG_ARG_MAX_LENGTH
 
 logger = logging.getLogger(__name__)
 
 
-def log_async(logger, args_formatter=None):
+def truncate_log_arg(arg) -> str:
+    s = str(arg)
+    if len(s) > XINFERENCE_LOG_ARG_MAX_LENGTH:
+        s = s[0:XINFERENCE_LOG_ARG_MAX_LENGTH] + "..."
+    return s
+
+
+def log_async(
+    logger,
+    level=logging.DEBUG,
+    ignore_kwargs: Optional[List[str]] = None,
+    log_exception=True,
+):
     import time
     from functools import wraps
 
     def decorator(func):
+        func_name = func.__name__
+
         @wraps(func)
         async def wrapped(*args, **kwargs):
-            if args_formatter is not None:
-                formatted_args, formatted_kwargs = copy.copy(args), copy.copy(kwargs)
-                args_formatter(formatted_args, formatted_kwargs)
-            else:
-                formatted_args, formatted_kwargs = args, kwargs
-            logger.debug(
-                f"Enter {func.__name__}, args: {formatted_args}, kwargs: {formatted_kwargs}"
+            request_id_str = kwargs.get("request_id", "")
+            if not request_id_str:
+                request_id_str = uuid.uuid1()
+            request_id_str = f"[request {request_id_str}]"
+            formatted_args = ",".join(map(truncate_log_arg, args))
+            formatted_kwargs = ",".join(
+                [
+                    "%s=%s" % (k, truncate_log_arg(v))
+                    for k, v in kwargs.items()
+                    if ignore_kwargs is None or k not in ignore_kwargs
+                ]
             )
-            start = time.time()
-            ret = await func(*args, **kwargs)
-            logger.debug(
-                f"Leave {func.__name__}, elapsed time: {int(time.time() - start)} s"
+            logger.log(
+                level,
+                f"{request_id_str} Enter {func_name}, args: {formatted_args}, kwargs: {formatted_kwargs}",
             )
-            return ret
+            start = time.time()
+            try:
+                ret = await func(*args, **kwargs)
+                logger.log(
+                    level,
+                    f"{request_id_str} Leave {func_name}, elapsed time: {int(time.time() - start)} s",
+                )
+                return ret
+            except Exception as e:
+                if log_exception:
+                    logger.error(
+                        f"{request_id_str} Leave {func_name}, error: {e}, elapsed time: {int(time.time() - start)} s",
+                        exc_info=True,
+                    )
+                else:
+                    logger.log(
+                        level,
+                        f"{request_id_str} Leave {func_name}, error: {e}, elapsed time: {int(time.time() - start)} s",
+                    )
+                raise
 
         return wrapped
 
     return decorator
 
 
-def log_sync(logger):
+def log_sync(logger, level=logging.DEBUG, log_exception=True):
     import time
     from functools import wraps
 
     def decorator(func):
         @wraps(func)
         def wrapped(*args, **kwargs):
-            logger.debug(f"Enter {func.__name__}, args: {args}, kwargs: {kwargs}")
-            start = time.time()
-            ret = func(*args, **kwargs)
-            logger.debug(
-                f"Leave {func.__name__}, elapsed time: {int(time.time() - start)} s"
+            formatted_args = ",".join(map(truncate_log_arg, args))
+            formatted_kwargs = ",".join(
+                map(lambda x: "%s=%s" % (x[0], truncate_log_arg(x[1])), kwargs.items())
             )
-            return ret
+            logger.log(
+                level,
+                f"Enter {func.__name__}, args: {formatted_args}, kwargs: {formatted_kwargs}",
+            )
+            start = time.time()
+            try:
+                ret = func(*args, **kwargs)
+                logger.log(
+                    level,
+                    f"Leave {func.__name__}, elapsed time: {int(time.time() - start)} s",
+                )
+                return ret
+            except Exception as e:
+                if log_exception:
+                    logger.error(
+                        f"Leave {func.__name__}, error: {e}, elapsed time: {int(time.time() - start)} s",
+                        exc_info=True,
+                    )
+                else:
+                    logger.log(
+                        level,
+                        f"Leave {func.__name__}, error: {e}, elapsed time: {int(time.time() - start)} s",
+                    )
+                raise
 
         return wrapped
 

xinference/core/worker.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import asyncio
+import logging
 import os
 import platform
 import queue
@@ -73,15 +74,15 @@ class WorkerActor(xo.StatelessActor):
         self._supervisor_ref: Optional[xo.ActorRefType] = None
         self._main_pool = main_pool
         self._main_pool.recover_sub_pool = self.recover_sub_pool
-        self._status_guard_ref: xo.ActorRefType["StatusGuardActor"] = (  # type: ignore
-            None
-        )
+        self._status_guard_ref: xo.ActorRefType[
+            "StatusGuardActor"
+        ] = None  # type: ignore
         self._event_collector_ref: xo.ActorRefType[  # type: ignore
             EventCollectorActor
         ] = None
-        self._cache_tracker_ref: xo.ActorRefType[CacheTrackerActor] = (  # type: ignore
-            None
-        )
+        self._cache_tracker_ref: xo.ActorRefType[
+            CacheTrackerActor
+        ] = None  # type: ignore
 
         # internal states.
         # temporary placeholder during model launch process:
@@ -185,7 +186,7 @@ class WorkerActor(xo.StatelessActor):
                 break
 
     @classmethod
-    def uid(cls) -> str:
+    def default_uid(cls) -> str:
         return "worker"
 
     async def __post_create__(self):
@@ -270,9 +271,9 @@ class WorkerActor(xo.StatelessActor):
 
         try:
             await self.get_supervisor_ref(add_worker=True)
-        except Exception as e:
+        except Exception:
             # Do not crash the worker if supervisor is down, auto re-connect later
-            logger.error(f"cannot connect to supervisor {e}")
+            logger.error(f"cannot connect to supervisor", exc_info=True)
 
         if not XINFERENCE_DISABLE_HEALTH_CHECK:
             from ..isolation import Isolation
@@ -324,7 +325,7 @@ class WorkerActor(xo.StatelessActor):
         if self._supervisor_ref is not None:
             return self._supervisor_ref
         supervisor_ref = await xo.actor_ref(  # type: ignore
-            address=self._supervisor_address, uid=SupervisorActor.uid()
+            address=self._supervisor_address, uid=SupervisorActor.default_uid()
         )
         # Prevent concurrent operations leads to double initialization, check again.
         if self._supervisor_ref is not None:
@@ -336,13 +337,13 @@ class WorkerActor(xo.StatelessActor):
             logger.info("Connected to supervisor as a fresh worker")
 
         self._status_guard_ref = await xo.actor_ref(
-            address=self._supervisor_address, uid=StatusGuardActor.uid()
+            address=self._supervisor_address, uid=StatusGuardActor.default_uid()
         )
         self._event_collector_ref = await xo.actor_ref(
-            address=self._supervisor_address, uid=EventCollectorActor.uid()
+            address=self._supervisor_address, uid=EventCollectorActor.default_uid()
         )
         self._cache_tracker_ref = await xo.actor_ref(
-            address=self._supervisor_address, uid=CacheTrackerActor.uid()
+            address=self._supervisor_address, uid=CacheTrackerActor.default_uid()
         )
         # cache_tracker is on supervisor
         from ..model.audio import get_audio_model_descriptions
@@ -770,7 +771,7 @@ class WorkerActor(xo.StatelessActor):
                 version_info["model_file_location"],
             )
 
-    @log_async(logger=logger)
+    @log_async(logger=logger, level=logging.INFO)
     async def launch_builtin_model(
         self,
         model_uid: str,
@@ -814,7 +815,7 @@ class WorkerActor(xo.StatelessActor):
                 )
         except Exception as e:
             # Report callback error can be log and ignore, should not interrupt the Process
-            logger.error("report_event error: %s" % (e))
+            logger.error("report_event error: %s" % (e), exc_info=True)
 
         if gpu_idx is not None:
             logger.info(
@@ -917,7 +918,7 @@ class WorkerActor(xo.StatelessActor):
             {"model_ability": abilities, "status": LaunchStatus.READY.name},
         )
 
-    @log_async(logger=logger)
+    @log_async(logger=logger, level=logging.INFO)
     async def terminate_model(self, model_uid: str, is_model_die=False):
         # Terminate model while its launching is not allow
         if model_uid in self._model_uid_launching_guard:

xinference/deploy/cmdline.py

@@ -17,7 +17,7 @@ import logging
 import os
 import sys
 import warnings
-from typing import List, Optional, Sequence, Tuple, Union
+from typing import Dict, List, Optional, Sequence, Tuple, Union
 
 import click
 from xoscar.utils import get_next_port
@@ -38,7 +38,6 @@ from ..constants import (
     XINFERENCE_LOG_MAX_BYTES,
 )
 from ..isolation import Isolation
-from ..types import ChatCompletionMessage
 from .utils import (
     get_config_dict,
     get_log_file,
@@ -1210,13 +1209,12 @@ def model_chat(
     stream: bool,
     api_key: Optional[str],
 ):
-    # TODO: chat model roles may not be user and assistant.
     endpoint = get_endpoint(endpoint)
     client = RESTfulClient(base_url=endpoint, api_key=api_key)
     if api_key is None:
         client._set_token(get_stored_token(endpoint, client))
 
-    chat_history: "List[ChatCompletionMessage]" = []
+    messages: List[Dict] = []
     if stream:
         # TODO: when stream=True, RestfulClient cannot generate words one by one.
         # So use Client in temporary. The implementation needs to be changed to
@@ -1229,10 +1227,10 @@ def model_chat(
                 if prompt == "":
                     break
                 print("Assistant: ", end="", file=sys.stdout)
+                messages.append(dict(role="user", content=prompt))
                 response_content = ""
                 for chunk in model.chat(
-                    prompt=prompt,
-                    chat_history=chat_history,
+                    messages,
                     generate_config={"stream": stream, "max_tokens": max_tokens},
                 ):
                     delta = chunk["choices"][0]["delta"]
@@ -1242,10 +1240,7 @@ def model_chat(
                         response_content += delta["content"]
                         print(delta["content"], end="", flush=True, file=sys.stdout)
                 print("", file=sys.stdout)
-                chat_history.append(ChatCompletionMessage(role="user", content=prompt))
-                chat_history.append(
-                    ChatCompletionMessage(role="assistant", content=response_content)
-                )
+                messages.append(dict(role="assistant", content=response_content))
 
         model = client.get_model(model_uid=model_uid)
 
@@ -1274,20 +1269,17 @@ def model_chat(
             prompt = input("User: ")
             if prompt == "":
                 break
-            chat_history.append(ChatCompletionMessage(role="user", content=prompt))
+            messages.append({"role": "user", "content": prompt})
             print("Assistant: ", end="", file=sys.stdout)
             response = restful_model.chat(
-                prompt=prompt,
-                chat_history=chat_history,
+                messages,
                 generate_config={"stream": stream, "max_tokens": max_tokens},
             )
             if not isinstance(response, dict):
                 raise ValueError("chat result is not valid")
             response_content = response["choices"][0]["message"]["content"]
             print(f"{response_content}\n", file=sys.stdout)
-            chat_history.append(
-                ChatCompletionMessage(role="assistant", content=response_content)
-            )
+            messages.append(dict(role="assistant", content=response_content))
 
 
 @cli.command("vllm-models", help="Query and display models compatible with vLLM.")

xinference/deploy/local.py

@@ -49,7 +49,7 @@ async def _start_local_cluster(
             address=address, logging_conf=logging_conf
         )
         await xo.create_actor(
-            SupervisorActor, address=address, uid=SupervisorActor.uid()
+            SupervisorActor, address=address, uid=SupervisorActor.default_uid()
         )
         await start_worker_components(
             address=address,

xinference/deploy/supervisor.py

@@ -41,7 +41,7 @@ async def _start_supervisor(address: str, logging_conf: Optional[Dict] = None):
             address=address, n_process=0, logging_conf={"dict": logging_conf}
         )
         await xo.create_actor(
-            SupervisorActor, address=address, uid=SupervisorActor.uid()
+            SupervisorActor, address=address, uid=SupervisorActor.default_uid()
         )
         await pool.join()
     except asyncio.exceptions.CancelledError:

xinference/deploy/utils.py

@@ -167,7 +167,7 @@ def health_check(address: str, max_attempts: int, sleep_interval: int = 3) -> bo
                 from ..core.supervisor import SupervisorActor
 
                 supervisor_ref: xo.ActorRefType[SupervisorActor] = await xo.actor_ref(  # type: ignore
-                    address=address, uid=SupervisorActor.uid()
+                    address=address, uid=SupervisorActor.default_uid()
                 )
 
                 await supervisor_ref.get_status()

xinference/deploy/worker.py

@@ -43,7 +43,7 @@ async def start_worker_components(
     await xo.create_actor(
         WorkerActor,
         address=address,
-        uid=WorkerActor.uid(),
+        uid=WorkerActor.default_uid(),
         supervisor_address=supervisor_address,
         main_pool=main_pool,
         gpu_devices=gpu_device_indices,

xinference/model/audio/cosyvoice.py

@@ -53,7 +53,82 @@ class CosyVoiceModel:
 
         from cosyvoice.cli.cosyvoice import CosyVoice
 
-        self._model = CosyVoice(self._model_path)
+        self._model = CosyVoice(
+            self._model_path, load_jit=self._kwargs.get("load_jit", False)
+        )
+
+    def _speech_handle(
+        self,
+        stream,
+        input,
+        instruct_text,
+        prompt_speech,
+        prompt_text,
+        voice,
+        response_format,
+    ):
+        if prompt_speech:
+            from cosyvoice.utils.file_utils import load_wav
+
+            with io.BytesIO(prompt_speech) as prompt_speech_io:
+                prompt_speech_16k = load_wav(prompt_speech_io, 16000)
+
+            if prompt_text:
+                logger.info("CosyVoice inference_zero_shot")
+                output = self._model.inference_zero_shot(
+                    input, prompt_text, prompt_speech_16k, stream=stream
+                )
+            else:
+                logger.info("CosyVoice inference_cross_lingual")
+                output = self._model.inference_cross_lingual(
+                    input, prompt_speech_16k, stream=stream
+                )
+        else:
+            available_speakers = self._model.list_avaliable_spks()
+            if not voice:
+                voice = available_speakers[0]
+            else:
+                assert (
+                    voice in available_speakers
+                ), f"Invalid voice {voice}, CosyVoice available speakers: {available_speakers}"
+            if instruct_text:
+                logger.info("CosyVoice inference_instruct")
+                output = self._model.inference_instruct(
+                    input, voice, instruct_text=instruct_text, stream=stream
+                )
+            else:
+                logger.info("CosyVoice inference_sft")
+                output = self._model.inference_sft(input, voice, stream=stream)
+
+        import torch
+        import torchaudio
+
+        def _generator_stream():
+            with BytesIO() as out:
+                writer = torchaudio.io.StreamWriter(out, format=response_format)
+                writer.add_audio_stream(sample_rate=22050, num_channels=1)
+                i = 0
+                last_pos = 0
+                with writer.open():
+                    for chunk in output:
+                        chunk = chunk["tts_speech"]
+                        trans_chunk = torch.transpose(chunk, 0, 1)
+                        writer.write_audio_chunk(i, trans_chunk)
+                        new_last_pos = out.tell()
+                        if new_last_pos != last_pos:
+                            out.seek(last_pos)
+                            encoded_bytes = out.read()
+                            yield encoded_bytes
+                            last_pos = new_last_pos
+
+        def _generator_block():
+            chunk = next(output)
+            assert isinstance(chunk, dict), "Expected data to be of type dict"
+            with BytesIO() as out:
+                torchaudio.save(out, chunk["tts_speech"], 22050, format=response_format)
+                return out.getvalue()
+
+        return _generator_stream() if stream else _generator_block()
 
     def speech(
         self,
@@ -64,12 +139,6 @@ class CosyVoiceModel:
         stream: bool = False,
         **kwargs,
     ):
-        if stream:
-            raise Exception("CosyVoiceModel does not support stream.")
-
-        import torchaudio
-        from cosyvoice.utils.file_utils import load_wav
-
         prompt_speech: Optional[bytes] = kwargs.pop("prompt_speech", None)
         prompt_text: Optional[str] = kwargs.pop("prompt_text", None)
         instruct_text: Optional[str] = kwargs.pop("instruct_text", None)
@@ -103,39 +172,15 @@ class CosyVoiceModel:
             ), "CosyVoice model does not support instruct_text"
 
         assert self._model is not None
+
         set_all_random_seed(seed)
-        if prompt_speech:
-            assert not voice, "voice can't be set with prompt speech."
-            with io.BytesIO(prompt_speech) as prompt_speech_io:
-                prompt_speech_16k = load_wav(prompt_speech_io, 16000)
-                if prompt_text:
-                    logger.info("CosyVoice inference_zero_shot")
-                    output = self._model.inference_zero_shot(
-                        input, prompt_text, prompt_speech_16k
-                    )
-                else:
-                    logger.info("CosyVoice inference_cross_lingual")
-                    output = self._model.inference_cross_lingual(
-                        input, prompt_speech_16k
-                    )
-        else:
-            available_speakers = self._model.list_avaliable_spks()
-            if not voice:
-                voice = available_speakers[0]
-            else:
-                assert (
-                    voice in available_speakers
-                ), f"Invalid voice {voice}, CosyVoice available speakers: {available_speakers}"
-            if instruct_text:
-                logger.info("CosyVoice inference_instruct")
-                output = self._model.inference_instruct(
-                    input, voice, instruct_text=instruct_text
-                )
-            else:
-                logger.info("CosyVoice inference_sft")
-                output = self._model.inference_sft(input, voice)
 
-        # Save the generated audio
-        with BytesIO() as out:
-            torchaudio.save(out, output["tts_speech"], 22050, format=response_format)
-            return out.getvalue()
+        return self._speech_handle(
+            stream,
+            input,
+            instruct_text,
+            prompt_speech,
+            prompt_text,
+            voice,
+            response_format,
+        )

xinference/model/audio/fish_speech.py

@@ -92,7 +92,7 @@ class FishSpeechModel:
 
         checkpoint_path = os.path.join(
             self._model_path,
-            "firefly-gan-vq-fsq-4x1024-42hz-generator.pth",
+            "firefly-gan-vq-fsq-8x1024-21hz-generator.pth",
         )
         self._model = load_decoder_model(
             config_name="firefly_gan_vq",
@@ -159,11 +159,11 @@ class FishSpeechModel:
         segments = []
 
         while True:
-            result: WrappedGenerateResponse = response_queue.get()
+            result: WrappedGenerateResponse = response_queue.get()  # type: ignore
             if result.status == "error":
                 raise Exception(str(result.response))
 
-            result: GenerateResponse = result.response
+            result: GenerateResponse = result.response  # type: ignore
             if result.action == "next":
                 break
 
@@ -213,12 +213,12 @@ class FishSpeechModel:
                 text=input,
                 enable_reference_audio=False,
                 reference_audio=None,
-                reference_text="",
-                max_new_tokens=0,
-                chunk_length=100,
-                top_p=0.7,
-                repetition_penalty=1.2,
-                temperature=0.7,
+                reference_text=kwargs.get("reference_text", ""),
+                max_new_tokens=kwargs.get("max_new_tokens", 1024),
+                chunk_length=kwargs.get("chunk_length", 200),
+                top_p=kwargs.get("top_p", 0.7),
+                repetition_penalty=kwargs.get("repetition_penalty", 1.2),
+                temperature=kwargs.get("temperature", 0.7),
             )
         )
         sample_rate, audio = result[0][1]

xinference/model/audio/model_spec.json

@@ -126,32 +126,32 @@
   {
     "model_name": "CosyVoice-300M",
     "model_family": "CosyVoice",
-    "model_id": "model-scope/CosyVoice-300M",
-    "model_revision": "ca4e036d2db2aa4731cc1747859a68044b6a4694",
+    "model_id": "FunAudioLLM/CosyVoice-300M",
+    "model_revision": "39c4e13d46bd4dfb840d214547623e5fcd2428e2",
     "model_ability": "audio-to-audio",
     "multilingual": true
   },
   {
     "model_name": "CosyVoice-300M-SFT",
     "model_family": "CosyVoice",
-    "model_id": "model-scope/CosyVoice-300M-SFT",
-    "model_revision": "ab918940c6c134b1fc1f069246e67bad6b66abcb",
+    "model_id": "FunAudioLLM/CosyVoice-300M-SFT",
+    "model_revision": "096a5cff8d497fabb3dec2756a200f3688457a1b",
     "model_ability": "text-to-audio",
     "multilingual": true
   },
   {
     "model_name": "CosyVoice-300M-Instruct",
     "model_family": "CosyVoice",
-    "model_id": "model-scope/CosyVoice-300M-Instruct",
-    "model_revision": "fb5f676733139f35670bed9b59a77d476b1aa898",
+    "model_id": "FunAudioLLM/CosyVoice-300M-Instruct",
+    "model_revision": "ba5265d9a3169c1fedce145122c9dd4bc24e062c",
     "model_ability": "text-to-audio",
     "multilingual": true
   },
   {
-    "model_name": "FishSpeech-1.2-SFT",
+    "model_name": "FishSpeech-1.4",
     "model_family": "FishAudio",
-    "model_id": "fishaudio/fish-speech-1.2-sft",
-    "model_revision": "180288e21ec5c50cfc564023a22f789e4b88a0e0",
+    "model_id": "fishaudio/fish-speech-1.4",
+    "model_revision": "3c49651b8e583b6b13f55e375432e0d57e1aa84d",
     "model_ability": "text-to-audio",
     "multilingual": true
   }

xinference/model/audio/whisper.py

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
+import os
+from glob import glob
 from typing import TYPE_CHECKING, Dict, List, Optional, Union
 
 from ...device_utils import (
@@ -56,12 +58,13 @@ class WhisperModel:
                 raise ValueError(f"Device {self._device} is not available!")
 
         torch_dtype = get_device_preferred_dtype(self._device)
+        use_safetensors = any(glob(os.path.join(self._model_path, "*.safetensors")))
 
         model = AutoModelForSpeechSeq2Seq.from_pretrained(
             self._model_path,
             torch_dtype=torch_dtype,
             low_cpu_mem_usage=True,
-            use_safetensors=True,
+            use_safetensors=use_safetensors,
         )
         model.to(self._device)