xinference 0.14.0__py3-none-any.whl → 0.14.1__py3-none-any.whl

xinference/core/worker.py

@@ -68,7 +68,7 @@ class WorkerActor(xo.StatelessActor):
         # static attrs.
         self._total_gpu_devices = gpu_devices
         self._supervisor_address = supervisor_address
-        self._supervisor_ref = None
+        self._supervisor_ref: Optional[xo.ActorRefType] = None
         self._main_pool = main_pool
         self._main_pool.recover_sub_pool = self.recover_sub_pool
 
@@ -147,17 +147,20 @@ class WorkerActor(xo.StatelessActor):
                             )
                             event_model_uid, _, __ = parse_replica_model_uid(model_uid)
                             try:
-                                await self._event_collector_ref.report_event(
-                                    event_model_uid,
-                                    Event(
-                                        event_type=EventType.WARNING,
-                                        event_ts=int(time.time()),
-                                        event_content="Recreate model",
-                                    ),
-                                )
+                                if self._event_collector_ref is not None:
+                                    await self._event_collector_ref.report_event(
+                                        event_model_uid,
+                                        Event(
+                                            event_type=EventType.WARNING,
+                                            event_ts=int(time.time()),
+                                            event_content="Recreate model",
+                                        ),
+                                    )
                             except Exception as e:
                                 # Report callback error can be log and ignore, should not interrupt the Process
                                 logger.error("report_event error: %s" % (e))
+                            finally:
+                                del event_model_uid
 
                             self._model_uid_to_recover_count[model_uid] = (
                                 recover_count - 1
@@ -175,79 +178,39 @@ class WorkerActor(xo.StatelessActor):
         return "worker"
 
     async def __post_create__(self):
-        from ..isolation import Isolation
-        from .cache_tracker import CacheTrackerActor
-        from .status_guard import StatusGuardActor
-        from .supervisor import SupervisorActor
-
-        self._status_guard_ref: xo.ActorRefType[  # type: ignore
-            "StatusGuardActor"
-        ] = await xo.actor_ref(
-            address=self._supervisor_address, uid=StatusGuardActor.uid()
-        )
-        self._event_collector_ref: xo.ActorRefType[  # type: ignore
-            EventCollectorActor
-        ] = await xo.actor_ref(
-            address=self._supervisor_address, uid=EventCollectorActor.uid()
-        )
-        self._cache_tracker_ref: xo.ActorRefType[  # type: ignore
-            "CacheTrackerActor"
-        ] = await xo.actor_ref(
-            address=self._supervisor_address, uid=CacheTrackerActor.uid()
-        )
-        self._supervisor_ref: xo.ActorRefType["SupervisorActor"] = await xo.actor_ref(  # type: ignore
-            address=self._supervisor_address, uid=SupervisorActor.uid()
-        )
-        await self._supervisor_ref.add_worker(self.address)
-        if not XINFERENCE_DISABLE_HEALTH_CHECK:
-            # Run _periodical_report_status() in a dedicated thread.
-            self._isolation = Isolation(asyncio.new_event_loop(), threaded=True)
-            self._isolation.start()
-            asyncio.run_coroutine_threadsafe(
-                self._periodical_report_status(), loop=self._isolation.loop
-            )
-        logger.info(f"Xinference worker {self.address} started")
-        logger.info("Purge cache directory: %s", XINFERENCE_CACHE_DIR)
-        purge_dir(XINFERENCE_CACHE_DIR)
-
         from ..model.audio import (
             CustomAudioModelFamilyV1,
             generate_audio_description,
-            get_audio_model_descriptions,
             register_audio,
             unregister_audio,
         )
         from ..model.embedding import (
             CustomEmbeddingModelSpec,
             generate_embedding_description,
-            get_embedding_model_descriptions,
             register_embedding,
             unregister_embedding,
         )
         from ..model.flexible import (
             FlexibleModelSpec,
-            get_flexible_model_descriptions,
+            generate_flexible_model_description,
             register_flexible_model,
             unregister_flexible_model,
         )
         from ..model.image import (
             CustomImageModelFamilyV1,
             generate_image_description,
-            get_image_model_descriptions,
             register_image,
             unregister_image,
         )
         from ..model.llm import (
             CustomLLMFamilyV1,
             generate_llm_description,
-            get_llm_model_descriptions,
             register_llm,
             unregister_llm,
         )
         from ..model.rerank import (
             CustomRerankModelSpec,
             generate_rerank_description,
-            get_rerank_model_descriptions,
             register_rerank,
             unregister_rerank,
         )
@@ -287,27 +250,37 @@ class WorkerActor(xo.StatelessActor):
                 FlexibleModelSpec,
                 register_flexible_model,
                 unregister_flexible_model,
+                generate_flexible_model_description,
             ),
         }
 
-        # record model version
-        model_version_infos: Dict[str, List[Dict]] = {}  # type: ignore
-        model_version_infos.update(get_llm_model_descriptions())
-        model_version_infos.update(get_embedding_model_descriptions())
-        model_version_infos.update(get_rerank_model_descriptions())
-        model_version_infos.update(get_image_model_descriptions())
-        model_version_infos.update(get_audio_model_descriptions())
-        model_version_infos.update(get_flexible_model_descriptions())
-        await self._cache_tracker_ref.record_model_version(
-            model_version_infos, self.address
-        )
+        logger.info("Purge cache directory: %s", XINFERENCE_CACHE_DIR)
+        purge_dir(XINFERENCE_CACHE_DIR)
+
+        try:
+            await self.get_supervisor_ref(add_worker=True)
+        except Exception as e:
+            # Do not crash the worker if supervisor is down, auto re-connect later
+            logger.error(f"cannot connect to supervisor {e}")
+
+        if not XINFERENCE_DISABLE_HEALTH_CHECK:
+            from ..isolation import Isolation
+
+            # Run _periodical_report_status() in a dedicated thread.
+            self._isolation = Isolation(asyncio.new_event_loop(), threaded=True)
+            self._isolation.start()
+            asyncio.run_coroutine_threadsafe(
+                self._periodical_report_status(), loop=self._isolation.loop
+            )
+        logger.info(f"Xinference worker {self.address} started")
 
         # Windows does not have signal handler
         if os.name != "nt":
 
             async def signal_handler():
                 try:
-                    await self._supervisor_ref.remove_worker(self.address)
+                    supervisor_ref = await self.get_supervisor_ref(add_worker=False)
+                    await supervisor_ref.remove_worker(self.address)
                 except Exception as e:
                     # Ignore the error of rpc, anyway we are exiting
                     logger.exception("remove worker rpc error: %s", e)
@@ -329,6 +302,64 @@ class WorkerActor(xo.StatelessActor):
             return False
         return True
 
+    async def get_supervisor_ref(self, add_worker: bool = True) -> xo.ActorRefType:
+        """
+        Try connect to supervisor and return ActorRef. Raise exception on error
+        Params:
+            add_worker: By default will call supervisor.add_worker after first connect
+        """
+        from .status_guard import StatusGuardActor
+        from .supervisor import SupervisorActor
+
+        if self._supervisor_ref is not None:
+            return self._supervisor_ref
+        self._supervisor_ref: xo.ActorRefType["SupervisorActor"] = await xo.actor_ref(  # type: ignore
+            address=self._supervisor_address, uid=SupervisorActor.uid()
+        )
+        if add_worker and len(self._model_uid_to_model) == 0:
+            # Newly started (or restarted), has no model, notify supervisor
+            await self._supervisor_ref.add_worker(self.address)
+            logger.info("Connected to supervisor as a fresh worker")
+
+            self._status_guard_ref: xo.ActorRefType[  # type: ignore
+                "StatusGuardActor"
+            ] = await xo.actor_ref(
+                address=self._supervisor_address, uid=StatusGuardActor.uid()
+            )
+
+            self._event_collector_ref: xo.ActorRefType[  # type: ignore
+                EventCollectorActor
+            ] = await xo.actor_ref(
+                address=self._supervisor_address, uid=EventCollectorActor.uid()
+            )
+            from .cache_tracker import CacheTrackerActor
+
+            self._cache_tracker_ref: xo.ActorRefType[  # type: ignore
+                "CacheTrackerActor"
+            ] = await xo.actor_ref(
+                address=self._supervisor_address, uid=CacheTrackerActor.uid()
+            )
+            # cache_tracker is on supervisor
+            from ..model.audio import get_audio_model_descriptions
+            from ..model.embedding import get_embedding_model_descriptions
+            from ..model.flexible import get_flexible_model_descriptions
+            from ..model.image import get_image_model_descriptions
+            from ..model.llm import get_llm_model_descriptions
+            from ..model.rerank import get_rerank_model_descriptions
+
+            # record model version
+            model_version_infos: Dict[str, List[Dict]] = {}  # type: ignore
+            model_version_infos.update(get_llm_model_descriptions())
+            model_version_infos.update(get_embedding_model_descriptions())
+            model_version_infos.update(get_rerank_model_descriptions())
+            model_version_infos.update(get_image_model_descriptions())
+            model_version_infos.update(get_audio_model_descriptions())
+            model_version_infos.update(get_flexible_model_descriptions())
+            await self._cache_tracker_ref.record_model_version(
+                model_version_infos, self.address
+            )
+        return self._supervisor_ref
+
     @staticmethod
     def get_devices_count():
         from ..device_utils import gpu_count
@@ -340,9 +371,9 @@ class WorkerActor(xo.StatelessActor):
         return len(self._model_uid_to_model)
 
     async def is_model_vllm_backend(self, model_uid: str) -> bool:
-        assert self._supervisor_ref is not None
         _model_uid, _, _ = parse_replica_model_uid(model_uid)
-        model_ref = await self._supervisor_ref.get_model(_model_uid)
+        supervisor_ref = await self.get_supervisor_ref()
+        model_ref = await supervisor_ref.get_model(_model_uid)
         return await model_ref.is_vllm_backend()
 
     async def allocate_devices_for_embedding(self, model_uid: str) -> int:
@@ -704,6 +735,8 @@ class WorkerActor(xo.StatelessActor):
             return ["text_to_image"]
         elif model_type == "audio":
             return ["audio_to_text"]
+        elif model_type == "video":
+            return ["text_to_video"]
         elif model_type == "flexible":
             return ["flexible"]
         else:
@@ -760,14 +793,15 @@ class WorkerActor(xo.StatelessActor):
             logger.exception(e)
             raise
         try:
-            await self._event_collector_ref.report_event(
-                origin_uid,
-                Event(
-                    event_type=EventType.INFO,
-                    event_ts=int(time.time()),
-                    event_content="Launch model",
-                ),
-            )
+            if self._event_collector_ref is not None:
+                await self._event_collector_ref.report_event(
+                    origin_uid,
+                    Event(
+                        event_type=EventType.INFO,
+                        event_ts=int(time.time()),
+                        event_content="Launch model",
+                    ),
+                )
         except Exception as e:
             # Report callback error can be log and ignore, should not interrupt the Process
             logger.error("report_event error: %s" % (e))
@@ -863,6 +897,11 @@ class WorkerActor(xo.StatelessActor):
 
         # update status to READY
         abilities = await self._get_model_ability(model, model_type)
+        _ = await self.get_supervisor_ref(add_worker=False)
+
+        if self._status_guard_ref is None:
+            _ = await self.get_supervisor_ref()
+        assert self._status_guard_ref is not None
         await self._status_guard_ref.update_instance_info(
             origin_uid,
             {"model_ability": abilities, "status": LaunchStatus.READY.name},
@@ -875,21 +914,23 @@ class WorkerActor(xo.StatelessActor):
             raise ValueError(f"{model_uid} is launching")
         origin_uid, _, __ = parse_replica_model_uid(model_uid)
         try:
-            await self._event_collector_ref.report_event(
-                origin_uid,
-                Event(
-                    event_type=EventType.INFO,
-                    event_ts=int(time.time()),
-                    event_content="Terminate model",
-                ),
-            )
+            if self._event_collector_ref is not None:
+                await self._event_collector_ref.report_event(
+                    origin_uid,
+                    Event(
+                        event_type=EventType.INFO,
+                        event_ts=int(time.time()),
+                        event_content="Terminate model",
+                    ),
+                )
         except Exception as e:
             # Report callback error can be log and ignore, should not interrupt the Process
             logger.error("report_event error: %s" % (e))
 
-        await self._status_guard_ref.update_instance_info(
-            origin_uid, {"status": LaunchStatus.TERMINATING.name}
-        )
+        if self._status_guard_ref is not None:
+            await self._status_guard_ref.update_instance_info(
+                origin_uid, {"status": LaunchStatus.TERMINATING.name}
+            )
         model_ref = self._model_uid_to_model.get(model_uid, None)
         if model_ref is None:
             logger.debug("Model not found, uid: %s", model_uid)
@@ -914,6 +955,10 @@ class WorkerActor(xo.StatelessActor):
             self._model_uid_to_addr.pop(model_uid, None)
             self._model_uid_to_recover_count.pop(model_uid, None)
             self._model_uid_to_launch_args.pop(model_uid, None)
+
+            if self._status_guard_ref is None:
+                _ = await self.get_supervisor_ref()
+            assert self._status_guard_ref is not None
             await self._status_guard_ref.update_instance_info(
                 origin_uid, {"status": LaunchStatus.TERMINATED.name}
             )
@@ -966,7 +1011,8 @@ class WorkerActor(xo.StatelessActor):
             raise
         except Exception:
             logger.exception("Report status got error.")
-        await self._supervisor_ref.report_worker_status(self.address, status)
+        supervisor_ref = await self.get_supervisor_ref()
+        await supervisor_ref.report_worker_status(self.address, status)
 
     async def _periodical_report_status(self):
         while True:

xinference/deploy/cmdline.py

@@ -25,7 +25,6 @@ from xoscar.utils import get_next_port
 from .. import __version__
 from ..client import RESTfulClient
 from ..client.restful.restful_client import (
-    RESTfulChatglmCppChatModelHandle,
     RESTfulChatModelHandle,
     RESTfulGenerateModelHandle,
 )
@@ -1268,9 +1267,7 @@ def model_chat(
                 task.exception()
     else:
         restful_model = client.get_model(model_uid=model_uid)
-        if not isinstance(
-            restful_model, (RESTfulChatModelHandle, RESTfulChatglmCppChatModelHandle)
-        ):
+        if not isinstance(restful_model, RESTfulChatModelHandle):
             raise ValueError(f"model {model_uid} has no chat method")
 
         while True:

xinference/model/audio/core.py

@@ -14,13 +14,14 @@
 import logging
 import os
 from collections import defaultdict
-from typing import Dict, List, Literal, Optional, Tuple, Union
+from typing import Any, Dict, List, Literal, Optional, Tuple, Union
 
 from ...constants import XINFERENCE_CACHE_DIR
 from ..core import CacheableModelSpec, ModelDescription
 from ..utils import valid_model_revision
 from .chattts import ChatTTSModel
 from .cosyvoice import CosyVoiceModel
+from .funasr import FunASRModel
 from .whisper import WhisperModel
 
 MAX_ATTEMPTS = 3
@@ -45,6 +46,8 @@ class AudioModelFamilyV1(CacheableModelSpec):
     model_id: str
     model_revision: str
     multilingual: bool
+    default_model_config: Optional[Dict[str, Any]]
+    default_transcription_config: Optional[Dict[str, Any]]
 
 
 class AudioModelDescription(ModelDescription):
@@ -152,13 +155,18 @@ def create_audio_model_instance(
     download_hub: Optional[Literal["huggingface", "modelscope", "csghub"]] = None,
     model_path: Optional[str] = None,
     **kwargs,
-) -> Tuple[Union[WhisperModel, ChatTTSModel, CosyVoiceModel], AudioModelDescription]:
+) -> Tuple[
+    Union[WhisperModel, FunASRModel, ChatTTSModel, CosyVoiceModel],
+    AudioModelDescription,
+]:
     model_spec = match_audio(model_name, download_hub)
     if model_path is None:
         model_path = cache(model_spec)
-    model: Union[WhisperModel, ChatTTSModel, CosyVoiceModel]
+    model: Union[WhisperModel, FunASRModel, ChatTTSModel, CosyVoiceModel]
     if model_spec.model_family == "whisper":
         model = WhisperModel(model_uid, model_path, model_spec, **kwargs)
+    elif model_spec.model_family == "funasr":
+        model = FunASRModel(model_uid, model_path, model_spec, **kwargs)
     elif model_spec.model_family == "ChatTTS":
         model = ChatTTSModel(model_uid, model_path, model_spec, **kwargs)
     elif model_spec.model_family == "CosyVoice":

xinference/model/audio/funasr.py

@@ -0,0 +1,114 @@
+# Copyright 2022-2023 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import tempfile
+from typing import TYPE_CHECKING, List, Optional
+
+from ...device_utils import get_available_device, is_device_available
+
+if TYPE_CHECKING:
+    from .core import AudioModelFamilyV1
+
+logger = logging.getLogger(__name__)
+
+
+class FunASRModel:
+    def __init__(
+        self,
+        model_uid: str,
+        model_path: str,
+        model_spec: "AudioModelFamilyV1",
+        device: Optional[str] = None,
+        **kwargs,
+    ):
+        self._model_uid = model_uid
+        self._model_path = model_path
+        self._model_spec = model_spec
+        self._device = device
+        self._model = None
+        self._kwargs = kwargs
+
+    def load(self):
+        try:
+            from funasr import AutoModel
+        except ImportError:
+            error_message = "Failed to import module 'funasr'"
+            installation_guide = [
+                "Please make sure 'funasr' is installed. ",
+                "You can install it by `pip install funasr`\n",
+            ]
+
+            raise ImportError(f"{error_message}\n\n{''.join(installation_guide)}")
+
+        if self._device is None:
+            self._device = get_available_device()
+        else:
+            if not is_device_available(self._device):
+                raise ValueError(f"Device {self._device} is not available!")
+
+        kwargs = self._model_spec.default_model_config.copy()
+        kwargs.update(self._kwargs)
+        logger.debug("Loading FunASR model with kwargs: %s", kwargs)
+        self._model = AutoModel(model=self._model_path, device=self._device, **kwargs)
+
+    def transcriptions(
+        self,
+        audio: bytes,
+        language: Optional[str] = None,
+        prompt: Optional[str] = None,
+        response_format: str = "json",
+        temperature: float = 0,
+        timestamp_granularities: Optional[List[str]] = None,
+        **kwargs,
+    ):
+        from funasr.utils.postprocess_utils import rich_transcription_postprocess
+
+        if temperature != 0:
+            raise RuntimeError("`temperature`is not supported for FunASR")
+        if timestamp_granularities is not None:
+            raise RuntimeError("`timestamp_granularities`is not supported for FunASR")
+        if prompt is not None:
+            logger.warning(
+                "Prompt for funasr transcriptions will be ignored: %s", prompt
+            )
+
+        language = "auto" if language is None else language
+
+        with tempfile.NamedTemporaryFile(buffering=0) as f:
+            f.write(audio)
+
+            kw = self._model_spec.default_transcription_config.copy()  # type: ignore
+            kw.update(kwargs)
+            logger.debug("Calling FunASR model with kwargs: %s", kw)
+            result = self._model.generate(  # type: ignore
+                input=f.name, cache={}, language=language, **kw
+            )
+            text = rich_transcription_postprocess(result[0]["text"])
+
+            if response_format == "json":
+                return {"text": text}
+            else:
+                raise ValueError(f"Unsupported response format: {response_format}")
+
+    def translations(
+        self,
+        audio: bytes,
+        language: Optional[str] = None,
+        prompt: Optional[str] = None,
+        response_format: str = "json",
+        temperature: float = 0,
+        timestamp_granularities: Optional[List[str]] = None,
+    ):
+        raise RuntimeError("FunASR does not support translations API")

xinference/model/audio/model_spec.json

@@ -95,6 +95,26 @@
     "ability": "audio-to-text",
     "multilingual": false
   },
+  {
+    "model_name": "SenseVoiceSmall",
+    "model_family": "funasr",
+    "model_id": "FunAudioLLM/SenseVoiceSmall",
+    "model_revision": "3eb3b4eeffc2f2dde6051b853983753db33e35c3",
+    "ability": "audio-to-text",
+    "multilingual": true,
+    "default_model_config": {
+      "vad_model": "fsmn-vad",
+      "vad_kwargs": {
+        "max_single_segment_time": 30000
+      }
+    },
+    "default_transcription_config": {
+      "use_itn": true,
+      "batch_size_s": 60,
+      "merge_vad": true,
+      "merge_length_s": 15
+    }
+  },
   {
     "model_name": "ChatTTS",
     "model_family": "ChatTTS",

xinference/model/audio/model_spec_modelscope.json

@@ -8,6 +8,27 @@
     "ability": "audio-to-text",
     "multilingual": true
   },
+  {
+    "model_name": "SenseVoiceSmall",
+    "model_family": "funasr",
+    "model_hub": "modelscope",
+    "model_id": "iic/SenseVoiceSmall",
+    "model_revision": "master",
+    "ability": "audio-to-text",
+    "multilingual": true,
+    "default_model_config": {
+      "vad_model": "fsmn-vad",
+      "vad_kwargs": {
+        "max_single_segment_time": 30000
+      }
+    },
+    "default_transcription_config": {
+      "use_itn": true,
+      "batch_size_s": 60,
+      "merge_vad": true,
+      "merge_length_s": 15
+    }
+  },
   {
     "model_name": "ChatTTS",
     "model_family": "ChatTTS",

xinference/model/audio/whisper.py

@@ -14,7 +14,7 @@
 import logging
 from typing import TYPE_CHECKING, Dict, List, Optional, Union
 
-from xinference.device_utils import (
+from ...device_utils import (
     get_available_device,
     get_device_preferred_dtype,
     is_device_available,

xinference/model/core.py

@@ -65,6 +65,7 @@ def create_model_instance(
     from .image.core import create_image_model_instance
     from .llm.core import create_llm_model_instance
     from .rerank.core import create_rerank_model_instance
+    from .video.core import create_video_model_instance
 
     if model_type == "LLM":
         return create_llm_model_instance(
@@ -127,6 +128,17 @@ def create_model_instance(
             model_path,
             **kwargs,
         )
+    elif model_type == "video":
+        kwargs.pop("trust_remote_code", None)
+        return create_video_model_instance(
+            subpool_addr,
+            devices,
+            model_uid,
+            model_name,
+            download_hub,
+            model_path,
+            **kwargs,
+        )
     elif model_type == "flexible":
         kwargs.pop("trust_remote_code", None)
         return create_flexible_model_instance(

xinference/model/embedding/core.py

@@ -151,8 +151,8 @@ class EmbeddingModel:
 
         patch_trust_remote_code()
         if (
-            "gte-Qwen2" in self._model_spec.model_id
-            or "gte-Qwen2" in self._model_spec.model_name
+            "gte" in self._model_spec.model_name.lower()
+            and "qwen2" in self._model_spec.model_name.lower()
         ):
             self._model = XSentenceTransformer(
                 self._model_path,
@@ -260,8 +260,8 @@ class EmbeddingModel:
                 device = model._target_device
 
             if (
-                "gte-Qwen2" not in self._model_spec.model_id
-                and "gte-Qwen2" not in self._model_spec.model_name
+                "gte" in self._model_spec.model_name.lower()
+                and "qwen2" in self._model_spec.model_name.lower()
             ):
                 model.to(device)
 
@@ -342,8 +342,8 @@ class EmbeddingModel:
             return all_embeddings, all_token_nums
 
         if (
-            "gte-Qwen2" in self._model_spec.model_id
-            or "gte-Qwen2" in self._model_spec.model_name
+            "gte" in self._model_spec.model_name.lower()
+            and "qwen2" in self._model_spec.model_name.lower()
         ):
             all_embeddings, all_token_nums = encode(
                 self._model,