xinference 1.5.0.post2__py3-none-any.whl → 1.6.0__py3-none-any.whl

xinference/core/model.py

@@ -632,6 +632,8 @@ class ModelActor(xo.StatelessActor, CancelMixin):
                     return await _gen.__anext__()  # noqa: F821
                 except StopAsyncIteration:
                     return stop
+                except Exception as e:
+                    return e
 
             def _wrapper(_gen):
                 # Avoid issue: https://github.com/python/cpython/issues/112182
@@ -639,6 +641,8 @@ class ModelActor(xo.StatelessActor, CancelMixin):
                     return next(_gen)
                 except StopIteration:
                     return stop
+                except Exception as e:
+                    return e
 
             while True:
                 try:
@@ -699,6 +703,8 @@ class ModelActor(xo.StatelessActor, CancelMixin):
                             o = stream_out.get()
                             if o is stop:
                                 break
+                            elif isinstance(o, Exception):
+                                raise o
                             else:
                                 yield o
 
@@ -715,6 +721,8 @@ class ModelActor(xo.StatelessActor, CancelMixin):
                             o = await stream_out.get()
                             if o is stop:
                                 break
+                            elif isinstance(o, Exception):
+                                raise o
                             else:
                                 yield o
 
@@ -1229,19 +1237,51 @@ class ModelActor(xo.StatelessActor, CancelMixin):
         *args,
         **kwargs,
     ):
-        kwargs.pop("request_id", None)
-        if hasattr(self._model, "text_to_video"):
-            return await self._call_wrapper_json(
-                self._model.text_to_video,
-                prompt,
-                n,
-                *args,
-                **kwargs,
-            )
+        progressor = kwargs["progressor"] = await self._get_progressor(
+            kwargs.pop("request_id", None)
+        )
+        with progressor:
+            if hasattr(self._model, "text_to_video"):
+                return await self._call_wrapper_json(
+                    self._model.text_to_video,
+                    prompt,
+                    n,
+                    *args,
+                    **kwargs,
+                )
         raise AttributeError(
             f"Model {self._model.model_spec} is not for creating video."
         )
 
+    @request_limit
+    @log_async(logger=logger)
+    async def image_to_video(
+        self,
+        image: "PIL.Image",
+        prompt: str,
+        negative_prompt: Optional[str] = None,
+        n: int = 1,
+        *args,
+        **kwargs,
+    ):
+        kwargs["negative_prompt"] = negative_prompt
+        progressor = kwargs["progressor"] = await self._get_progressor(
+            kwargs.pop("request_id", None)
+        )
+        with progressor:
+            if hasattr(self._model, "image_to_video"):
+                return await self._call_wrapper_json(
+                    self._model.image_to_video,
+                    image,
+                    prompt,
+                    n,
+                    *args,
+                    **kwargs,
+                )
+        raise AttributeError(
+            f"Model {self._model.model_spec} is not for creating video from image."
+        )
+
     async def record_metrics(self, name, op, kwargs):
         worker_ref = await self._get_worker_ref()
         await worker_ref.record_metrics(name, op, kwargs)

xinference/core/supervisor.py

@@ -1102,8 +1102,8 @@ class SupervisorActor(xo.StatelessActor):
                 xavier_config=xavier_config,
                 **kwargs,
             )
-            await worker_ref.wait_for_load(_replica_model_uid)
             self._replica_model_uid_to_worker[_replica_model_uid] = worker_ref
+            await worker_ref.wait_for_load(_replica_model_uid)
             return subpool_address
 
         async def _launch_model():

xinference/core/utils.py

@@ -263,7 +263,7 @@ class CancelMixin:
     _CANCEL_TASK_NAME = "abort_block"
 
     def __init__(self):
-        self._running_tasks: weakref.WeakValueDictionary[
+        self._running_tasks: weakref.WeakValueDictionary[  # type: ignore
             str, asyncio.Task
         ] = weakref.WeakValueDictionary()
 

xinference/core/worker.py

@@ -148,7 +148,7 @@ class WorkerActor(xo.StatelessActor):
         elif metrics_exporter_host is not None or metrics_exporter_port is not None:
             # metrics export server.
             logger.info(
-                f"Starting metrics export server at {metrics_exporter_host}:{metrics_exporter_port}"
+                f"Starting metrics export server at {metrics_exporter_host}:{metrics_exporter_port}"  # noqa: E231
             )
             q: queue.Queue = queue.Queue()
             self._metrics_thread = threading.Thread(
@@ -162,7 +162,9 @@ class WorkerActor(xo.StatelessActor):
             while self._metrics_thread.is_alive():
                 try:
                     host, port = q.get(block=False)[:2]
-                    logger.info(f"Metrics server is started at: http://{host}:{port}")
+                    logger.info(
+                        f"Metrics server is started at: http://{host}:{port}"  # noqa: E231
+                    )
                     break
                 except queue.Empty:
                     pass
@@ -584,6 +586,7 @@ class WorkerActor(xo.StatelessActor):
         n_gpu: Optional[Union[int, str]] = "auto",
         gpu_idx: Optional[List[int]] = None,
         env: Optional[Dict[str, str]] = None,
+        start_python: Optional[str] = None,
     ) -> Tuple[str, List[str]]:
         env = {} if env is None else env
         devices = []
@@ -609,14 +612,8 @@ class WorkerActor(xo.StatelessActor):
             )
             env[env_name] = ",".join([str(dev) for dev in devices])
 
-        if os.name != "nt" and platform.system() != "Darwin":
-            # Linux
-            start_method = "forkserver"
-        else:
-            # Windows and macOS
-            start_method = "spawn"
         subpool_address = await self._main_pool.append_sub_pool(
-            env=env, start_method=start_method
+            env=env, start_python=start_python
         )
         return subpool_address, [str(dev) for dev in devices]
 
@@ -787,9 +784,9 @@ class WorkerActor(xo.StatelessActor):
         elif model_type == "image":
             return model.model_ability
         elif model_type == "audio":
-            return [model.model_ability]
+            return model.model_ability
         elif model_type == "video":
-            return ["text_to_video"]
+            return model.model_ability
         elif model_type == "flexible":
             return ["flexible"]
         else:
@@ -833,6 +830,8 @@ class WorkerActor(xo.StatelessActor):
         virtual_env_manager: VirtualEnvManager = get_virtual_env_manager(
             virtual_env_name or "uv", env_path
         )
+        # create env
+        virtual_env_manager.create_env()
         return virtual_env_manager
 
     @classmethod
@@ -845,9 +844,6 @@ class WorkerActor(xo.StatelessActor):
             # no settings or no packages
             return
 
-        # create env
-        virtual_env_manager.create_env()
-
         if settings.inherit_pip_config:
             # inherit pip config
             pip_config = get_pip_config_args()
@@ -1001,22 +997,26 @@ class WorkerActor(xo.StatelessActor):
             # virtualenv
             enable_virtual_env = kwargs.pop("enable_virtual_env", None)
             virtual_env_name = kwargs.pop("virtual_env_name", None)
-            virtual_env_path = os.path.join(XINFERENCE_VIRTUAL_ENV_DIR, model_name)
+            virtual_env_path = os.path.join(
+                XINFERENCE_VIRTUAL_ENV_DIR, "v2", model_name
+            )
             virtual_env_manager = await asyncio.to_thread(
                 self._create_virtual_env_manager,
                 enable_virtual_env,
                 virtual_env_name,
                 virtual_env_path,
             )
-            # setting os.environ if virtualenv created
-            env = (
-                {"PYTHONPATH": virtual_env_manager.get_lib_path()}
-                if virtual_env_manager
-                else None
+            subpool_python_path = (
+                None
+                if virtual_env_manager is None
+                else virtual_env_manager.get_python_path()
             )
-
             subpool_address, devices = await self._create_subpool(
-                model_uid, model_type, n_gpu=n_gpu, gpu_idx=gpu_idx, env=env
+                model_uid,
+                model_type,
+                n_gpu=n_gpu,
+                gpu_idx=gpu_idx,
+                start_python=subpool_python_path,
             )
             all_subpool_addresses = [subpool_address]
             try:
@@ -1116,7 +1116,7 @@ class WorkerActor(xo.StatelessActor):
                         coros.append(
                             self._main_pool.append_sub_pool(
                                 env={env_name: env_value},
-                                start_method=self._get_start_method(),
+                                start_python=subpool_python_path,
                             )
                         )
                     pool_addresses = await asyncio.gather(*coros)
@@ -1255,7 +1255,14 @@ class WorkerActor(xo.StatelessActor):
         try:
             logger.debug("Start to destroy model actor: %s", model_ref)
             coro = xo.destroy_actor(model_ref)
-            await asyncio.wait_for(coro, timeout=5)
+            # see https://github.com/xorbitsai/xoscar/pull/140
+            # asyncio.wait_for cannot work for Xoscar actor call,
+            # because when time out, the coroutine will be cancelled via raise CancelledEror,
+            # inside actor call, the error will be caught and
+            # a CancelMessage will be sent to dest actor pool,
+            # however the actor pool may be stuck already,
+            # thus the timeout will never be raised
+            await xo.wait_for(coro, timeout=5)
         except Exception as e:
             logger.debug(
                 "Destroy model actor failed, model uid: %s, error: %s", model_uid, e
@@ -1434,7 +1441,7 @@ class WorkerActor(xo.StatelessActor):
                 else:
                     logger.debug(f"{path} is not a valid path.")
             except Exception as e:
-                logger.error(f"Fail to delete {path} with error:{e}.")
+                logger.error(f"Fail to delete {path} with error:{e}.")  # noqa: E231
                 return False
         await self._cache_tracker_ref.confirm_and_remove_model(
             model_version, self.address
@@ -1467,26 +1474,13 @@ class WorkerActor(xo.StatelessActor):
         model_ref = self._model_uid_to_model[rep_model_uid]
         await model_ref.start_transfer_for_vllm(rank_addresses)
 
-    @staticmethod
-    def _get_start_method():
-        if os.name != "nt" and platform.system() != "Darwin":
-            # Linux
-            start_method = "forkserver"
-        else:
-            # Windows and macOS
-            start_method = "spawn"
-        return start_method
-
     @log_async(logger=logger, level=logging.INFO)
     async def launch_rank0_model(
         self, rep_model_uid: str, xavier_config: Dict[str, Any]
     ) -> Tuple[str, int]:
         from ..model.llm.vllm.xavier.collective_manager import Rank0ModelActor
 
-        start_method = self._get_start_method()
-        subpool_address = await self._main_pool.append_sub_pool(
-            start_method=start_method
-        )
+        subpool_address = await self._main_pool.append_sub_pool()
 
         store_address = subpool_address.split(":")[0]
         # Note that `store_port` needs to be generated on the worker,

xinference/deploy/cmdline.py

@@ -805,6 +805,14 @@ def remove_cache(
     type=(str, str),
     multiple=True,
 )
+@click.option(
+    "--quantization-config",
+    "-qc",
+    "quantization_config",
+    type=(str, str),
+    multiple=True,
+    help="bnb quantization config for `transformers` engine.",
+)
 @click.option(
     "--worker-ip",
     default=None,
@@ -853,6 +861,7 @@ def model_launch(
     trust_remote_code: bool,
     api_key: Optional[str],
     model_path: Optional[str],
+    quantization_config: Optional[Tuple],
 ):
     kwargs = {}
     for i in range(0, len(ctx.args), 2):
@@ -884,6 +893,12 @@ def model_launch(
     else:
         _n_gpu = int(n_gpu)
 
+    bnb_quantization_config = (
+        {k: handle_click_args_type(v) for k, v in dict(quantization_config).items()}
+        if quantization_config
+        else None
+    )
+
     image_lora_load_params = (
         {k: handle_click_args_type(v) for k, v in dict(image_lora_load_kwargs).items()}
         if image_lora_load_kwargs
@@ -929,6 +944,8 @@ def model_launch(
 
     # do not wait for launching.
     kwargs["wait_ready"] = False
+    if bnb_quantization_config:
+        kwargs["quantization_config"] = {**bnb_quantization_config}
 
     model_uid = client.launch_model(
         model_name=model_name,

xinference/deploy/utils.py

@@ -141,13 +141,10 @@ def get_config_dict(
 async def create_worker_actor_pool(
     address: str, logging_conf: Optional[dict] = None
 ) -> "MainActorPoolType":
-    subprocess_start_method = "forkserver" if os.name != "nt" else "spawn"
-
     return await xo.create_actor_pool(
         address=address,
         n_process=0,
         auto_recover="process",
-        subprocess_start_method=subprocess_start_method,
         logging_conf={"dict": logging_conf},
     )
 

xinference/model/audio/__init__.py

@@ -66,31 +66,8 @@ def _need_filter(spec: dict):
 
 
 def _install():
-    _model_spec_json = os.path.join(os.path.dirname(__file__), "model_spec.json")
-    _model_spec_modelscope_json = os.path.join(
-        os.path.dirname(__file__), "model_spec_modelscope.json"
-    )
-    BUILTIN_AUDIO_MODELS.update(
-        dict(
-            (spec["model_name"], AudioModelFamilyV1(**spec))
-            for spec in json.load(codecs.open(_model_spec_json, "r", encoding="utf-8"))
-            if not _need_filter(spec)
-        )
-    )
-    for model_name, model_spec in BUILTIN_AUDIO_MODELS.items():
-        MODEL_NAME_TO_REVISION[model_name].append(model_spec.model_revision)
-
-    MODELSCOPE_AUDIO_MODELS.update(
-        dict(
-            (spec["model_name"], AudioModelFamilyV1(**spec))
-            for spec in json.load(
-                codecs.open(_model_spec_modelscope_json, "r", encoding="utf-8")
-            )
-            if not _need_filter(spec)
-        )
-    )
-    for model_name, model_spec in MODELSCOPE_AUDIO_MODELS.items():
-        MODEL_NAME_TO_REVISION[model_name].append(model_spec.model_revision)
+    load_model_family_from_json("model_spec.json", BUILTIN_AUDIO_MODELS)
+    load_model_family_from_json("model_spec_modelscope.json", MODELSCOPE_AUDIO_MODELS)
 
     # register model description after recording model revision
     for model_spec_info in [BUILTIN_AUDIO_MODELS, MODELSCOPE_AUDIO_MODELS]:
@@ -104,5 +81,17 @@ def _install():
     for ud_audio in get_user_defined_audios():
         AUDIO_MODEL_DESCRIPTIONS.update(generate_audio_description(ud_audio))
 
-    del _model_spec_json
-    del _model_spec_modelscope_json
+
+def load_model_family_from_json(json_filename, target_families):
+    json_path = os.path.join(os.path.dirname(__file__), json_filename)
+    target_families.update(
+        dict(
+            (spec["model_name"], AudioModelFamilyV1(**spec))
+            for spec in json.load(codecs.open(json_path, "r", encoding="utf-8"))
+            if not _need_filter(spec)
+        )
+    )
+    for model_name, model_spec in target_families.items():
+        MODEL_NAME_TO_REVISION[model_name].append(model_spec.model_revision)
+
+    del json_path

xinference/model/audio/core.py

@@ -52,7 +52,7 @@ class AudioModelFamilyV1(CacheableModelSpec):
     model_revision: Optional[str]
     multilingual: bool
     language: Optional[str]
-    model_ability: Optional[str]
+    model_ability: Optional[List[str]]
     default_model_config: Optional[Dict[str, Any]]
     default_transcription_config: Optional[Dict[str, Any]]
     engine: Optional[str]
@@ -82,6 +82,7 @@ class AudioModelDescription(ModelDescription):
             "model_name": self._model_spec.model_name,
             "model_family": self._model_spec.model_family,
             "model_revision": self._model_spec.model_revision,
+            "model_ability": self._model_spec.model_ability,
         }
 
     def to_version_info(self):

xinference/model/audio/cosyvoice.py

@@ -55,10 +55,12 @@ class CosyVoiceModel:
         thirdparty_dir = os.path.join(os.path.dirname(__file__), "../../thirdparty")
         sys.path.insert(0, thirdparty_dir)
 
+        kwargs = {}
         if "CosyVoice2" in self._model_spec.model_name:
             from cosyvoice.cli.cosyvoice import CosyVoice2 as CosyVoice
 
             self._is_cosyvoice2 = True
+            kwargs = {"use_flow_cache": self._kwargs.get("use_flow_cache", False)}
         else:
             from cosyvoice.cli.cosyvoice import CosyVoice
 
@@ -69,7 +71,7 @@ class CosyVoiceModel:
             "compile", False
         )
         logger.info("Loading CosyVoice model, compile=%s...", load_jit)
-        self._model = CosyVoice(self._model_path, load_jit=load_jit)
+        self._model = CosyVoice(self._model_path, load_jit=load_jit, **kwargs)
         if self._is_cosyvoice2:
             spk2info_file = os.path.join(thirdparty_dir, "cosyvoice/bin/spk2info.pt")
             self._model.frontend.spk2info = torch.load(
@@ -112,7 +114,7 @@ class CosyVoiceModel:
                     input, prompt_speech_16k, stream=stream
                 )
         else:
-            available_speakers = self._model.list_avaliable_spks()
+            available_speakers = self._model.list_available_spks()
             if not voice:
                 voice = available_speakers[0]
                 logger.info("Auto select speaker: %s", voice)