xinference 1.6.0.post1__py3-none-any.whl → 1.6.1__py3-none-any.whl

xinference/model/llm/transformers/minicpmv25.py

@@ -1,193 +0,0 @@
-# Copyright 2022-2023 XProbe Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import json
-import logging
-import uuid
-from concurrent.futures import ThreadPoolExecutor
-from typing import Dict, Iterator, List, Optional, Union
-
-import torch
-
-from ....types import ChatCompletion, ChatCompletionChunk, CompletionChunk
-from ...utils import select_device
-from ..llm_family import LLMFamilyV1, LLMSpecV1
-from ..utils import (
-    _decode_image,
-    generate_chat_completion,
-    generate_completion_chunk,
-    parse_messages,
-)
-from .core import PytorchChatModel, PytorchGenerateConfig
-from .utils import cache_clean
-
-logger = logging.getLogger(__name__)
-
-
-class MiniCPMV25Model(PytorchChatModel):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self._device = None
-        self._tokenizer = None
-        self._model = None
-
-    @classmethod
-    def match_json(
-        cls, model_family: "LLMFamilyV1", model_spec: "LLMSpecV1", quantization: str
-    ) -> bool:
-        family = model_family.model_family or model_family.model_name
-        if "MiniCPM-Llama3-V-2_5".lower() in family.lower():
-            return True
-        return False
-
-    def _get_model_class(self):
-        from transformers import AutoModel
-
-        return AutoModel
-
-    def load(self):
-        from transformers import AutoModel, AutoTokenizer
-        from transformers.generation import GenerationConfig
-
-        device = self._pytorch_model_config.get("device", "auto")
-        self._device = select_device(device)
-        self._device = "auto" if self._device == "cuda" else self._device
-
-        if "int4" in self.model_path and device == "mps":
-            logger.error(
-                "Error: running int4 model with bitsandbytes on Mac is not supported right now."
-            )
-            exit()
-
-        if self._check_tensorizer_integrity():
-            self._model, self._tokenizer = self._load_tensorizer()
-            return
-
-        if "int4" in self.model_path:
-            model = AutoModel.from_pretrained(self.model_path, trust_remote_code=True)
-        else:
-            kwargs = self.apply_bnb_quantization()
-            model = AutoModel.from_pretrained(
-                self.model_path,
-                trust_remote_code=True,
-                torch_dtype=torch.float16,
-                device_map=self._device,
-                **kwargs
-            )
-        tokenizer = AutoTokenizer.from_pretrained(
-            self.model_path, trust_remote_code=True
-        )
-        self._model = model.eval()
-        self._tokenizer = tokenizer
-
-        # Specify hyperparameters for generation
-        self._model.generation_config = GenerationConfig.from_pretrained(
-            self.model_path,
-            trust_remote_code=True,
-        )
-        self._save_tensorizer()
-
-    def _message_content_to_chat(self, content):
-        if not isinstance(content, str):
-            texts = []
-            image_urls = []
-            for c in content:
-                c_type = c.get("type")
-                if c_type == "text":
-                    texts.append(c["text"])
-                elif c_type == "image_url":
-                    image_urls.append(c["image_url"]["url"])
-            image_futures = []
-            with ThreadPoolExecutor() as executor:
-                for image_url in image_urls:
-                    fut = executor.submit(_decode_image, image_url)
-                    image_futures.append(fut)
-            images = [fut.result() for fut in image_futures]
-            text = " ".join(texts)
-            if len(images) == 0:
-                return text, []
-            elif len(images) == 1:
-                return text, images
-            else:
-                raise RuntimeError("Only one image per message is supported")
-        return content, []
-
-    @cache_clean
-    def chat(
-        self,
-        messages: List[Dict],
-        generate_config: Optional[PytorchGenerateConfig] = None,
-    ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
-        stream = generate_config.get("stream", False) if generate_config else False
-        prompt, _, chat_history = parse_messages(messages)
-        content, images_chat = self._message_content_to_chat(prompt)
-
-        msgs = []
-        query_to_response: List[Dict] = []
-        images_history = []
-        for h in chat_history or []:
-            role = h["role"]
-            content_h, images_tmp = self._message_content_to_chat(h["content"])
-            if images_tmp != []:
-                images_history = images_tmp
-            if len(query_to_response) == 0 and role == "user":
-                query_to_response.append({"role": "user", "content": content_h})
-            if len(query_to_response) == 1 and role == "assistant":
-                query_to_response.append({"role": "assistant", "content": content_h})
-            if len(query_to_response) == 2:
-                msgs.extend(query_to_response)
-                query_to_response = []
-        image = None
-        if len(images_chat) > 0:
-            image = images_chat[0]
-        elif len(images_history) > 0:
-            image = images_history[0]
-        msgs.append({"role": "user", "content": content})
-
-        chat = self._model.chat(
-            image=image,
-            msgs=json.dumps(msgs, ensure_ascii=True),
-            tokenizer=self._tokenizer,
-            sampling=True,
-            **generate_config
-        )
-        if stream:
-            it = self.chat_stream(chat)
-            return self._to_chat_completion_chunks(it)
-        else:
-            return generate_chat_completion(self.model_uid, chat)
-
-    def chat_stream(self, chat) -> Iterator[CompletionChunk]:
-        completion_id = str(uuid.uuid1())
-        for new_text in chat:
-            yield generate_completion_chunk(
-                chunk_text=new_text,
-                finish_reason=None,
-                chunk_id=completion_id,
-                model_uid=self.model_uid,
-                prompt_tokens=-1,
-                completion_tokens=-1,
-                total_tokens=-1,
-            )
-
-        yield generate_completion_chunk(
-            chunk_text=None,
-            finish_reason="stop",
-            chunk_id=completion_id,
-            model_uid=self.model_uid,
-            prompt_tokens=-1,
-            completion_tokens=-1,
-            total_tokens=-1,
-            has_choice=True,
-            has_content=False,
-        )

xinference/model/llm/transformers/omnilmm.py

@@ -1,132 +0,0 @@
-# Copyright 2022-2023 XProbe Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import base64
-import json
-import logging
-import operator
-import tempfile
-from typing import Dict, Iterator, List, Optional, Tuple, Union
-
-from ....thirdparty.omnilmm.chat import OmniLMMChat, img2base64
-from ....types import ChatCompletion, ChatCompletionChunk
-from ...utils import select_device
-from ..llm_family import LLMFamilyV1, LLMSpecV1
-from ..utils import generate_chat_completion, parse_messages
-from .core import PytorchChatModel, PytorchGenerateConfig
-from .utils import cache_clean
-
-logger = logging.getLogger(__name__)
-
-
-class OmniLMMModel(PytorchChatModel):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self._model = None
-
-    @classmethod
-    def match_json(
-        cls, model_family: "LLMFamilyV1", model_spec: "LLMSpecV1", quantization: str
-    ) -> bool:
-        llm_family = model_family.model_family or model_family.model_name
-        if "OmniLMM" in llm_family:
-            return True
-        return False
-
-    def load(self):
-        device = self._pytorch_model_config.get("device", "auto")
-        device = select_device(device)
-        self._model = OmniLMMChat(self.model_path, device_map=device)
-
-    def _message_content_to_OmniLMM(
-        self, content
-    ) -> Tuple[List[Dict[str, str]], List[Dict[str, str]]]:
-        def _ensure_url(_url):
-            if _url.startswith("data:"):
-                logging.info("Parse url by base64 decoder.")
-                # https://platform.openai.com/docs/guides/vision/uploading-base-64-encoded-images
-                # e.g. f"data:image/jpeg;base64,{base64_image}"
-                _type, data = _url.split(";")
-                _, ext = _type.split("/")
-                data = data[len("base64,") :]
-                data = base64.b64decode(data.encode("utf-8"))
-
-                with tempfile.NamedTemporaryFile(suffix=f".{ext}", delete=False) as f:
-                    f.write(data)
-                logging.info("Dump base64 data to %s", f.name)
-                return f.name
-            else:
-                if len(_url) > 2048:
-                    raise Exception(f"Image url is too long, {len(_url)} > 2048.")
-                return _url
-
-        if not isinstance(content, str):
-            images = []
-            other_content = []
-
-            for c in content:
-                if c.get("type") == "image_url":
-                    images.append(
-                        {"image": _ensure_url(c["image_url"]["url"]), "type": "image"}
-                    )
-                else:
-                    other_content.append(c)
-
-            images = sorted(images, key=operator.itemgetter("type"))
-            other_content = sorted(other_content, key=operator.itemgetter("type"))
-
-            return images, other_content
-        return [], [{"type": "text", "text": content}]
-
-    @cache_clean
-    def chat(
-        self,
-        messages: List[Dict],
-        generate_config: Optional[PytorchGenerateConfig] = None,
-    ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
-        if generate_config and generate_config.get("stream"):
-            raise Exception(
-                f"Chat with model {self.model_family.model_name} does not support stream."
-            )
-        prompt, _, chat_history = parse_messages(messages)
-        image_first, prompt = self._message_content_to_OmniLMM(prompt)
-
-        msgs = []
-        query_to_response: List[Dict] = []
-        image_another = []
-        for h in chat_history or []:
-            role = h["role"]
-            image_tmp, content = self._message_content_to_OmniLMM(h["content"])
-            if image_tmp != []:
-                image_another = image_tmp
-            if len(query_to_response) == 0 and role == "user":
-                query_to_response.append(
-                    {"role": "user", "content": content[0]["text"]}
-                )
-            if len(query_to_response) == 1 and role == "assistant":
-                query_to_response.append(
-                    {"role": "assistant", "content": content[0]["text"]}
-                )
-            if len(query_to_response) == 2:
-                msgs.extend(query_to_response)
-                query_to_response = []
-        if image_first != []:
-            image = image_first
-        if image_another != []:
-            image = image_another
-        im_64 = img2base64(image[0]["image"])
-        msgs.append({"role": "user", "content": prompt[0]["text"]})
-        input = {"image": im_64, "question": json.dumps(msgs, ensure_ascii=True)}
-        answer = self._model.chat(input=input)
-
-        return generate_chat_completion(self.model_uid, answer)

xinference/model/llm/transformers/qwen2_audio.py

@@ -1,179 +0,0 @@
-# Copyright 2022-2023 XProbe Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import logging
-import uuid
-from io import BytesIO
-from typing import Iterator, List, Optional, Union
-from urllib.request import urlopen
-
-import numpy as np
-
-from ....model.utils import select_device
-from ....types import (
-    ChatCompletion,
-    ChatCompletionChunk,
-    ChatCompletionMessage,
-    CompletionChunk,
-)
-from ..llm_family import LLMFamilyV1, LLMSpecV1
-from ..utils import generate_chat_completion, generate_completion_chunk
-from .core import PytorchChatModel, PytorchGenerateConfig
-from .utils import cache_clean
-
-logger = logging.getLogger(__name__)
-
-
-class Qwen2AudioChatModel(PytorchChatModel):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self._processor = None
-        self._model = None
-        self._device = None
-
-    @classmethod
-    def match_json(
-        cls, model_family: "LLMFamilyV1", model_spec: "LLMSpecV1", quantization: str
-    ) -> bool:
-        llm_family = model_family.model_family or model_family.model_name
-        if "qwen2-audio".lower() in llm_family.lower():
-            return True
-        return False
-
-    def load(self):
-        from transformers import AutoProcessor, Qwen2AudioForConditionalGeneration
-
-        device = self._pytorch_model_config.get("device", "auto")
-        device = select_device(device)
-        # for multiple GPU, set back to auto to make multiple devices work
-        device = "auto" if device == "cuda" else device
-        self._device = device
-        kwargs = self.apply_bnb_quantization()
-
-        self._processor = AutoProcessor.from_pretrained(
-            self.model_path,
-            device_map=device,
-            # trust_remote_code=True,
-            code_revision=self.model_spec.model_revision,
-        )
-        self._model = Qwen2AudioForConditionalGeneration.from_pretrained(
-            self.model_path,
-            device_map=device,
-            # trust_remote_code=True,
-            revision=self.model_spec.model_revision,
-            **kwargs,
-        )
-
-    def _transform_messages(
-        self,
-        messages: Union[List[ChatCompletionMessage], List[dict]],
-    ):
-        import librosa
-
-        text = self._processor.apply_chat_template(
-            messages, add_generation_prompt=True, tokenize=False
-        )
-        audios: List[np.ndarray] = []
-        for msg in messages:
-            content = msg["content"]
-            if isinstance(content, List):
-                for item in content:  # type: ignore
-                    if item.get("type") == "audio" and "audio_url" in item:
-                        audio = librosa.load(
-                            BytesIO(urlopen(item["audio_url"]).read()),
-                            sr=self._processor.feature_extractor.sampling_rate,
-                        )[0]
-                        audios.append(audio)
-
-        return text, audios
-
-    @cache_clean
-    def chat(
-        self,
-        messages: List[ChatCompletionMessage],
-        generate_config: Optional[PytorchGenerateConfig] = None,
-    ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
-        text, audios = self._transform_messages(messages)
-        inputs = self._processor(
-            text=text, audios=audios, return_tensors="pt", padding=True
-        )
-        # Make sure that the inputs and the model are on the same device.
-        inputs.data = {k: v.to(self._device) for k, v in inputs.data.items()}
-        inputs.input_ids = inputs.input_ids.to(self._device)
-        generate_config = generate_config if generate_config else {}
-        stream = generate_config.get("stream", False) if generate_config else False
-
-        if stream:
-            it = self._generate_stream(inputs, generate_config)
-            return self._to_chat_completion_chunks(it)
-        else:
-            c = self._generate(inputs, generate_config)
-            return c
-
-    def _generate(self, inputs, config: PytorchGenerateConfig = {}) -> ChatCompletion:
-        generate_ids = self._model.generate(
-            **inputs,
-            max_length=config.get("max_tokens", 512),
-        )
-        generate_ids = generate_ids[:, inputs.input_ids.size(1) :]
-        response = self._processor.batch_decode(
-            generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False
-        )[0]
-        return generate_chat_completion(self.model_uid, response)
-
-    def _generate_stream(
-        self, inputs, config: PytorchGenerateConfig = {}
-    ) -> Iterator[CompletionChunk]:
-        from threading import Thread
-
-        from transformers import TextIteratorStreamer
-
-        tokenizer = self._processor.tokenizer
-        streamer = TextIteratorStreamer(
-            tokenizer, timeout=60.0, skip_prompt=True, skip_special_tokens=True
-        )
-
-        gen_kwargs = {
-            "max_new_tokens": config.get("max_tokens", 512),
-            "streamer": streamer,
-            **inputs,
-        }
-
-        thread = Thread(target=self._model.generate, kwargs=gen_kwargs)
-        thread.start()
-
-        completion_id = str(uuid.uuid1())
-        for new_text in streamer:
-            yield generate_completion_chunk(
-                chunk_text=new_text,
-                finish_reason=None,
-                chunk_id=completion_id,
-                model_uid=self.model_uid,
-                prompt_tokens=-1,
-                completion_tokens=-1,
-                total_tokens=-1,
-                has_choice=True,
-                has_content=True,
-            )
-
-        yield generate_completion_chunk(
-            chunk_text=None,
-            finish_reason="stop",
-            chunk_id=completion_id,
-            model_uid=self.model_uid,
-            prompt_tokens=-1,
-            completion_tokens=-1,
-            total_tokens=-1,
-            has_choice=True,
-            has_content=False,
-        )