xinference 1.6.0__py3-none-any.whl → 1.6.1__py3-none-any.whl

xinference/model/llm/transformers/cogvlm2_video.py

@@ -1,333 +0,0 @@
-# Copyright 2022-2023 XProbe Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import logging
-import uuid
-from concurrent.futures import ThreadPoolExecutor
-from typing import Dict, Iterator, List, Optional, Tuple, Union
-
-import torch
-
-from ....model.utils import select_device
-from ....types import ChatCompletion, ChatCompletionChunk, CompletionChunk
-from ..llm_family import LLMFamilyV1, LLMSpecV1
-from ..utils import (
-    _decode_image,
-    generate_chat_completion,
-    generate_completion_chunk,
-    parse_messages,
-)
-from .core import PytorchChatModel, PytorchGenerateConfig
-from .utils import cache_clean
-
-logger = logging.getLogger(__name__)
-
-
-LANGUAGE_TOKEN_TYPE = 0
-VISION_TOKEN_TYPE = 1
-
-
-def recur_move_to(item, tgt, criterion_func):
-    """
-    This function is copied from https://github.com/THUDM/CogVLM2/blob/main/basic_demo/cli_demo_batch_inference.py
-    """
-    if criterion_func(item):
-        device_copy = item.to(tgt)
-        return device_copy
-    elif isinstance(item, list):
-        return [recur_move_to(v, tgt, criterion_func) for v in item]
-    elif isinstance(item, tuple):
-        return tuple([recur_move_to(v, tgt, criterion_func) for v in item])
-    elif isinstance(item, dict):
-        return {k: recur_move_to(v, tgt, criterion_func) for k, v in item.items()}
-    else:
-        return item
-
-
-class CogVLM2VideoModel(PytorchChatModel):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self._torch_type = None
-        self._device = None
-        self._tokenizer = None
-        self._model = None
-
-    @classmethod
-    def match_json(
-        cls, model_family: "LLMFamilyV1", model_spec: "LLMSpecV1", quantization: str
-    ) -> bool:
-        family = model_family.model_family or model_family.model_name
-        if "cogvlm2" in family.lower() and "video" in family.lower():
-            return True
-        return False
-
-    def load(self):
-        from transformers import AutoModelForCausalLM, AutoTokenizer
-        from transformers.generation import GenerationConfig
-
-        device = self._pytorch_model_config.get("device", "auto")
-        self._device = select_device(device)
-        self._torch_type = (
-            torch.bfloat16
-            if torch.cuda.is_available() and torch.cuda.get_device_capability()[0] >= 8
-            else torch.float16
-        )
-
-        if self._check_tensorizer_integrity():
-            self._model, self._tokenizer = self._load_tensorizer()
-            return
-
-        kwargs = self.apply_bnb_quantization()
-
-        self._tokenizer = AutoTokenizer.from_pretrained(
-            self.model_path,
-            trust_remote_code=True,
-        )
-
-        self._model = AutoModelForCausalLM.from_pretrained(
-            self.model_path,
-            torch_dtype=self._torch_type,
-            trust_remote_code=True,
-            low_cpu_mem_usage=True,
-            device_map="auto",
-            **kwargs
-        ).eval()
-
-        # Specify hyperparameters for generation
-        self._model.generation_config = GenerationConfig.from_pretrained(
-            self.model_path,
-            trust_remote_code=True,
-        )
-        self._save_tensorizer()
-
-    def _load_video(self, video_path):
-        import numpy as np
-        from decord import VideoReader, bridge, cpu
-
-        bridge.set_bridge("torch")
-        num_frames = 24
-
-        decord_vr = VideoReader(video_path, ctx=cpu(0))
-        frame_id_list = None
-        total_frames = len(decord_vr)
-        timestamps = decord_vr.get_frame_timestamp(np.arange(total_frames))
-        timestamps = [i[0] for i in timestamps]
-        max_second = round(max(timestamps)) + 1
-        frame_id_list = []
-        for second in range(max_second):
-            closest_num = min(timestamps, key=lambda x: abs(x - second))
-            index = timestamps.index(closest_num)
-            frame_id_list.append(index)
-            if len(frame_id_list) >= num_frames:
-                break
-        video_data = decord_vr.get_batch(frame_id_list)
-        video_data = video_data.permute(3, 0, 1, 2)
-        return video_data
-
-    def _message_content_to_cogvlm2(self, content):
-        if not isinstance(content, str):
-            texts = []
-            image_urls = []
-            video_urls = []
-            for c in content:
-                c_type = c.get("type")
-                if c_type == "text":
-                    texts.append(c["text"])
-                elif c_type == "image_url":
-                    image_urls.append(c["image_url"]["url"])
-                elif c_type == "video_url":
-                    video_urls.append(c["video_url"]["url"])
-            if len(video_urls) > 1:
-                raise RuntimeError("Only one video per message is supported")
-            image_futures = []
-            video = None
-            with ThreadPoolExecutor() as executor:
-                for image_url in image_urls:
-                    fut = executor.submit(_decode_image, image_url)
-                    image_futures.append(fut)
-            images = [fut.result() for fut in image_futures]
-            for v in video_urls:
-                video = self._load_video(v)
-            text = " ".join(texts)
-            return text, images, video
-        return content, [], None
-
-    def _history_content_to_cogvlm2(self, system_prompt: str, chat_history: List[Dict]):
-        query = system_prompt
-        history: List[Tuple] = []
-        pixel_values = None
-        video_urls: List[str] = []
-        for i in range(0, len(chat_history), 2):
-            user = chat_history[i]["content"]
-            if isinstance(user, List):
-                for content in user:
-                    c_type = content.get("type")
-                    if c_type == "text":
-                        user = content["text"]
-                    elif c_type == "image_url" and not pixel_values:
-                        pixel_values = _decode_image(content["image_url"]["url"])
-                    elif c_type == "video_url":
-                        video_urls.append(content["video_url"]["url"])
-            assistant = chat_history[i + 1]["content"]
-            history.append((user, assistant))
-            query = assistant  # type: ignore
-        if len(video_urls) > 1:
-            raise RuntimeError("Only one video per message is supported")
-        video = None
-        for v in video_urls:
-            video = self._load_video(v)
-        return query, history, [pixel_values], video
-
-    def get_query_and_history(
-        self,
-        prompt: Union[str, List[Dict]],
-        system_prompt: Optional[str] = None,
-        chat_history: Optional[List[Dict]] = None,
-    ):
-        content, image, video = self._message_content_to_cogvlm2(prompt)
-
-        history = []
-        history_image = None
-        history_video = None
-        if chat_history:
-            (
-                query,
-                history,
-                history_image,
-                history_video,
-            ) = self._history_content_to_cogvlm2(
-                system_prompt, chat_history  # type: ignore
-            )
-
-        if image and history_image:
-            history = []
-            query = content
-        else:
-            image = image if image else history_image
-            query = content
-
-        if video is not None and history_video is not None:
-            history = []
-            query = content
-        else:
-            video = video if video is not None else history_video
-            query = content
-
-        return query, image, video, history
-
-    @cache_clean
-    def chat(
-        self,
-        messages: List[Dict],
-        generate_config: Optional[PytorchGenerateConfig] = None,
-    ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
-        system_prompt = ""
-        if messages[0]["role"] == "system":
-            system_prompt = messages[0]["content"]
-        stream = generate_config.get("stream", False) if generate_config else False
-
-        sanitized_config = {
-            "pad_token_id": 128002,
-            "max_new_tokens": generate_config.get("max_tokens", 512)
-            if generate_config
-            else 512,
-        }
-
-        prompt, _, chat_history = parse_messages(messages)
-        query, image, video, history = self.get_query_and_history(
-            prompt, system_prompt=system_prompt, chat_history=chat_history
-        )
-
-        if video is not None:
-            image = [video]
-
-        input_by_model = self._model.build_conversation_input_ids(
-            self._tokenizer,
-            query=query,
-            history=history,
-            images=image,
-            template_version="chat",
-        )
-
-        inputs = {
-            "input_ids": input_by_model["input_ids"].unsqueeze(0).to(self._device),
-            "token_type_ids": input_by_model["token_type_ids"]
-            .unsqueeze(0)
-            .to(self._device),
-            "attention_mask": input_by_model["attention_mask"]
-            .unsqueeze(0)
-            .to(self._device),
-            "images": [
-                [input_by_model["images"][0].to(self._device).to(self._torch_type)]
-            ]
-            if image is not None
-            else None,
-        }
-
-        if stream:
-            it = self._streaming_chat_response(inputs, sanitized_config)
-            return self._to_chat_completion_chunks(it)
-        else:
-            with torch.no_grad():
-                outputs = self._model.generate(**inputs, **sanitized_config)
-                outputs = outputs[:, inputs["input_ids"].shape[1] :]
-                response = self._tokenizer.decode(outputs[0])
-                response = response.split("<|end_of_text|>")[0]
-
-            return generate_chat_completion(self.model_uid, response)
-
-    def _streaming_chat_response(
-        self, inputs: Dict, config: Dict
-    ) -> Iterator[CompletionChunk]:
-        from threading import Thread
-
-        from transformers import TextIteratorStreamer
-
-        streamer = TextIteratorStreamer(
-            self._tokenizer, skip_prompt=True, skip_special_tokens=True
-        )
-        generation_kwargs = {
-            "input_ids": inputs["input_ids"],
-            "attention_mask": inputs["attention_mask"],
-            "token_type_ids": inputs["token_type_ids"],
-            "images": inputs["images"],
-            "max_new_tokens": config["max_new_tokens"],
-            "pad_token_id": config["pad_token_id"],
-            "streamer": streamer,
-        }
-
-        thread = Thread(target=self._model.generate, kwargs=generation_kwargs)
-        thread.start()
-
-        completion_id = str(uuid.uuid1())
-        for new_text in streamer:
-            yield generate_completion_chunk(
-                chunk_text=new_text,
-                finish_reason=None,
-                chunk_id=completion_id,
-                model_uid=self.model_uid,
-                prompt_tokens=-1,
-                completion_tokens=-1,
-                total_tokens=-1,
-            )
-        yield generate_completion_chunk(
-            chunk_text=None,
-            finish_reason="stop",
-            chunk_id=completion_id,
-            model_uid=self.model_uid,
-            prompt_tokens=-1,
-            completion_tokens=-1,
-            total_tokens=-1,
-            has_choice=True,
-            has_content=False,
-        )

xinference/model/llm/transformers/deepseek_vl.py

@@ -1,280 +0,0 @@
-# Copyright 2022-2023 XProbe Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import base64
-import logging
-import os.path
-import tempfile
-import uuid
-from concurrent.futures import ThreadPoolExecutor
-from io import BytesIO
-from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
-
-import requests
-import torch
-
-from ....model.utils import select_device
-from ....types import ChatCompletion, ChatCompletionChunk, CompletionChunk
-from ..llm_family import LLMFamilyV1, LLMSpecV1
-from ..utils import generate_chat_completion, generate_completion_chunk
-from .core import PytorchChatModel, PytorchGenerateConfig
-from .utils import cache_clean
-
-logger = logging.getLogger(__name__)
-
-
-class DeepSeekVLChatModel(PytorchChatModel):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self._tokenizer = None
-        self._model = None
-        self._vl_chat_processor = None
-        self._type = None
-
-    @classmethod
-    def match_json(
-        cls, model_family: "LLMFamilyV1", model_spec: "LLMSpecV1", quantization: str
-    ) -> bool:
-        llm_family = model_family.model_family or model_family.model_name
-        if "deepseek-vl-chat" == llm_family.lower():
-            return True
-        return False
-
-    def load(self):
-        from transformers import AutoModelForCausalLM
-
-        from ....thirdparty.deepseek_vl.models import (
-            MultiModalityCausalLM,
-            VLChatProcessor,
-        )
-
-        self._device = self._pytorch_model_config.get("device", "auto")
-        self._device = select_device(self._device)
-        self._type = torch.float16 if self._device == "mps" else torch.bfloat16
-
-        kwargs = self.apply_bnb_quantization()
-
-        # specify the path to the model
-        self._vl_chat_processor: VLChatProcessor = VLChatProcessor.from_pretrained(  # type: ignore
-            self.model_path
-        )
-        self._tokenizer = self._vl_chat_processor.tokenizer
-
-        vl_gpt: MultiModalityCausalLM = AutoModelForCausalLM.from_pretrained(  # type: ignore
-            self.model_path,
-            trust_remote_code=True,
-            device_map=self._device,
-            torch_dtype=self._type,
-            **kwargs,
-        )
-        self._model = vl_gpt.eval()
-
-    @staticmethod
-    def _message_content_to_deepseek(content) -> Tuple[str, List[str]]:
-        def _ensure_url(_url):
-            if _url.startswith("data:"):
-                logging.info("Parse url by base64 decoder.")
-                # https://platform.openai.com/docs/guides/vision/uploading-base-64-encoded-images
-                # e.g. f"data:image/jpeg;base64,{base64_image}"
-                _type, data = _url.split(";")
-                _, ext = _type.split("/")
-                data = data[len("base64,") :]
-                data = base64.b64decode(data.encode("utf-8"))
-
-                with tempfile.NamedTemporaryFile(suffix=f".{ext}", delete=False) as f:
-                    f.write(data)
-                logging.info("Dump base64 data to %s", f.name)
-                return f.name
-            else:
-                if len(_url) > 2048:
-                    raise Exception(f"Image url is too long, {len(_url)} > 2048.")
-
-                return _url
-
-        def _download(_images):
-            local_images = []
-
-            # To make requests.get works
-            headers = {
-                "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36"
-            }
-            with ThreadPoolExecutor() as executor:
-                for url in images:
-                    try:
-                        if os.path.exists(url):
-                            local_images.append(url)
-                            continue
-                    except Exception as e:
-                        logger.debug("Image is remote: %s, e: %s", url, e)
-                        pass
-                    # Append a placeholder
-                    local_images.append(None)
-
-                    def _fill_placeholder(_url, _index):
-                        response = requests.get(url, headers=headers)
-                        local_images[_index] = BytesIO(response.content)
-
-                    executor.submit(_fill_placeholder, url, len(local_images) - 1)
-            return local_images
-
-        if not isinstance(content, str):
-            # TODO(codingl2k1): Optimize _ensure_url
-
-            images = []
-            new_content = []
-            for c in content:
-                c_type = c.get("type")
-                if c_type == "image_url":
-                    images.append(_ensure_url(c["image_url"]["url"]))
-                elif c_type == "text":
-                    new_content.append(c["text"])
-            if images:
-                new_content.insert(0, "<image_placeholder>")
-                images = _download(images)
-            return "".join(new_content), images
-        return content, []
-
-    @cache_clean
-    def chat(
-        self,
-        messages: List[Dict],
-        generate_config: Optional[PytorchGenerateConfig] = None,
-    ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
-        if not generate_config:
-            generate_config = {}
-
-        stream = generate_config.get("stream", False)
-        stream_options = generate_config.pop("stream_options", None)
-        include_usage = (
-            stream_options["include_usage"]
-            if isinstance(stream_options, dict)
-            else False
-        )
-
-        prompt = ""
-        deepseek_messages = []
-        for i, message in enumerate(messages):
-            role = message["role"]
-            content = message["content"]
-            if role == "user":
-                if isinstance(content, str):
-                    deepseek_messages.append({"role": "User", "content": content})
-                else:
-                    content, images = self._message_content_to_deepseek(content)
-                    msg: Dict[str, Any] = {
-                        "role": "User",
-                        "content": content,
-                    }
-                    if images:
-                        msg["images"] = images
-                    deepseek_messages.append(msg)
-                if i == len(messages) - 1:
-                    prompt = content
-            elif role == "assistant":
-                deepseek_messages.append({"role": "Assistant", "content": content})
-            else:
-                logger.error(
-                    f"Unexpected message in messages: role: {role}, message: {message}"
-                )
-
-        from ....thirdparty.deepseek_vl.serve.inference import generate
-        from ....thirdparty.deepseek_vl.utils.io import load_pil_images
-
-        # load images and prepare for inputs
-        pil_images = load_pil_images(deepseek_messages)
-        prepare_inputs = self._vl_chat_processor(
-            conversations=deepseek_messages, images=pil_images, force_batchify=True
-        ).to(self._model.device, self._model.dtype)
-
-        temperature = generate_config.get("temperature", 0.2)
-        top_p = generate_config.get("top_p", 0.95)
-        max_new_tokens = generate_config.get("max_tokens", 512)
-        repetition_penalty = generate_config.get("repetition_penalty", 1.1)
-
-        conversation = self._vl_chat_processor.new_chat_template()
-        stop_str = conversation.sep2
-        stop_words = [stop_str]
-
-        streamer = generate(
-            vl_gpt=self._model,
-            tokenizer=self._tokenizer,
-            prepare_inputs=prepare_inputs,
-            max_gen_len=max_new_tokens,
-            temperature=temperature,
-            repetition_penalty=repetition_penalty,
-            top_p=top_p,
-            stop_words=stop_words,
-        )
-
-        if stream:
-            it = self._generate_stream(streamer, stop_str, include_usage, prompt)
-            return self._to_chat_completion_chunks(it)
-        else:
-            return self._generate(streamer, stop_str)
-
-    def _generate(self, streamer, stop_str) -> ChatCompletion:
-        generated_text = ""
-        for new_text in streamer:
-            if new_text.endswith(stop_str):
-                new_text = new_text[: -len(stop_str)]
-            generated_text += new_text
-
-        return generate_chat_completion(self.model_uid, generated_text)
-
-    def _generate_stream(
-        self, streamer, stop_str, include_usage, prompt
-    ) -> Iterator[CompletionChunk]:
-        completion_id = str(uuid.uuid1())
-        prompt_tokens, completion_tokens, total_tokens = 0, 0, 0
-        input_ids = self._tokenizer(prompt).input_ids
-        prompt_tokens = len(input_ids)
-        for i, new_text in enumerate(streamer):
-            if new_text.endswith(stop_str):
-                new_text = new_text[: -len(stop_str)]
-            completion_tokens = i
-            total_tokens = prompt_tokens + completion_tokens
-            yield generate_completion_chunk(
-                chunk_text=new_text,
-                finish_reason=None,
-                chunk_id=completion_id,
-                model_uid=self.model_uid,
-                prompt_tokens=prompt_tokens,
-                completion_tokens=completion_tokens,
-                total_tokens=total_tokens,
-                has_choice=True,
-                has_content=True,
-            )
-        yield generate_completion_chunk(
-            chunk_text=None,
-            finish_reason="stop",
-            chunk_id=completion_id,
-            model_uid=self.model_uid,
-            prompt_tokens=prompt_tokens,
-            completion_tokens=completion_tokens,
-            total_tokens=total_tokens,
-            has_choice=True,
-            has_content=False,
-        )
-
-        if include_usage:
-            yield generate_completion_chunk(
-                chunk_text=None,
-                finish_reason=None,
-                chunk_id=completion_id,
-                model_uid=self.model_uid,
-                prompt_tokens=prompt_tokens,
-                completion_tokens=completion_tokens,
-                total_tokens=total_tokens,
-                has_choice=False,
-                has_content=False,
-            )