xinference 1.6.0.post1__py3-none-any.whl → 1.7.0__py3-none-any.whl

xinference/thirdparty/omnilmm/chat.py

@@ -1,218 +0,0 @@
-import base64
-import io
-import json
-import os
-
-import torch
-from PIL import Image
-from transformers import AutoModel, AutoTokenizer
-
-DEFAULT_IMAGE_TOKEN = "<image>"
-DEFAULT_IMAGE_PATCH_TOKEN = "<im_patch>"
-DEFAULT_IM_START_TOKEN = "<im_start>"
-DEFAULT_IM_END_TOKEN = "<im_end>"
-
-
-def init_omni_lmm(model_path, device_map):
-    from accelerate import init_empty_weights, load_checkpoint_and_dispatch
-
-    from .model.omnilmm import OmniLMMForCausalLM
-    from .model.utils import build_transform
-    from .utils import disable_torch_init
-
-    torch.backends.cuda.matmul.allow_tf32 = True
-    disable_torch_init()
-    model_name = os.path.expanduser(model_path)
-    print(f"Load omni_lmm model and tokenizer from {model_name}")
-    tokenizer = AutoTokenizer.from_pretrained(model_name, model_max_length=2048)
-
-    if False:
-        # model on multiple devices for small size gpu memory (Nvidia 3090 24G x2)
-        with init_empty_weights():
-            model = OmniLMMForCausalLM.from_pretrained(
-                model_name, tune_clip=True, torch_dtype=torch.bfloat16
-            )
-        model = load_checkpoint_and_dispatch(
-            model,
-            model_name,
-            dtype=torch.bfloat16,
-            device_map="auto",
-            no_split_module_classes=[
-                "Eva",
-                "MistralDecoderLayer",
-                "ModuleList",
-                "Resampler",
-            ],
-        )
-    else:
-        model = OmniLMMForCausalLM.from_pretrained(
-            model_name,
-            tune_clip=True,
-            torch_dtype=torch.bfloat16,
-            device_map=device_map,
-        ).to(dtype=torch.bfloat16)
-
-    image_processor = build_transform(
-        is_train=False, input_size=model.model.config.image_size, std_mode="OPENAI_CLIP"
-    )
-
-    mm_use_im_start_end = getattr(model.config, "mm_use_im_start_end", False)
-    assert mm_use_im_start_end
-
-    tokenizer.add_tokens(
-        [DEFAULT_IMAGE_PATCH_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN],
-        special_tokens=True,
-    )
-
-    vision_config = model.model.vision_config
-    vision_config.im_patch_token = tokenizer.convert_tokens_to_ids(
-        [DEFAULT_IMAGE_PATCH_TOKEN]
-    )[0]
-    vision_config.use_im_start_end = mm_use_im_start_end
-    (
-        vision_config.im_start_token,
-        vision_config.im_end_token,
-    ) = tokenizer.convert_tokens_to_ids([DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN])
-    image_token_len = model.model.config.num_query
-
-    return model, image_processor, image_token_len, tokenizer
-
-
-def expand_question_into_multimodal(
-    question_text, image_token_len, im_st_token, im_ed_token, im_patch_token
-):
-    if "<image>" in question_text[0]["content"]:
-        question_text[0]["content"] = question_text[0]["content"].replace(
-            "<image>", im_st_token + im_patch_token * image_token_len + im_ed_token
-        )
-    else:
-        question_text[0]["content"] = (
-            im_st_token
-            + im_patch_token * image_token_len
-            + im_ed_token
-            + "\n"
-            + question_text[0]["content"]
-        )
-    return question_text
-
-
-def wrap_question_for_omni_lmm(question, image_token_len, tokenizer):
-    from .train.train_utils import omni_preprocess
-
-    question = expand_question_into_multimodal(
-        question,
-        image_token_len,
-        DEFAULT_IM_START_TOKEN,
-        DEFAULT_IM_END_TOKEN,
-        DEFAULT_IMAGE_PATCH_TOKEN,
-    )
-
-    conversation = question
-    data_dict = omni_preprocess(
-        sources=[conversation], tokenizer=tokenizer, generation=True
-    )
-
-    data_dict = dict(input_ids=data_dict["input_ids"][0], labels=data_dict["labels"][0])
-    return data_dict
-
-
-class OmniLMM12B:
-    def __init__(self, model_path, device_map) -> None:
-        model, img_processor, image_token_len, tokenizer = init_omni_lmm(
-            model_path, device_map
-        )
-        self.model = model
-        self.image_token_len = image_token_len
-        self.image_transform = img_processor
-        self.tokenizer = tokenizer
-        self.model.eval()
-
-    def decode(self, image, input_ids):
-        with torch.inference_mode():
-            output = self.model.generate_vllm(
-                input_ids=input_ids.unsqueeze(0).cuda(),
-                images=image.unsqueeze(0).half().cuda(),
-                temperature=0.6,
-                max_new_tokens=1024,
-                # num_beams=num_beams,
-                do_sample=True,
-                output_scores=True,
-                return_dict_in_generate=True,
-                repetition_penalty=1.1,
-                top_k=30,
-                top_p=0.9,
-            )
-
-            response = self.tokenizer.decode(
-                output.sequences[0], skip_special_tokens=True
-            )
-            response = response.strip()
-            return response
-
-    def chat(self, input):
-        try:
-            image = Image.open(io.BytesIO(base64.b64decode(input["image"]))).convert(
-                "RGB"
-            )
-        except Exception as e:
-            return f"Image decode error: {e}"
-
-        msgs = json.loads(input["question"])
-        input_ids = wrap_question_for_omni_lmm(
-            msgs, self.image_token_len, self.tokenizer
-        )["input_ids"]
-        input_ids = torch.as_tensor(input_ids)
-        # print('input_ids', input_ids)
-        image = self.image_transform(image)
-
-        out = self.decode(image, input_ids)
-
-        return out
-
-
-def img2base64(file_name):
-    with open(file_name, "rb") as f:
-        encoded_string = base64.b64encode(f.read())
-        return encoded_string
-
-
-class OmniLMM3B:
-    def __init__(self, model_path, device_map) -> None:
-        self.model = AutoModel.from_pretrained(
-            model_path, trust_remote_code=True, device_map=device_map
-        ).to(dtype=torch.bfloat16)
-        self.tokenizer = AutoTokenizer.from_pretrained(
-            model_path, trust_remote_code=True
-        )
-        self.model.eval().cuda()
-
-    def chat(self, input):
-        try:
-            image = Image.open(io.BytesIO(base64.b64decode(input["image"]))).convert(
-                "RGB"
-            )
-        except Exception as e:
-            return f"Image decode error: {e}"
-
-        msgs = json.loads(input["question"])
-
-        answer, context, _ = self.model.chat(
-            image=image,
-            msgs=msgs,
-            context=None,
-            tokenizer=self.tokenizer,
-            sampling=True,
-            temperature=0.7,
-        )
-        return answer
-
-
-class OmniLMMChat:
-    def __init__(self, model_path, device_map) -> None:
-        if "12b" in model_path:
-            self.model = OmniLMM12B(model_path, device_map)
-        else:
-            self.model = OmniLMM3B(model_path, device_map)
-
-    def chat(self, input):
-        return self.model.chat(input)

xinference/thirdparty/omnilmm/constants.py

@@ -1,4 +0,0 @@
-CONTROLLER_HEART_BEAT_EXPIRATION = 30
-WORKER_HEART_BEAT_INTERVAL = 15
-
-LOGDIR = "."

xinference/thirdparty/omnilmm/conversation.py

@@ -1,332 +0,0 @@
-import dataclasses
-from enum import Enum, auto
-from typing import List
-
-
-class SeparatorStyle(Enum):
-    """Different separator style."""
-
-    SINGLE = auto()
-    TWO = auto()
-
-
-@dataclasses.dataclass
-class Conversation:
-    """A class that keeps all conversation history."""
-
-    system: str
-    roles: List[str]
-    messages: List[List[str]]
-    offset: int
-    sep_style: SeparatorStyle = SeparatorStyle.SINGLE
-    sep: str = "###"
-    sep2: str = None
-    version: str = "Unknown"
-
-    skip_next: bool = False
-
-    def get_prompt(self):
-        if self.sep_style == SeparatorStyle.SINGLE:
-            ret = self.system + self.sep
-            for role, message in self.messages:
-                if message:
-                    if type(message) is tuple:
-                        message, _, _ = message
-                    ret += role + ": " + message + self.sep
-                else:
-                    ret += role + ":"
-            return ret
-        elif self.sep_style == SeparatorStyle.TWO:
-            seps = [self.sep, self.sep2]
-            ret = self.system + seps[0]
-            for i, (role, message) in enumerate(self.messages):
-                if message:
-                    if type(message) is tuple:
-                        message, _, _ = message
-                    ret += role + ": " + message + seps[i % 2]
-                else:
-                    ret += role + ":"
-            return ret
-        else:
-            raise ValueError(f"Invalid style: {self.sep_style}")
-
-    def append_message(self, role, message):
-        self.messages.append([role, message])
-
-    def get_images(self, return_pil=False):
-        images = []
-        for i, (role, msg) in enumerate(self.messages[self.offset :]):
-            if i % 2 == 0:
-                if type(msg) is tuple:
-                    import base64
-                    from io import BytesIO
-
-                    from PIL import Image
-
-                    msg, image, image_process_mode = msg
-                    if image_process_mode == "Pad":
-
-                        def expand2square(pil_img, background_color=(122, 116, 104)):
-                            width, height = pil_img.size
-                            if width == height:
-                                return pil_img
-                            elif width > height:
-                                result = Image.new(
-                                    pil_img.mode, (width, width), background_color
-                                )
-                                result.paste(pil_img, (0, (width - height) // 2))
-                                return result
-                            else:
-                                result = Image.new(
-                                    pil_img.mode, (height, height), background_color
-                                )
-                                result.paste(pil_img, ((height - width) // 2, 0))
-                                return result
-
-                        image = expand2square(image)
-                    elif image_process_mode == "Crop":
-                        pass
-                    elif image_process_mode == "Resize":
-                        image = image.resize((224, 224))
-                    else:
-                        raise ValueError(
-                            f"Invalid image_process_mode: {image_process_mode}"
-                        )
-                    max_hw, min_hw = max(image.size), min(image.size)
-                    aspect_ratio = max_hw / min_hw
-                    max_len, min_len = 800, 400
-                    shortest_edge = int(min(max_len / aspect_ratio, min_len, min_hw))
-                    longest_edge = int(shortest_edge * aspect_ratio)
-                    W, H = image.size
-                    if H > W:
-                        H, W = longest_edge, shortest_edge
-                    else:
-                        H, W = shortest_edge, longest_edge
-                    image = image.resize((W, H))
-                    if return_pil:
-                        images.append(image)
-                    else:
-                        buffered = BytesIO()
-                        image.save(buffered, format="JPEG")
-                        img_b64_str = base64.b64encode(buffered.getvalue()).decode()
-                        images.append(img_b64_str)
-        return images
-
-    def to_gradio_chatbot(self):
-        ret = []
-        for i, (role, msg) in enumerate(self.messages[self.offset :]):
-            if i % 2 == 0:
-                if type(msg) is tuple:
-                    import base64
-                    from io import BytesIO
-
-                    msg, image, image_process_mode = msg
-                    max_hw, min_hw = max(image.size), min(image.size)
-                    aspect_ratio = max_hw / min_hw
-                    max_len, min_len = 800, 400
-                    shortest_edge = int(min(max_len / aspect_ratio, min_len, min_hw))
-                    longest_edge = int(shortest_edge * aspect_ratio)
-                    W, H = image.size
-                    if H > W:
-                        H, W = longest_edge, shortest_edge
-                    else:
-                        H, W = shortest_edge, longest_edge
-                    image = image.resize((W, H))
-                    # image = image.resize((224, 224))
-                    buffered = BytesIO()
-                    image.save(buffered, format="JPEG")
-                    img_b64_str = base64.b64encode(buffered.getvalue()).decode()
-                    img_str = f'<img src="data:image/png;base64,{img_b64_str}" alt="user upload image" />'
-                    msg = msg.replace("<image>", img_str)
-                ret.append([msg, None])
-            else:
-                ret[-1][-1] = msg
-        return ret
-
-    def copy(self):
-        return Conversation(
-            system=self.system,
-            roles=self.roles,
-            messages=[[x, y] for x, y in self.messages],
-            offset=self.offset,
-            sep_style=self.sep_style,
-            sep=self.sep,
-            sep2=self.sep2,
-        )
-
-    def dict(self):
-        if len(self.get_images()) > 0:
-            return {
-                "system": self.system,
-                "roles": self.roles,
-                "messages": [
-                    [x, y[0] if type(y) is tuple else y] for x, y in self.messages
-                ],
-                "offset": self.offset,
-                "sep": self.sep,
-                "sep2": self.sep2,
-            }
-        return {
-            "system": self.system,
-            "roles": self.roles,
-            "messages": self.messages,
-            "offset": self.offset,
-            "sep": self.sep,
-            "sep2": self.sep2,
-        }
-
-
-conv_v1 = Conversation(
-    system="A chat between a curious human and an artificial intelligence assistant. "
-    "The assistant gives helpful, detailed, and polite answers to the human's questions.",
-    roles=("Human", "Assistant"),
-    messages=(
-        ("Human", "Give three tips for staying healthy."),
-        (
-            "Assistant",
-            "Sure, here are three tips for staying healthy:\n"
-            "1. Exercise regularly: Regular physical activity can help improve your overall health and wellbeing. "
-            "It can also help reduce your risk of chronic conditions such as obesity, diabetes, heart disease, "
-            "and certain cancers. Aim for at least 150 minutes of moderate-intensity aerobic exercise or "
-            "75 minutes of vigorous-intensity aerobic exercise per week, along with muscle-strengthening "
-            "activities at least two days per week.\n"
-            "2. Eat a balanced diet: Eating a balanced diet that is rich in fruits, "
-            "vegetables, whole grains, lean proteins, and healthy fats can help support "
-            "your overall health. Try to limit your intake of processed and high-sugar foods, "
-            "and aim to drink plenty of water throughout the day.\n"
-            "3. Get enough sleep: Getting enough quality sleep is essential for your physical "
-            "and mental health. Adults should aim for seven to nine hours of sleep per night. "
-            "Establish a regular sleep schedule and try to create a relaxing bedtime routine to "
-            "help improve the quality of your sleep.",
-        ),
-    ),
-    offset=2,
-    sep_style=SeparatorStyle.SINGLE,
-    sep="###",
-)
-
-conv_v1_2 = Conversation(
-    system="A chat between a curious human and an artificial intelligence assistant. "
-    "The assistant gives helpful, detailed, and polite answers to the human's questions.",
-    roles=("Human", "Assistant"),
-    messages=(
-        (
-            "Human",
-            "What are the key differences between renewable and non-renewable energy sources?",
-        ),
-        (
-            "Assistant",
-            "Renewable energy sources are those that can be replenished naturally in a relatively "
-            "short amount of time, such as solar, wind, hydro, geothermal, and biomass. "
-            "Non-renewable energy sources, on the other hand, are finite and will eventually be "
-            "depleted, such as coal, oil, and natural gas. Here are some key differences between "
-            "renewable and non-renewable energy sources:\n"
-            "1. Availability: Renewable energy sources are virtually inexhaustible, while non-renewable "
-            "energy sources are finite and will eventually run out.\n"
-            "2. Environmental impact: Renewable energy sources have a much lower environmental impact "
-            "than non-renewable sources, which can lead to air and water pollution, greenhouse gas emissions, "
-            "and other negative effects.\n"
-            "3. Cost: Renewable energy sources can be more expensive to initially set up, but they typically "
-            "have lower operational costs than non-renewable sources.\n"
-            "4. Reliability: Renewable energy sources are often more reliable and can be used in more remote "
-            "locations than non-renewable sources.\n"
-            "5. Flexibility: Renewable energy sources are often more flexible and can be adapted to different "
-            "situations and needs, while non-renewable sources are more rigid and inflexible.\n"
-            "6. Sustainability: Renewable energy sources are more sustainable over the long term, while "
-            "non-renewable sources are not, and their depletion can lead to economic and social instability.\n",
-        ),
-    ),
-    offset=2,
-    sep_style=SeparatorStyle.SINGLE,
-    sep="###",
-)
-
-conv_vicuna_v1_1 = Conversation(
-    system="A chat between a curious user and an artificial intelligence assistant. "
-    "The assistant gives helpful, detailed, and polite answers to the user's questions.",
-    roles=("USER", "ASSISTANT"),
-    version="v1",
-    messages=(),
-    offset=0,
-    sep_style=SeparatorStyle.TWO,
-    sep=" ",
-    sep2="</s>",
-)
-
-conv_bair_v1 = Conversation(
-    system="BEGINNING OF CONVERSATION:",
-    roles=("USER", "GPT"),
-    messages=(),
-    offset=0,
-    sep_style=SeparatorStyle.TWO,
-    sep=" ",
-    sep2="</s>",
-)
-
-simple_conv = Conversation(
-    system="You are LLaVA, a large language model trained by UW Madison WAIV Lab, based on LLaMA architecture."
-    "You are designed to assist human with a variety of tasks using natural language."
-    "Follow the instructions carefully.",
-    roles=("Human", "Assistant"),
-    messages=(
-        ("Human", "Hi!"),
-        ("Assistant", "Hi there!  How can I help you today?\n"),
-    ),
-    offset=2,
-    sep_style=SeparatorStyle.SINGLE,
-    sep="###",
-)
-
-simple_conv_multimodal = Conversation(
-    system="A chat between a curious user and an artificial intelligence assistant. "
-    "The assistant gives helpful, detailed, and polite answers to the user's questions.",
-    roles=("Human", "Assistant"),
-    messages=(),
-    offset=0,
-    sep_style=SeparatorStyle.SINGLE,
-    sep="###",
-)
-
-simple_conv_legacy = Conversation(
-    system="You are LLaVA, a large language model trained by UW Madison WAIV Lab."
-    "You are designed to assist human with a variety of tasks using natural language."
-    "Follow the instructions carefully.",
-    roles=("Human", "Assistant"),
-    messages=(
-        ("Human", "Hi!\n\n### Response:"),
-        ("Assistant", "Hi there!  How can I help you today?\n"),
-    ),
-    offset=2,
-    sep_style=SeparatorStyle.SINGLE,
-    sep="###",
-)
-
-conv_llava_v1 = Conversation(
-    system="You are LLaVA, a large language and vision assistant trained by UW Madison WAIV Lab."
-    "You are able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language."
-    "Follow the instructions carefully and explain your answers in detail.",
-    roles=("USER", "ASSISTANT"),
-    version="v1",
-    messages=(),
-    offset=0,
-    sep_style=SeparatorStyle.TWO,
-    sep=" ",
-    sep2="</s>",
-)
-
-default_conversation = conv_v1_2
-conv_templates = {
-    "default": conv_v1_2,
-    "simple": simple_conv,
-    "simple_legacy": simple_conv_legacy,
-    "multimodal": simple_conv_multimodal,
-    "llava_v1": conv_llava_v1,
-    # fastchat
-    "v1": conv_v1_2,
-    "bair_v1": conv_bair_v1,
-    "vicuna_v1_1": conv_vicuna_v1_1,
-}
-
-
-if __name__ == "__main__":
-    print(default_conversation.get_prompt())

xinference/thirdparty/omnilmm/model/__init__.py

@@ -1 +0,0 @@
-from .omnilmm import OmniLMMForCausalLM