doc-page-extractor 0.2.0__py3-none-any.whl → 1.0.2__py3-none-any.whl

doc_page_extractor/struct_eqtable/internvl/conversation.py

@@ -1,394 +0,0 @@
-"""
-Conversation prompt templates.
-
-We kindly request that you import fastchat instead of copying this file if you wish to use it.
-If you have changes in mind, please contribute back so the community can benefit collectively and continue to maintain these valuable templates.
-"""
-
-import dataclasses
-from enum import IntEnum, auto
-from typing import Any, Dict, List, Tuple, Union
-
-
-class SeparatorStyle(IntEnum):
-    """Separator styles."""
-
-    ADD_COLON_SINGLE = auto()
-    ADD_COLON_TWO = auto()
-    ADD_COLON_SPACE_SINGLE = auto()
-    NO_COLON_SINGLE = auto()
-    NO_COLON_TWO = auto()
-    ADD_NEW_LINE_SINGLE = auto()
-    LLAMA2 = auto()
-    CHATGLM = auto()
-    CHATML = auto()
-    CHATINTERN = auto()
-    DOLLY = auto()
-    RWKV = auto()
-    PHOENIX = auto()
-    ROBIN = auto()
-    FALCON_CHAT = auto()
-    CHATGLM3 = auto()
-    INTERNVL_ZH = auto()
-    MPT = auto()
-
-
-@dataclasses.dataclass
-class Conversation:
-    """A class that manages prompt templates and keeps all conversation history."""
-
-    # The name of this template
-    name: str
-    # The template of the system prompt
-    system_template: str = '{system_message}'
-    # The system message
-    system_message: str = ''
-    # The names of two roles
-    roles: Tuple[str] = ('USER', 'ASSISTANT')
-    # All messages. Each item is (role, message).
-    messages: List[List[str]] = ()
-    # The number of few shot examples
-    offset: int = 0
-    # The separator style and configurations
-    sep_style: SeparatorStyle = SeparatorStyle.ADD_COLON_SINGLE
-    sep: str = '\n'
-    sep2: str = None
-    # Stop criteria (the default one is EOS token)
-    stop_str: Union[str, List[str]] = None
-    # Stops generation if meeting any token in this list
-    stop_token_ids: List[int] = None
-
-    def get_prompt(self) -> str:
-        """Get the prompt for generation."""
-        system_prompt = self.system_template.format(system_message=self.system_message)
-        if self.sep_style == SeparatorStyle.ADD_COLON_SINGLE:
-            ret = system_prompt + self.sep
-            for role, message in self.messages:
-                if message:
-                    ret += role + ': ' + message + self.sep
-                else:
-                    ret += role + ':'
-            return ret
-        elif self.sep_style == SeparatorStyle.ADD_COLON_TWO:
-            seps = [self.sep, self.sep2]
-            ret = system_prompt + seps[0]
-            for i, (role, message) in enumerate(self.messages):
-                if message:
-                    ret += role + ': ' + message + seps[i % 2]
-                else:
-                    ret += role + ':'
-            return ret
-        elif self.sep_style == SeparatorStyle.ADD_COLON_SPACE_SINGLE:
-            ret = system_prompt + self.sep
-            for role, message in self.messages:
-                if message:
-                    ret += role + ': ' + message + self.sep
-                else:
-                    ret += role + ': '  # must be end with a space
-            return ret
-        elif self.sep_style == SeparatorStyle.ADD_NEW_LINE_SINGLE:
-            ret = '' if system_prompt == '' else system_prompt + self.sep
-            for role, message in self.messages:
-                if message:
-                    ret += role + '\n' + message + self.sep
-                else:
-                    ret += role + '\n'
-            return ret
-        elif self.sep_style == SeparatorStyle.NO_COLON_SINGLE:
-            ret = system_prompt
-            for role, message in self.messages:
-                if message:
-                    ret += role + message + self.sep
-                else:
-                    ret += role
-            return ret
-        elif self.sep_style == SeparatorStyle.NO_COLON_TWO:
-            seps = [self.sep, self.sep2]
-            ret = system_prompt
-            for i, (role, message) in enumerate(self.messages):
-                if message:
-                    ret += role + message + seps[i % 2]
-                else:
-                    ret += role
-            return ret
-        elif self.sep_style == SeparatorStyle.RWKV:
-            ret = system_prompt
-            for i, (role, message) in enumerate(self.messages):
-                if message:
-                    ret += (
-                        role
-                        + ': '
-                        + message.replace('\r\n', '\n').replace('\n\n', '\n')
-                    )
-                    ret += '\n\n'
-                else:
-                    ret += role + ':'
-            return ret
-        elif self.sep_style == SeparatorStyle.LLAMA2:
-            seps = [self.sep, self.sep2]
-            if self.system_message:
-                ret = system_prompt
-            else:
-                ret = '[INST] '
-            for i, (role, message) in enumerate(self.messages):
-                tag = self.roles[i % 2]
-                if message:
-                    if i == 0:
-                        ret += message + ' '
-                    else:
-                        ret += tag + ' ' + message + seps[i % 2]
-                else:
-                    ret += tag
-            return ret
-        elif self.sep_style == SeparatorStyle.CHATGLM:
-            # source: https://huggingface.co/THUDM/chatglm-6b/blob/1d240ba371910e9282298d4592532d7f0f3e9f3e/modeling_chatglm.py#L1302-L1308
-            # source2: https://huggingface.co/THUDM/chatglm2-6b/blob/e186c891cf64310ac66ef10a87e6635fa6c2a579/modeling_chatglm.py#L926
-            round_add_n = 1 if self.name == 'chatglm2' else 0
-            if system_prompt:
-                ret = system_prompt + self.sep
-            else:
-                ret = ''
-
-            for i, (role, message) in enumerate(self.messages):
-                if i % 2 == 0:
-                    ret += f'[Round {i//2 + round_add_n}]{self.sep}'
-
-                if message:
-                    ret += f'{role}：{message}{self.sep}'
-                else:
-                    ret += f'{role}：'
-            return ret
-        elif self.sep_style == SeparatorStyle.CHATML:
-            ret = '' if system_prompt == '' else system_prompt + self.sep + '\n'
-            for role, message in self.messages:
-                if message:
-                    ret += role + '\n' + message + self.sep + '\n'
-                else:
-                    ret += role + '\n'
-            return ret
-        elif self.sep_style == SeparatorStyle.CHATGLM3:
-            ret = ''
-            if self.system_message:
-                ret += system_prompt
-            for role, message in self.messages:
-                if message:
-                    ret += role + '\n' + ' ' + message
-                else:
-                    ret += role
-            return ret
-        elif self.sep_style == SeparatorStyle.CHATINTERN:
-            # source: https://huggingface.co/internlm/internlm-chat-7b-8k/blob/bd546fa984b4b0b86958f56bf37f94aa75ab8831/modeling_internlm.py#L771
-            seps = [self.sep, self.sep2]
-            ret = system_prompt
-            for i, (role, message) in enumerate(self.messages):
-                # if i % 2 == 0:
-                #     ret += "<s>"
-                if message:
-                    ret += role + ':' + message + seps[i % 2] + '\n'
-                else:
-                    ret += role + ':'
-            return ret
-        elif self.sep_style == SeparatorStyle.DOLLY:
-            seps = [self.sep, self.sep2]
-            ret = system_prompt
-            for i, (role, message) in enumerate(self.messages):
-                if message:
-                    ret += role + ':\n' + message + seps[i % 2]
-                    if i % 2 == 1:
-                        ret += '\n\n'
-                else:
-                    ret += role + ':\n'
-            return ret
-        elif self.sep_style == SeparatorStyle.PHOENIX:
-            ret = system_prompt
-            for role, message in self.messages:
-                if message:
-                    ret += role + ': ' + '<s>' + message + '</s>'
-                else:
-                    ret += role + ': ' + '<s>'
-            return ret
-        elif self.sep_style == SeparatorStyle.ROBIN:
-            ret = system_prompt + self.sep
-            for role, message in self.messages:
-                if message:
-                    ret += role + ':\n' + message + self.sep
-                else:
-                    ret += role + ':\n'
-            return ret
-        elif self.sep_style == SeparatorStyle.FALCON_CHAT:
-            ret = ''
-            if self.system_message:
-                ret += system_prompt + self.sep
-            for role, message in self.messages:
-                if message:
-                    ret += role + ': ' + message + self.sep
-                else:
-                    ret += role + ':'
-
-            return ret
-        elif self.sep_style == SeparatorStyle.INTERNVL_ZH:
-            seps = [self.sep, self.sep2]
-            ret = self.system_message + seps[0]
-            for i, (role, message) in enumerate(self.messages):
-                if message:
-                    ret += role + ': ' + message + seps[i % 2]
-                else:
-                    ret += role + ':'
-            return ret
-        elif self.sep_style == SeparatorStyle.MPT:
-            ret = system_prompt + self.sep
-            for role, message in self.messages:
-                if message:
-                    if type(message) is tuple:
-                        message, _, _ = message
-                    ret += role + message + self.sep
-                else:
-                    ret += role
-            return ret
-        else:
-            raise ValueError(f'Invalid style: {self.sep_style}')
-
-    def set_system_message(self, system_message: str):
-        """Set the system message."""
-        self.system_message = system_message
-
-    def append_message(self, role: str, message: str):
-        """Append a new message."""
-        self.messages.append([role, message])
-
-    def update_last_message(self, message: str):
-        """Update the last output.
-
-        The last message is typically set to be None when constructing the prompt,
-        so we need to update it in-place after getting the response from a model.
-        """
-        self.messages[-1][1] = message
-
-    def to_gradio_chatbot(self):
-        """Convert the conversation to gradio chatbot format."""
-        ret = []
-        for i, (role, msg) in enumerate(self.messages[self.offset :]):
-            if i % 2 == 0:
-                ret.append([msg, None])
-            else:
-                ret[-1][-1] = msg
-        return ret
-
-    def to_openai_api_messages(self):
-        """Convert the conversation to OpenAI chat completion format."""
-        ret = [{'role': 'system', 'content': self.system_message}]
-
-        for i, (_, msg) in enumerate(self.messages[self.offset :]):
-            if i % 2 == 0:
-                ret.append({'role': 'user', 'content': msg})
-            else:
-                if msg is not None:
-                    ret.append({'role': 'assistant', 'content': msg})
-        return ret
-
-    def copy(self):
-        return Conversation(
-            name=self.name,
-            system_template=self.system_template,
-            system_message=self.system_message,
-            roles=self.roles,
-            messages=[[x, y] for x, y in self.messages],
-            offset=self.offset,
-            sep_style=self.sep_style,
-            sep=self.sep,
-            sep2=self.sep2,
-            stop_str=self.stop_str,
-            stop_token_ids=self.stop_token_ids,
-        )
-
-    def dict(self):
-        return {
-            'template_name': self.name,
-            'system_message': self.system_message,
-            'roles': self.roles,
-            'messages': self.messages,
-            'offset': self.offset,
-        }
-
-
-# A global registry for all conversation templates
-conv_templates: Dict[str, Conversation] = {}
-
-
-def register_conv_template(template: Conversation, override: bool = False):
-    """Register a new conversation template."""
-    if not override:
-        assert (
-            template.name not in conv_templates
-        ), f'{template.name} has been registered.'
-
-    conv_templates[template.name] = template
-
-
-def get_conv_template(name: str) -> Conversation:
-    """Get a conversation template."""
-    return conv_templates[name].copy()
-
-
-# Both Hermes-2 and internlm2-chat are chatml-format conversation templates. The difference
-# is that during training, the preprocessing function for the Hermes-2 template doesn't add
-# <s> at the beginning of the tokenized sequence, while the internlm2-chat template does.
-# Therefore, they are completely equivalent during inference.
-register_conv_template(
-    Conversation(
-        name='Hermes-2',
-        system_template='<|im_start|>system\n{system_message}',
-        # note: The new system prompt was not used here to avoid changes in benchmark performance.
-        # system_message='我是书生·万象，英文名是InternVL，是由上海人工智能实验室、清华大学及多家合作单位联合开发的多模态大语言模型。',
-        # system_message='你是由上海人工智能实验室联合商汤科技开发的书生多模态大模型，英文名叫InternVL, 是一个有用无害的人工智能助手。',
-        system_message='You are a Table Image to LaTeX/Markdown/HMTL Code converter.',
-        roles=('<|im_start|>user\n', '<|im_start|>assistant\n'),
-        sep_style=SeparatorStyle.MPT,
-        sep='<|im_end|>',
-        stop_token_ids=[
-            2,
-            6,
-            7,
-            8,
-        ],
-        stop_str='<|endoftext|>',
-    )
-)
-
-
-register_conv_template(
-    Conversation(
-        name='internlm2-chat',
-        system_template='<|im_start|>system\n{system_message}',
-        # note: The new system prompt was not used here to avoid changes in benchmark performance.
-        # system_message='我是书生·万象，英文名是InternVL，是由上海人工智能实验室、清华大学及多家合作单位联合开发的多模态大语言模型。',
-        system_message='你是由上海人工智能实验室联合商汤科技开发的书生多模态大模型，英文名叫InternVL, 是一个有用无害的人工智能助手。',
-        roles=('<|im_start|>user\n', '<|im_start|>assistant\n'),
-        sep_style=SeparatorStyle.MPT,
-        sep='<|im_end|>',
-        stop_token_ids=[
-            2,
-            92543,
-            92542
-        ]
-    )
-)
-
-
-register_conv_template(
-    Conversation(
-        name='phi3-chat',
-        system_template='<|system|>\n{system_message}',
-        # note: The new system prompt was not used here to avoid changes in benchmark performance.
-        # system_message='我是书生·万象，英文名是InternVL，是由上海人工智能实验室、清华大学及多家合作单位联合开发的多模态大语言模型。',
-        system_message='你是由上海人工智能实验室联合商汤科技开发的书生多模态大模型，英文名叫InternVL, 是一个有用无害的人工智能助手。',
-        roles=('<|user|>\n', '<|assistant|>\n'),
-        sep_style=SeparatorStyle.MPT,
-        sep='<|end|>',
-        stop_token_ids=[
-            2,
-            32000,
-            32007
-        ]
-    )
-)

doc_page_extractor/struct_eqtable/internvl/internvl.py

@@ -1,198 +0,0 @@
-import torch
-
-from torch import nn
-from transformers import AutoModel, AutoTokenizer, AutoImageProcessor, GenerationConfig
-
-from .conversation import get_conv_template
-
-class InternVL(nn.Module):
-    def __init__(
-            self,
-            model_path='U4R/StructTable-InternVL2-1B',
-            max_new_tokens=1024,
-            max_time=30,
-            flash_attn=True,
-            cache_dir=None,
-            local_files_only=None,
-            **kwargs,
-        ):
-        super().__init__()
-        self.model_path = model_path
-        self.max_new_tokens = max_new_tokens
-        self.max_generate_time = max_time
-        self.flash_attn = flash_attn
-        self.cache_dir = cache_dir
-        self.local_files_only = local_files_only
-
-        # init model and image processor from ckpt path
-        self.init_tokenizer(model_path)
-        self.init_image_processor(model_path)
-        self.init_model(model_path)
-
-        self.prompt_template = {
-            'latex': '<latex>',
-            'html': '<html>',
-            'markdown': '<markdown>',
-        }
-        # support output format
-        self.supported_output_format = ['latex', 'html', 'markdown']
-
-    def init_model(self, model_path):
-        self.model = AutoModel.from_pretrained(
-            pretrained_model_name_or_path=model_path,
-            trust_remote_code=True,
-            torch_dtype=torch.bfloat16,
-            low_cpu_mem_usage=True,
-            use_flash_attn=self.flash_attn,
-            cache_dir=self.cache_dir,
-            local_files_only=self.local_files_only,
-        )
-        self.model.eval()
-
-    def init_image_processor(self, image_processor_path):
-        self.image_processor = AutoImageProcessor.from_pretrained(
-            pretrained_model_name_or_path=image_processor_path,
-            trust_remote_code=True,
-            cache_dir=self.cache_dir,
-            local_files_only=self.local_files_only,
-        )
-
-    def init_tokenizer(self, tokenizer_path):
-        self.tokenizer = AutoTokenizer.from_pretrained(
-            pretrained_model_name_or_path=tokenizer_path,
-            trust_remote_code=True,
-            use_fast=False,
-            cache_dir=self.cache_dir,
-            local_files_only=self.local_files_only,
-        )
-
-        self.image_context_token = '<IMG_CONTEXT>'
-        self.image_token_num = 256
-        self.image_start_token = '<img>'
-        self.image_end_token = '</img>'
-        self.img_context_token_id = self.tokenizer.convert_tokens_to_ids(self.image_context_token)
-
-    def format_image_tokens(self, path_num):
-        return f'{self.image_start_token}{self.image_context_token* self.image_token_num * path_num}{self.image_end_token}'
-
-    def forward(self, images, output_format='latex', **kwargs):
-        # process image to tokens
-        if not isinstance(images, list):
-            images = [images]
-
-        pixel_values_list = []
-        for image in images:
-            path_images = self.dynamic_preprocess(
-                image, image_size=448, max_num=12
-            )
-            pixel_values = self.image_processor(
-                path_images,
-                return_tensors='pt'
-            )['pixel_values'].to(torch.bfloat16)
-            pixel_values_list.append(pixel_values)
-
-        batch_size = len(pixel_values_list)
-        conversation_list = []
-        for bs_idx in range(batch_size):
-            pixel_values= pixel_values_list[bs_idx].to(torch.bfloat16)
-
-            image_tokens = self.format_image_tokens(pixel_values.shape[0])
-            question = '<image>\n' + self.prompt_template[output_format]
-            answer = None
-
-            template = get_conv_template(self.model.config.template)
-            template.append_message(template.roles[0], question)
-            template.append_message(template.roles[1], answer)
-            conversation = template.get_prompt()
-            conversation = conversation.replace('<image>', image_tokens, 1)
-            conversation_list.append(conversation)
-
-        device = next(self.parameters()).device
-        self.tokenizer.padding_side = 'left'
-        model_inputs = self.tokenizer(
-            conversation_list,
-            return_tensors='pt',
-            padding=True,
-            max_length=self.tokenizer.model_max_length,
-            truncation=True,
-        ).to(device)
-        pixel_values = torch.cat(pixel_values_list, axis=0).to(device)
-
-        # generation config
-        generation_config = dict(
-            max_new_tokens=self.max_new_tokens,
-            max_time=self.max_generate_time,
-            img_context_token_id=self.img_context_token_id,
-            pad_token_id=self.tokenizer.pad_token_id,
-            eos_token_id=self.tokenizer.eos_token_id,
-            do_sample=False,
-            no_repeat_ngram_size=20,
-        )
-
-        # generate text from image tokens
-        model_output = self.model.generate(
-            pixel_values=pixel_values,
-            input_ids=model_inputs.input_ids,
-            attention_mask=model_inputs.attention_mask,
-            **generation_config,
-            # **kwargs
-        )
-
-        batch_decode_texts = self.tokenizer.batch_decode(
-            model_output,
-            skip_special_tokens=True
-        )
-        return batch_decode_texts
-
-    def find_closest_aspect_ratio(self, aspect_ratio, target_ratios, width, height, image_size):
-        best_ratio_diff = float('inf')
-        best_ratio = (1, 1)
-        area = width * height
-        for ratio in target_ratios:
-            target_aspect_ratio = ratio[0] / ratio[1]
-            ratio_diff = abs(aspect_ratio - target_aspect_ratio)
-            if ratio_diff < best_ratio_diff:
-                best_ratio_diff = ratio_diff
-                best_ratio = ratio
-            elif ratio_diff == best_ratio_diff:
-                if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
-                    best_ratio = ratio
-        return best_ratio
-
-    def dynamic_preprocess(self, image, min_num=1, max_num=12, image_size=448, use_thumbnail=True):
-        orig_width, orig_height = image.size
-        aspect_ratio = orig_width / orig_height
-
-        # calculate the existing image aspect ratio
-        target_ratios = set(
-            (i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if
-            i * j <= max_num and i * j >= min_num)
-        target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
-
-        # find the closest aspect ratio to the target
-        target_aspect_ratio = self.find_closest_aspect_ratio(
-            aspect_ratio, target_ratios, orig_width, orig_height, image_size)
-
-        # calculate the target width and height
-        target_width = image_size * target_aspect_ratio[0]
-        target_height = image_size * target_aspect_ratio[1]
-        blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
-
-        # resize the image
-        resized_img = image.resize((target_width, target_height))
-        processed_images = []
-        for i in range(blocks):
-            box = (
-                (i % (target_width // image_size)) * image_size,
-                (i // (target_width // image_size)) * image_size,
-                ((i % (target_width // image_size)) + 1) * image_size,
-                ((i // (target_width // image_size)) + 1) * image_size
-            )
-            # split the image
-            split_img = resized_img.crop(box)
-            processed_images.append(split_img)
-        assert len(processed_images) == blocks
-        if use_thumbnail and len(processed_images) != 1:
-            thumbnail_img = image.resize((image_size, image_size))
-            processed_images.append(thumbnail_img)
-        return processed_images

doc_page_extractor/struct_eqtable/internvl/internvl_lmdeploy.py

@@ -1,81 +0,0 @@
-import torch
-from torch import nn
-
-from transformers import AutoTokenizer
-try:
-    from lmdeploy import pipeline, GenerationConfig, PytorchEngineConfig, ChatTemplateConfig
-except:
-    print("\033[93mimport lmdeploy failed, if do not use lmdeploy, ignore this message\033[0m")
-
-
-class InternVL_LMDeploy(nn.Module):
-    def __init__(
-            self,
-            model_path='U4R/StructTable-InternVL2-1B',
-            max_new_tokens=1024,
-            batch_size=4,
-            cache_dir=None,
-            local_files_only=None,
-            **kwargs,
-        ):
-        super().__init__()
-        self.model_path = model_path
-        self.max_new_tokens = max_new_tokens
-        self.max_batch_size = batch_size
-        self.cache_dir = cache_dir
-        self.local_files_only = local_files_only
-
-        # init model and tokenizer from ckpt path
-        self.init_tokenizer(model_path)
-        self.init_model(model_path)
-
-        self.prompt_template = {
-            'latex': '<latex>',
-            'html': '<html>',
-            'markdown': '<markdown>',
-        }
-        # support output format
-        self.supported_output_format = ['latex', 'html', 'markdown']
-
-    def init_tokenizer(self, tokenizer_path):
-        self.tokenizer = AutoTokenizer.from_pretrained(
-            pretrained_model_name_or_path=tokenizer_path,
-            trust_remote_code=True,
-            use_fast=False,
-            cache_dir=self.cache_dir,
-            local_files_only=self.local_files_only,
-        )
-
-    def init_model(self, model_path):
-        engine_config = PytorchEngineConfig(
-            dtype='bfloat16',
-            max_batch_size=self.max_batch_size,
-            cache_max_entry_count=0.1
-        )
-        self.pipeline = pipeline(
-            model_path,
-            backend_config=engine_config,
-            chat_template_config=ChatTemplateConfig(model_name='internvl2-internlm2')
-        )
-
-    def forward(self, images, output_format='latex', **kwargs):
-        # process image to tokens
-        if not isinstance(images, list):
-            images = [images]
-
-        prompts = [self.prompt_template[output_format]] * len(images)
-        generation_config = GenerationConfig(
-            max_new_tokens=self.max_new_tokens,
-            do_sample=False,
-            temperature=1.0,
-            stop_token_ids=[self.tokenizer.eos_token_id],
-        )
-
-        responses = self.pipeline(
-            [(x, y) for x, y in zip(prompts, images)],
-            gen_config=generation_config,
-        )
-        batch_decode_texts = [responce.text for responce in responses]
-        return batch_decode_texts
-
-

doc_page_extractor/struct_eqtable/pix2s/__init__.py

@@ -1,3 +0,0 @@
-from .pix2s import Pix2Struct
-from .pix2s_trt import Pix2StructTensorRT
-