doc-page-extractor 0.2.2__tar.gz → 0.2.3__tar.gz

{doc_page_extractor-0.2.2 → doc_page_extractor-0.2.3}/PKG-INFO

@@ -1,30 +1,33 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.3
 Name: doc-page-extractor
-Version: 0.2.2
-Summary: doc page extractor can identify text and format in images and return structured data.
-Home-page: https://github.com/Moskize91/doc-page-extractor
+Version: 0.2.3
+Summary: 
+License: AGPL-3.0
 Author: Tao Zeyu
 Author-email: i@taozeyu.com
+Maintainer: Tao Zeyu
+Maintainer-email: i@taozeyu.com
+Requires-Python: >=3.10,<3.13
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: GNU Affero General Public License v3
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: accelerate (>=1.6.0,<2.0)
+Requires-Dist: doclayout_yolo (>=0.0.3)
+Requires-Dist: huggingface_hub (>=0.33.0,<1.0)
+Requires-Dist: numpy (>=1.24.0,<2.0)
+Requires-Dist: opencv-python (>=4.10.0,<5.0)
+Requires-Dist: pillow (>=10.3,<11.0)
+Requires-Dist: pix2tex (>=0.1.4,<=0.2.0)
+Requires-Dist: pyclipper (>=1.2.0,<2.0)
+Requires-Dist: shapely (>=2.0.0,<3.0)
+Requires-Dist: transformers (>=4.42.4,<=4.47)
+Project-URL: Repository, https://github.com/moskize91/doc-page-extractor
 Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: opencv-python<5.0,>=4.10.0
-Requires-Dist: pillow<11.0,>=10.3
-Requires-Dist: pyclipper<2.0,>=1.2.0
-Requires-Dist: numpy<2.0,>=1.24.0
-Requires-Dist: shapely<3.0,>=2.0.0
-Requires-Dist: transformers<=4.47,>=4.42.4
-Requires-Dist: doclayout_yolo>=0.0.3
-Requires-Dist: pix2tex<=0.2.0,>=0.1.4
-Requires-Dist: accelerate<2.0,>=1.6.0
-Requires-Dist: huggingface_hub<1.0,>=0.30.2
-Dynamic: author
-Dynamic: author-email
-Dynamic: description
-Dynamic: description-content-type
-Dynamic: home-page
-Dynamic: license-file
-Dynamic: requires-dist
-Dynamic: summary
 
 # doc page extractor
 

{doc_page_extractor-0.2.2 → doc_page_extractor-0.2.3}/doc_page_extractor/model.py

@@ -1,10 +1,14 @@
 from os import PathLike
+from time import sleep
 from typing import runtime_checkable, Protocol
 from pathlib import Path
 from threading import Lock
 from huggingface_hub import hf_hub_download, snapshot_download, try_to_load_from_cache
 
 
+_RETRY_TIMES = 6
+_RETRY_SLEEP = 3.5
+
 @runtime_checkable
 class Model(Protocol):
   def get_onnx_ocr_path(self) -> Path:
@@ -96,19 +100,34 @@ class HuggingfaceModel(Model):
             model_path = model_path.parent
 
       else:
-        if is_snapshot:
-          model_path = snapshot_download(
-            cache_dir=self._model_cache_dir,
-            repo_id=repo_id,
-            repo_type=repo_type,
-          )
-        else:
-          model_path = hf_hub_download(
-            cache_dir=self._model_cache_dir,
-            repo_id=repo_id,
-            repo_type=repo_type,
-            filename=filename,
-          )
+        # https://github.com/huggingface/huggingface_hub/issues/1542#issuecomment-1630465844
+        latest_error: ConnectionError | None = None
+        for i in range(_RETRY_TIMES + 1):
+          if latest_error is not None:
+            print(f"Retrying to download {repo_id} model, attempt {i + 1}/{_RETRY_TIMES}...")
+            sleep(_RETRY_SLEEP)
+          try:
+            if is_snapshot:
+              model_path = snapshot_download(
+                cache_dir=self._model_cache_dir,
+                repo_id=repo_id,
+                repo_type=repo_type,
+                resume_download=True,
+              )
+            else:
+              model_path = hf_hub_download(
+                cache_dir=self._model_cache_dir,
+                repo_id=repo_id,
+                repo_type=repo_type,
+                filename=filename,
+                resume_download=True,
+              )
+            latest_error = None
+          except ConnectionError as err:
+            latest_error = err
+
+        if latest_error is not None:
+          raise latest_error
         model_path = Path(model_path)
 
       return model_path

doc_page_extractor-0.2.3/pyproject.toml

@@ -0,0 +1,48 @@
+[project]
+name = "doc-page-extractor"
+version = "0.2.3"
+description = ""
+authors = [
+    {name = "Tao Zeyu",email = "i@taozeyu.com"}
+]
+maintainers = [
+    {name = "Tao Zeyu", email = "i@taozeyu.com"}
+]
+license = {text = "AGPL-3.0"}
+readme = "README.md"
+requires-python = ">=3.10,<3.13"
+dependencies = [
+    "opencv-python>=4.10.0,<5.0",
+    "pillow>=10.3,<11.0",
+    "pyclipper>=1.2.0,<2.0",
+    "numpy>=1.24.0,<2.0",
+    "shapely>=2.0.0,<3.0",
+    "transformers>=4.42.4,<=4.47",
+    "doclayout_yolo>=0.0.3",
+    "pix2tex>=0.1.4,<=0.2.0",
+    "accelerate>=1.6.0,<2.0",
+    "huggingface_hub>=0.33.0,<1.0",
+]
+
+
+[build-system]
+requires = ["poetry-core>=2.0.0,<3.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.poetry]
+license = "AGPL-3.0"
+readme = "README.md"
+repository = "https://github.com/moskize91/doc-page-extractor"
+packages = [
+    {include = "doc_page_extractor" }
+]
+classifiers=[
+    "Development Status :: 2 - Pre-Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: GNU Affero General Public License v3",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3.10",
+]
+
+[tool.poetry.group.dev.dependencies]
+pylint = "^3.3.7"

doc_page_extractor-0.2.2/doc_page_extractor/struct_eqtable/__init__.py

@@ -1,49 +0,0 @@
-from .pix2s import Pix2Struct, Pix2StructTensorRT
-from .internvl import InternVL, InternVL_LMDeploy
-
-from transformers import AutoConfig
-
-
-__ALL_MODELS__ = {
-    'Pix2Struct': Pix2Struct,
-    'Pix2StructTensorRT': Pix2StructTensorRT,
-    'InternVL': InternVL,
-    'InternVL_LMDeploy': InternVL_LMDeploy,
-}
-
-
-def get_model_name(model_path):
-    model_config = AutoConfig.from_pretrained(
-        model_path,
-        trust_remote_code=True,
-    )
-
-    if 'Pix2Struct' in model_config.architectures[0]:
-        model_name = 'Pix2Struct'
-    elif 'InternVL' in model_config.architectures[0]:
-        model_name = 'InternVL'
-    else:
-        raise ValueError(f"Unsupported model type: {model_config.architectures[0]}")
-
-    return model_name
-
-
-def build_model(
-        model_ckpt='U4R/StructTable-InternVL2-1B',
-        cache_dir=None,
-        local_files_only=None,
-        **kwargs,
-    ):
-    model_name = get_model_name(model_ckpt)
-    if model_name == 'InternVL' and kwargs.get('lmdeploy', False):
-        model_name = 'InternVL_LMDeploy'
-    elif model_name == 'Pix2Struct' and kwargs.get('tensorrt_path', None):
-        model_name = 'Pix2StructTensorRT'
-
-    model = __ALL_MODELS__[model_name](
-        model_ckpt,
-        cache_dir=cache_dir,
-        local_files_only=local_files_only,
-        **kwargs
-    )
-    return model

doc_page_extractor-0.2.2/doc_page_extractor/struct_eqtable/internvl/__init__.py

@@ -1,2 +0,0 @@
-from .internvl import InternVL
-from .internvl_lmdeploy import InternVL_LMDeploy

doc_page_extractor-0.2.2/doc_page_extractor/struct_eqtable/internvl/conversation.py

@@ -1,394 +0,0 @@
-"""
-Conversation prompt templates.
-
-We kindly request that you import fastchat instead of copying this file if you wish to use it.
-If you have changes in mind, please contribute back so the community can benefit collectively and continue to maintain these valuable templates.
-"""
-
-import dataclasses
-from enum import IntEnum, auto
-from typing import Any, Dict, List, Tuple, Union
-
-
-class SeparatorStyle(IntEnum):
-    """Separator styles."""
-
-    ADD_COLON_SINGLE = auto()
-    ADD_COLON_TWO = auto()
-    ADD_COLON_SPACE_SINGLE = auto()
-    NO_COLON_SINGLE = auto()
-    NO_COLON_TWO = auto()
-    ADD_NEW_LINE_SINGLE = auto()
-    LLAMA2 = auto()
-    CHATGLM = auto()
-    CHATML = auto()
-    CHATINTERN = auto()
-    DOLLY = auto()
-    RWKV = auto()
-    PHOENIX = auto()
-    ROBIN = auto()
-    FALCON_CHAT = auto()
-    CHATGLM3 = auto()
-    INTERNVL_ZH = auto()
-    MPT = auto()
-
-
-@dataclasses.dataclass
-class Conversation:
-    """A class that manages prompt templates and keeps all conversation history."""
-
-    # The name of this template
-    name: str
-    # The template of the system prompt
-    system_template: str = '{system_message}'
-    # The system message
-    system_message: str = ''
-    # The names of two roles
-    roles: Tuple[str] = ('USER', 'ASSISTANT')
-    # All messages. Each item is (role, message).
-    messages: List[List[str]] = ()
-    # The number of few shot examples
-    offset: int = 0
-    # The separator style and configurations
-    sep_style: SeparatorStyle = SeparatorStyle.ADD_COLON_SINGLE
-    sep: str = '\n'
-    sep2: str = None
-    # Stop criteria (the default one is EOS token)
-    stop_str: Union[str, List[str]] = None
-    # Stops generation if meeting any token in this list
-    stop_token_ids: List[int] = None
-
-    def get_prompt(self) -> str:
-        """Get the prompt for generation."""
-        system_prompt = self.system_template.format(system_message=self.system_message)
-        if self.sep_style == SeparatorStyle.ADD_COLON_SINGLE:
-            ret = system_prompt + self.sep
-            for role, message in self.messages:
-                if message:
-                    ret += role + ': ' + message + self.sep
-                else:
-                    ret += role + ':'
-            return ret
-        elif self.sep_style == SeparatorStyle.ADD_COLON_TWO:
-            seps = [self.sep, self.sep2]
-            ret = system_prompt + seps[0]
-            for i, (role, message) in enumerate(self.messages):
-                if message:
-                    ret += role + ': ' + message + seps[i % 2]
-                else:
-                    ret += role + ':'
-            return ret
-        elif self.sep_style == SeparatorStyle.ADD_COLON_SPACE_SINGLE:
-            ret = system_prompt + self.sep
-            for role, message in self.messages:
-                if message:
-                    ret += role + ': ' + message + self.sep
-                else:
-                    ret += role + ': '  # must be end with a space
-            return ret
-        elif self.sep_style == SeparatorStyle.ADD_NEW_LINE_SINGLE:
-            ret = '' if system_prompt == '' else system_prompt + self.sep
-            for role, message in self.messages:
-                if message:
-                    ret += role + '\n' + message + self.sep
-                else:
-                    ret += role + '\n'
-            return ret
-        elif self.sep_style == SeparatorStyle.NO_COLON_SINGLE:
-            ret = system_prompt
-            for role, message in self.messages:
-                if message:
-                    ret += role + message + self.sep
-                else:
-                    ret += role
-            return ret
-        elif self.sep_style == SeparatorStyle.NO_COLON_TWO:
-            seps = [self.sep, self.sep2]
-            ret = system_prompt
-            for i, (role, message) in enumerate(self.messages):
-                if message:
-                    ret += role + message + seps[i % 2]
-                else:
-                    ret += role
-            return ret
-        elif self.sep_style == SeparatorStyle.RWKV:
-            ret = system_prompt
-            for i, (role, message) in enumerate(self.messages):
-                if message:
-                    ret += (
-                        role
-                        + ': '
-                        + message.replace('\r\n', '\n').replace('\n\n', '\n')
-                    )
-                    ret += '\n\n'
-                else:
-                    ret += role + ':'
-            return ret
-        elif self.sep_style == SeparatorStyle.LLAMA2:
-            seps = [self.sep, self.sep2]
-            if self.system_message:
-                ret = system_prompt
-            else:
-                ret = '[INST] '
-            for i, (role, message) in enumerate(self.messages):
-                tag = self.roles[i % 2]
-                if message:
-                    if i == 0:
-                        ret += message + ' '
-                    else:
-                        ret += tag + ' ' + message + seps[i % 2]
-                else:
-                    ret += tag
-            return ret
-        elif self.sep_style == SeparatorStyle.CHATGLM:
-            # source: https://huggingface.co/THUDM/chatglm-6b/blob/1d240ba371910e9282298d4592532d7f0f3e9f3e/modeling_chatglm.py#L1302-L1308
-            # source2: https://huggingface.co/THUDM/chatglm2-6b/blob/e186c891cf64310ac66ef10a87e6635fa6c2a579/modeling_chatglm.py#L926
-            round_add_n = 1 if self.name == 'chatglm2' else 0
-            if system_prompt:
-                ret = system_prompt + self.sep
-            else:
-                ret = ''
-
-            for i, (role, message) in enumerate(self.messages):
-                if i % 2 == 0:
-                    ret += f'[Round {i//2 + round_add_n}]{self.sep}'
-
-                if message:
-                    ret += f'{role}：{message}{self.sep}'
-                else:
-                    ret += f'{role}：'
-            return ret
-        elif self.sep_style == SeparatorStyle.CHATML:
-            ret = '' if system_prompt == '' else system_prompt + self.sep + '\n'
-            for role, message in self.messages:
-                if message:
-                    ret += role + '\n' + message + self.sep + '\n'
-                else:
-                    ret += role + '\n'
-            return ret
-        elif self.sep_style == SeparatorStyle.CHATGLM3:
-            ret = ''
-            if self.system_message:
-                ret += system_prompt
-            for role, message in self.messages:
-                if message:
-                    ret += role + '\n' + ' ' + message
-                else:
-                    ret += role
-            return ret
-        elif self.sep_style == SeparatorStyle.CHATINTERN:
-            # source: https://huggingface.co/internlm/internlm-chat-7b-8k/blob/bd546fa984b4b0b86958f56bf37f94aa75ab8831/modeling_internlm.py#L771
-            seps = [self.sep, self.sep2]
-            ret = system_prompt
-            for i, (role, message) in enumerate(self.messages):
-                # if i % 2 == 0:
-                #     ret += "<s>"
-                if message:
-                    ret += role + ':' + message + seps[i % 2] + '\n'
-                else:
-                    ret += role + ':'
-            return ret
-        elif self.sep_style == SeparatorStyle.DOLLY:
-            seps = [self.sep, self.sep2]
-            ret = system_prompt
-            for i, (role, message) in enumerate(self.messages):
-                if message:
-                    ret += role + ':\n' + message + seps[i % 2]
-                    if i % 2 == 1:
-                        ret += '\n\n'
-                else:
-                    ret += role + ':\n'
-            return ret
-        elif self.sep_style == SeparatorStyle.PHOENIX:
-            ret = system_prompt
-            for role, message in self.messages:
-                if message:
-                    ret += role + ': ' + '<s>' + message + '</s>'
-                else:
-                    ret += role + ': ' + '<s>'
-            return ret
-        elif self.sep_style == SeparatorStyle.ROBIN:
-            ret = system_prompt + self.sep
-            for role, message in self.messages:
-                if message:
-                    ret += role + ':\n' + message + self.sep
-                else:
-                    ret += role + ':\n'
-            return ret
-        elif self.sep_style == SeparatorStyle.FALCON_CHAT:
-            ret = ''
-            if self.system_message:
-                ret += system_prompt + self.sep
-            for role, message in self.messages:
-                if message:
-                    ret += role + ': ' + message + self.sep
-                else:
-                    ret += role + ':'
-
-            return ret
-        elif self.sep_style == SeparatorStyle.INTERNVL_ZH:
-            seps = [self.sep, self.sep2]
-            ret = self.system_message + seps[0]
-            for i, (role, message) in enumerate(self.messages):
-                if message:
-                    ret += role + ': ' + message + seps[i % 2]
-                else:
-                    ret += role + ':'
-            return ret
-        elif self.sep_style == SeparatorStyle.MPT:
-            ret = system_prompt + self.sep
-            for role, message in self.messages:
-                if message:
-                    if type(message) is tuple:
-                        message, _, _ = message
-                    ret += role + message + self.sep
-                else:
-                    ret += role
-            return ret
-        else:
-            raise ValueError(f'Invalid style: {self.sep_style}')
-
-    def set_system_message(self, system_message: str):
-        """Set the system message."""
-        self.system_message = system_message
-
-    def append_message(self, role: str, message: str):
-        """Append a new message."""
-        self.messages.append([role, message])
-
-    def update_last_message(self, message: str):
-        """Update the last output.
-
-        The last message is typically set to be None when constructing the prompt,
-        so we need to update it in-place after getting the response from a model.
-        """
-        self.messages[-1][1] = message
-
-    def to_gradio_chatbot(self):
-        """Convert the conversation to gradio chatbot format."""
-        ret = []
-        for i, (role, msg) in enumerate(self.messages[self.offset :]):
-            if i % 2 == 0:
-                ret.append([msg, None])
-            else:
-                ret[-1][-1] = msg
-        return ret
-
-    def to_openai_api_messages(self):
-        """Convert the conversation to OpenAI chat completion format."""
-        ret = [{'role': 'system', 'content': self.system_message}]
-
-        for i, (_, msg) in enumerate(self.messages[self.offset :]):
-            if i % 2 == 0:
-                ret.append({'role': 'user', 'content': msg})
-            else:
-                if msg is not None:
-                    ret.append({'role': 'assistant', 'content': msg})
-        return ret
-
-    def copy(self):
-        return Conversation(
-            name=self.name,
-            system_template=self.system_template,
-            system_message=self.system_message,
-            roles=self.roles,
-            messages=[[x, y] for x, y in self.messages],
-            offset=self.offset,
-            sep_style=self.sep_style,
-            sep=self.sep,
-            sep2=self.sep2,
-            stop_str=self.stop_str,
-            stop_token_ids=self.stop_token_ids,
-        )
-
-    def dict(self):
-        return {
-            'template_name': self.name,
-            'system_message': self.system_message,
-            'roles': self.roles,
-            'messages': self.messages,
-            'offset': self.offset,
-        }
-
-
-# A global registry for all conversation templates
-conv_templates: Dict[str, Conversation] = {}
-
-
-def register_conv_template(template: Conversation, override: bool = False):
-    """Register a new conversation template."""
-    if not override:
-        assert (
-            template.name not in conv_templates
-        ), f'{template.name} has been registered.'
-
-    conv_templates[template.name] = template
-
-
-def get_conv_template(name: str) -> Conversation:
-    """Get a conversation template."""
-    return conv_templates[name].copy()
-
-
-# Both Hermes-2 and internlm2-chat are chatml-format conversation templates. The difference
-# is that during training, the preprocessing function for the Hermes-2 template doesn't add
-# <s> at the beginning of the tokenized sequence, while the internlm2-chat template does.
-# Therefore, they are completely equivalent during inference.
-register_conv_template(
-    Conversation(
-        name='Hermes-2',
-        system_template='<|im_start|>system\n{system_message}',
-        # note: The new system prompt was not used here to avoid changes in benchmark performance.
-        # system_message='我是书生·万象，英文名是InternVL，是由上海人工智能实验室、清华大学及多家合作单位联合开发的多模态大语言模型。',
-        # system_message='你是由上海人工智能实验室联合商汤科技开发的书生多模态大模型，英文名叫InternVL, 是一个有用无害的人工智能助手。',
-        system_message='You are a Table Image to LaTeX/Markdown/HMTL Code converter.',
-        roles=('<|im_start|>user\n', '<|im_start|>assistant\n'),
-        sep_style=SeparatorStyle.MPT,
-        sep='<|im_end|>',
-        stop_token_ids=[
-            2,
-            6,
-            7,
-            8,
-        ],
-        stop_str='<|endoftext|>',
-    )
-)
-
-
-register_conv_template(
-    Conversation(
-        name='internlm2-chat',
-        system_template='<|im_start|>system\n{system_message}',
-        # note: The new system prompt was not used here to avoid changes in benchmark performance.
-        # system_message='我是书生·万象，英文名是InternVL，是由上海人工智能实验室、清华大学及多家合作单位联合开发的多模态大语言模型。',
-        system_message='你是由上海人工智能实验室联合商汤科技开发的书生多模态大模型，英文名叫InternVL, 是一个有用无害的人工智能助手。',
-        roles=('<|im_start|>user\n', '<|im_start|>assistant\n'),
-        sep_style=SeparatorStyle.MPT,
-        sep='<|im_end|>',
-        stop_token_ids=[
-            2,
-            92543,
-            92542
-        ]
-    )
-)
-
-
-register_conv_template(
-    Conversation(
-        name='phi3-chat',
-        system_template='<|system|>\n{system_message}',
-        # note: The new system prompt was not used here to avoid changes in benchmark performance.
-        # system_message='我是书生·万象，英文名是InternVL，是由上海人工智能实验室、清华大学及多家合作单位联合开发的多模态大语言模型。',
-        system_message='你是由上海人工智能实验室联合商汤科技开发的书生多模态大模型，英文名叫InternVL, 是一个有用无害的人工智能助手。',
-        roles=('<|user|>\n', '<|assistant|>\n'),
-        sep_style=SeparatorStyle.MPT,
-        sep='<|end|>',
-        stop_token_ids=[
-            2,
-            32000,
-            32007
-        ]
-    )
-)