bella-openapi 1.0.2.1__py3-none-any.whl → 1.0.2.3__py3-none-any.whl

bella_openapi/__init__.py

@@ -3,6 +3,8 @@ from .log import operation_log, submit_log
 from .openapi_contexvar import trace_id_context, caller_id_context, request_url_context
 from .auth_billing import ErrorInfo, async_authenticate_decorator_args, authenticate_user, print_context, \
 get_context, set_context, clean_context, report
+from .domtree import StandardDomTree, StandardNode
+
 __all__ = ["validate_token", "operation_log",
            "support_model",
            "account_balance_enough",
@@ -18,5 +20,7 @@ __all__ = ["validate_token", "operation_log",
            "get_context",
            "set_context",
            "clean_context",
-           "report"
+           "report",
+           "StandardDomTree",
+           "StandardNode"
            ]

bella_openapi/domtree/__init__.py

@@ -0,0 +1,3 @@
+from .standard_domtree import StandardDomTree, StandardNode
+
+__all__ = ["StandardDomTree", "StandardNode"]

bella_openapi/domtree/standard_domtree.py

@@ -0,0 +1,492 @@
+"""
+StandardDomTree - 新的DOM树协议定义
+
+这个模块定义了新的标准化DOM树结构，用于统一文档解析结果的格式。
+目前解析过程仍基于原有的DOM树结构，然后通过转换方法将其转换为标准格式。
+
+注意：当前解析器仍使用原有的DOM树结构，后续计划将直接解析为StandardDomTree格式。
+
+"""
+
+from typing import List, Optional, Literal, Any, Union
+from pydantic import BaseModel, Field
+
+from bella_openapi.domtree.utils import count_tokens
+
+# 布局类型映射表：将旧解析器的布局类型映射到新的标准类型
+# 注意：这个映射表是临时的，后续计划统一布局类型定义
+layout_type_mapping = {
+    "Catalog": "Catalog",
+    "Title": "Title",
+    "List": "ListItem",
+    "Formula": "Formula",
+    "Code": "Code",
+    "Text": "Text",
+
+    "Figure": "Figure",
+    "FigureName": "FigureName",
+    "FigureNote": "Text",  # 目前实际解析出来没有
+
+    "Table": "Table",
+    "TableName": "TableName",
+    "TableNote": "Text",  # 目前实际解析出来没有
+}
+
+
+class SourceFile(BaseModel):
+    id: str  # 文件ID，唯一标识符，类型为string
+    name: str  # 文件名，文档的名称，类型为string
+    type: Optional[str]  # 文件类型，例如：pdf、docx等
+    mime_type: Optional[str]  # MIME类型，例如：application/pdf、application/msword等
+
+
+class StandardPosition(BaseModel):
+    bbox: List[float]  # 文档中的矩形坐标信息，例如：[90.1,263.8,101.8,274.3]
+    page: int  # 页码
+
+
+class StandardImage(BaseModel):
+    type: Literal["image_url", "image_base64", "image_file"]  # 图片类型约束
+    url: Optional[str] = None  # 链接地址
+    base64: Optional[str] = None  # 图片base64编码
+    file_id: Optional[str] = None  # 上传到file-api的文件ID
+
+
+class Cell(BaseModel):
+    """表格单元格"""
+    path: Optional[List[Union[int, List[int]]]] = Field(default_factory=list)  # 单元格路径
+    text: Optional[str] = None  # 文本内容
+    nodes: Optional[List['StandardNode']] = Field(default_factory=list)
+
+
+class StandardRow(BaseModel):
+    """表格行"""
+    cells: List[Cell] = Field(default_factory=list)  # 单元格列表
+
+
+class StandardBaseElement(BaseModel):
+    type: str  # 类型，例如["Text","Title","List","Catalog","Table","Figure","Formula","Code","ListItem"]
+    positions: List[StandardPosition]  # 位置信息，可能跨页所以是个数组
+
+
+class StandardElement(StandardBaseElement):
+    text: Optional[str] = None  # 文本信息，图片ocr的文字
+
+
+class StandardTableElement(StandardBaseElement):
+    name: Optional[str] = None  # 如果类型是Table、Figure为其名字
+    description: Optional[str] = None  # 如果类型是Table、Figure为其描述
+    rows: List[StandardRow] = Field(default_factory=list)  # 表格行
+
+
+class StandardImageElement(StandardElement):
+    name: Optional[str] = None  # 如果类型是Table、Figure为其名字
+    description: Optional[str] = None  # 如果类型是Table、Figure为其描述
+    image: Optional[StandardImage] = None  # 图片信息
+
+
+class StandardNode(BaseModel):
+    source_file: Optional[SourceFile] = None  # 文档来源，表示文档的来源信息
+    summary: Optional[str] = None  # 摘要，文档的简要概述
+    tokens: Optional[int] = None  # token预估数量，文档中的token数量估计
+    path: Optional[List[int]] = Field(default_factory=list)  # 编号的层级信息，例如：1.2.1
+    element: Optional[Union[StandardElement, StandardImageElement, StandardTableElement]] = None  # 元素信息，当前节点的元素详情
+    children: Optional[List["StandardNode"]] = Field(default_factory=list)  # 子节点信息，当前节点的所有子节点
+
+
+class StandardDomTree(BaseModel):
+    root: StandardNode  # 根节点
+
+    def to_markdown(self) -> str:
+        """
+        将StandardDomTree转换为Markdown格式的字符串
+
+        Returns:
+            str: Markdown格式的字符串
+        """
+        markdown_res = ""
+
+        def _generate_markdown(node: StandardNode, level: int, low_than_text: int = 0) -> str:
+            nonlocal markdown_res
+            child_low_than_text = 0
+
+            if node.element:
+                # 根据不同的type生成Markdown
+                if node.element.type == "Figure" and isinstance(node.element, StandardImageElement):
+                    # 添加图片名称
+                    if node.element.name:
+                        markdown_res += f"**{node.element.name}**\n\n"
+                    if node.element.image and node.element.image.url:
+                        markdown_res += f"![Figure]({node.element.image.url})\n\n"
+                    if node.element.text:
+                        md_ocr_res = self._convert_to_markdown_quote(node.element.text)
+                        markdown_res += f"{md_ocr_res}\n\n"
+                    # 添加图片描述
+                    if node.element.description:
+                        markdown_res += f"*{node.element.description}*\n\n"
+
+                elif node.element.type == "Table" and isinstance(node.element, StandardTableElement):
+                    # 添加表格名称
+                    if node.element.name:
+                        markdown_res += f"**{node.element.name}**\n\n"
+                    table_md = self._list_to_html_table(node.element.rows)
+                    markdown_res += f"{table_md}\n\n"
+                    # 添加表格描述
+                    if node.element.description:
+                        markdown_res += f"*{node.element.description}*\n\n"
+
+                elif (level <= 6  # 标题必须小于等于6级
+                      and (node.element.type in ["Title"]  # 认定为Title 或者 父节点非text的ListItem
+                           or (node.element.type in ["ListItem"] and not low_than_text))):
+                    # Title只能识别6级，大于6级的按普通文本处理
+                    markdown_res += '#' * level + f" {node.element.text}\n\n"
+                elif node.element.type in ["Title"]:
+                    markdown_res += f"{node.element.text}\n\n"
+                elif node.element.type in ["Text"]:
+                    markdown_res += f"{node.element.text}\n\n"
+                    child_low_than_text = low_than_text + 1  # Text节点的子节点标记
+                elif node.element.type in ["ListItem"]:
+                    markdown_res += '\t' * (low_than_text - 1) + f"- {node.element.text}\n\n"
+                # Formula、Catalog、Code等元素的处理
+                else:
+                    markdown_res += f"{node.element.text}\n\n"
+
+            for child in node.children:
+                _generate_markdown(child, level + 1, child_low_than_text)
+
+        # 从根节点开始生成，跳过根节点本身
+        for child in self.root.children:
+            _generate_markdown(child, 1, 0)
+
+        return markdown_res
+
+    def _convert_to_markdown_quote(self, text: str) -> str:
+        """将文本转换为markdown引用格式"""
+        if not text:
+            return ""
+        lines = text.split('\n')
+        quoted_lines = ['> ' + line for line in lines]
+        return '\n'.join(quoted_lines)
+
+    def _list_to_html_table(self, rows: List[StandardRow]) -> str:
+        """将表格行转换为HTML表格"""
+        if not rows:
+            return ""
+
+        html_text = "<table>"
+        for row in rows:
+            html_text += "<tr>"
+            for cell in row.cells:
+                # 从path中提取rowspan和colspan信息
+                if len(cell.path) >= 4:
+                    start_row, end_row, start_col, end_col = cell.path[:4]
+                    rowspan = end_row - start_row + 1
+                    colspan = end_col - start_col + 1
+                else:
+                    rowspan = colspan = 1
+
+                cell_text = cell.text or ""
+                html_text += f"<td rowspan='{rowspan}' colspan='{colspan}'>{cell_text}</td>"
+            html_text += "</tr>"
+        html_text += "</table>"
+        return html_text
+
+    @classmethod
+    def from_domtree_dict(cls, domtree: dict, file_info):
+        """
+        将旧的DOM树字典格式转换为新的StandardDomTree格式
+
+        注意：这是临时的转换方法，用于兼容现有的解析器输出。
+        后续计划将解析器直接输出StandardDomTree格式，避免这个转换步骤。
+
+        Args:
+            domtree: 源DOM树字典对象（旧格式）
+            file_info: 文件信息字典，包含文件ID、文件名等信息
+        Returns:
+            StandardDomTree: 转换后的标准化DOM树对象
+        """
+        # 创建 SourceFile 对象
+        source_file = None
+        if file_info:
+            source_file = SourceFile(
+                id=file_info['id'],
+                name=file_info['filename'],
+                type=file_info.get('type'),
+                mime_type=file_info.get('mime_type')
+            )
+
+        # 转换根节点，构建树结构（不计算path）
+        standard_root = cls._from_domtree_nodes(domtree.get('root'), source_file)
+
+        return cls(root=standard_root)
+
+    @classmethod
+    def _from_domtree_nodes(cls, node: dict, source_file: SourceFile) -> StandardNode:
+        """
+        处理所有节点
+        """
+        # 根节点，创建一个空的 StandardNode
+        standard_root_node = StandardNode(
+            source_file=source_file,
+            summary="",
+            tokens=0,  # 先设置为 0，后面再计算
+            path=None,
+            element=None,
+            children=[]
+        )
+
+        # 递归处理子节点
+        for child in node.get('child', []):
+            standard_child = cls._from_domtree_node_to_base_info(child)  # 不传递path参数
+            if standard_child:  # 确保子节点不为 None
+                standard_root_node.children.append(standard_child)
+
+        # 处理 FigureName 和 TableName 节点（合并节点）
+        cls._process_special_nodes(standard_root_node)
+
+        # 计算所有节点的path
+        cls._calculate_paths(standard_root_node)
+
+        # 计算 token 数量：子节点 token 数量之和
+        tokens = 0
+        for child in standard_root_node.children:
+            tokens += child.tokens
+
+        # 设置 token 数量
+        standard_root_node.tokens = tokens
+
+        return standard_root_node
+
+    @classmethod
+    def _calculate_paths(cls, node: StandardNode, parent_path: List[int] = None):
+        """
+        计算所有节点的path
+
+        Args:
+            node: 当前处理的节点
+            parent_path: 父节点的path
+        """
+        if parent_path is None:
+            parent_path = []
+
+        # 为子节点计算path
+        for i, child in enumerate(node.children, start=1):
+            child_path = parent_path + [i]
+            child.path = child_path
+
+            # 递归计算子节点的path
+            cls._calculate_paths(child, child_path)
+
+    @classmethod
+    def _process_special_nodes(cls, node: StandardNode):
+        """
+        处理特殊节点（FigureName 和 TableName）
+
+        注意：这是临时的处理逻辑，用于处理旧解析器输出的特殊节点类型。
+        后续计划将解析器直接输出正确的节点结构，避免这种后处理。
+
+        Args:
+            node: 当前处理的节点
+        """
+        if not node or not node.children:
+            return
+
+        # 创建一个新的子节点列表，用于存储处理后的子节点
+        new_children = []
+        i = 0
+
+        while i < len(node.children):
+            current = node.children[i]
+
+            # 检查当前节点是否为 FigureName 或 TableName
+            if current.element and current.element.type in ['FigureName', 'TableName']:
+                target_type = 'Figure' if current.element.type == 'FigureName' else 'Table'
+                merged = False
+
+                # 检查前一个节点
+                if i > 0:
+                    prev_sibling = node.children[i - 1]
+                    if prev_sibling.element and prev_sibling.element.type == target_type:
+                        # 找到对应类型的前一个兄弟节点，合并节点
+                        if cls._merge_nodes(prev_sibling, current, target_type):
+                            merged = True
+
+                # 如果没有与前一个节点合并，检查后一个节点
+                if not merged and i < len(node.children) - 1:
+                    next_sibling = node.children[i + 1]
+                    if next_sibling.element and next_sibling.element.type == target_type:
+                        # 找到对应类型的后一个兄弟节点，合并节点
+                        if cls._merge_nodes(next_sibling, current, target_type):
+                            merged = True
+
+                # 如果没有找到对应类型的兄弟节点，将当前节点类型改为 Text
+                if not merged:
+                    current.element.type = 'text'
+                    new_children.append(current)
+            else:
+                new_children.append(current)
+
+            i += 1
+
+        # 更新子节点列表
+        node.children = new_children
+
+        # 递归处理子节点
+        for child in node.children:
+            cls._process_special_nodes(child)
+
+    @classmethod
+    def _merge_nodes(cls, target_node: StandardNode, source_node: StandardNode, node_type: str) -> bool:
+        """
+        合并两个节点，将source_node的信息合并到target_node中
+
+        Args:
+            target_node: 目标节点（Figure或Table节点）
+            source_node: 源节点（FigureName或TableName节点）
+            node_type: 节点类型（'Figure'或'Table'）
+
+        Returns:
+            bool: 是否成功合并
+        """
+        if node_type == 'Figure' and isinstance(target_node.element, StandardImageElement):
+            # 将 FigureName 的文本作为 Figure 的 name
+            target_node.element.name = source_node.element.text
+            # 更新 tokens 计数
+            target_node.tokens += source_node.tokens
+            # 将 FigureName 的位置添加到 Figure 中
+            target_node.element.positions += source_node.element.positions
+            return True
+        elif node_type == 'Table' and isinstance(target_node.element, StandardTableElement):
+            # 将 TableName 的文本作为 Table 的 name
+            target_node.element.name = source_node.element.text
+            # 更新 tokens 计数
+            target_node.tokens += source_node.tokens
+            # 将 Table 的位置添加到 Figure 中
+            target_node.element.positions += source_node.element.positions
+            return True
+        return False
+
+    @classmethod
+    def _from_domtree_node_to_base_info(cls, node: dict) -> Optional[StandardNode]:
+        """
+        将单个旧格式Node转换为StandardNode
+
+        注意：这是临时的转换方法，用于处理旧解析器输出的节点格式。
+        后续计划将解析器直接输出StandardNode格式，避免这种转换。
+
+        Args:
+            node: 源Node对象（旧格式字典）
+        Returns:
+            StandardNode: 转换后的标准化节点对象
+        """
+        if not node:
+            return
+
+        element = node['element']
+
+        text = ""
+        # 映射的类型
+        element_type = layout_type_mapping.get(element['layout_type'], "Text")  # 默认类型为 text
+        positions = [StandardPosition(bbox=element['bbox'], page=element['page_num'][0])]  # 位置列表，目前page_num元素个数只会是1个
+
+        standard_node = None
+        if element_type == "Figure":
+            # 处理图片信息
+            image = None
+            text = element.get('image_ocr_result', '')
+            if 'image_link' in element and element['image_link']:
+                image = StandardImage(
+                    type="image_url",
+                    url=element['image_link']
+                )
+
+            # 创建 StandardNode
+            standard_node = StandardNode(
+                summary="",
+                tokens=0,  # 先设置为 0，后面再计算
+                path=[],  # 初始化为空列表，后续再计算
+                element=StandardImageElement(
+                    type=element_type,
+                    positions=positions,
+                    name="",
+                    description="",
+                    text=text,
+                    image=image,
+                ),
+                children=[]
+            )
+        elif element_type == "Table":
+            rows = []
+            cell_texts = []  # 收集所有单元格的文本，用于计算 token 数量
+            if 'rows' in element:
+                for row_data in element['rows']:
+                    cells = []
+                    if 'cells' in row_data:
+                        for cell_data in row_data['cells']:
+                            cell_text = cell_data.get('text', '')
+                            cell_texts.append(cell_text)
+                            # 不计算cell_path，后续再计算
+                            cell = Cell(
+                                path=[cell_data['start_row'], cell_data['end_row'], cell_data['start_col'],
+                                      cell_data['end_col']],
+                                text=cell_text,
+                                # 目前只会有一个元素,且是Text类型，Path重新从头编号，相对cell是root
+                                nodes=[StandardNode(summary="", tokens=count_tokens(cell_text), path=[1], children=[],
+                                                    element=StandardElement(
+                                                        type='Text',
+                                                        positions=[],
+                                                        text=cell_text
+                                                    )
+                                                    )]
+                            )
+                            cells.append(cell)
+                    # 使用 StandardRow 的构造函数创建行
+                    row = StandardRow(cells=cells)
+                    rows.append(row)
+
+            # 将所有单元格的文本合并，用于计算 token 数量
+            text = " ".join(cell_texts)
+
+            standard_node = StandardNode(
+                summary="",
+                tokens=0,  # 先设置为 0，后面再计算
+                path=[],  # 初始化为空列表，后续再计算
+                element=StandardTableElement(
+                    type=element_type,
+                    positions=positions,
+                    name="",
+                    description="",
+                    rows=rows
+                ),
+                children=[]
+            )
+        else:
+            text = element.get('text', '')
+            standard_node = StandardNode(
+                summary="",
+                tokens=0,  # 先设置为 0，后面再计算
+                path=[],  # 初始化为空列表，后续再计算
+                element=StandardElement(
+                    type=element_type,
+                    positions=positions,
+                    text=text
+                ),
+                children=[]
+            )
+
+        # 递归处理子节点
+        if 'child' in node:
+            for child in node['child']:
+                standard_child = cls._from_domtree_node_to_base_info(child)
+                if standard_child:  # 确保子节点不为 None
+                    standard_node.children.append(standard_child)
+
+        # 计算 token 数量：自身 text 的 token 数量 + 子节点 token 数量
+        tokens = count_tokens(text)
+        for child in standard_node.children:
+            tokens += child.tokens
+
+        # 设置 token 数量
+        standard_node.tokens = tokens
+
+        return standard_node

bella_openapi/domtree/utils.py

@@ -0,0 +1,12 @@
+import tiktoken
+
+# 自研模型均用gpt-4计算（可能有误差，可忽略）
+def count_tokens(text: str, model: str = "gpt-4") -> int:
+    if not text:
+        return 0
+    encoding = tiktoken.encoding_for_model(model)
+    tokens = encoding.encode(text)
+    # 计算标记列表的长度，即标记的数量
+    token_count = len(tokens)
+    # 返回标记的数量
+    return token_count

{bella_openapi-1.0.2.1.dist-info → bella_openapi-1.0.2.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bella-openapi
-Version: 1.0.2.1
+Version: 1.0.2.3
 Summary: client for openapi service.
 Home-page: 
 Author: ['tangxiaolong', 'fanqiangwei', 'zhangxiaojia', 'liumin', 'wangyukun']
@@ -11,6 +11,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: httpx<=0.26.0,>=0.10.0
 Requires-Dist: Werkzeug==3.0.1
+Requires-Dist: tiktoken>=0.5.0
 Dynamic: author
 Dynamic: classifier
 Dynamic: description

{bella_openapi-1.0.2.1.dist-info → bella_openapi-1.0.2.3.dist-info}/RECORD

@@ -1,4 +1,4 @@
-bella_openapi/__init__.py,sha256=cTyozxQ6fo0FSPwfTJWn7DD-CyPtfKYCMuSFGqddnZA,897
+bella_openapi/__init__.py,sha256=NPsPxysa625xRIKgL6je4q1ZSJfnVar4Mb-2RljNwfs,1006
 bella_openapi/auth_billing.py,sha256=Hn0KS8GuG48etnvnd1Faej4IfFXD3tjzalUzDnpZh7Q,3520
 bella_openapi/authorize.py,sha256=cO6J-wx9dmmkDAeqpXT7QlyCr13hO-HSC5SWQSw2gZw,2150
 bella_openapi/config.py,sha256=Dn8vnToDaOesPGboauxCCwNrW5awQLeSkmDjNjXS4bQ,319
@@ -13,10 +13,13 @@ bella_openapi/bella_trace/record_log.py,sha256=qSZXp_VTzIzMVlQNnZKLIbmyGvggFSQL5
 bella_openapi/bella_trace/trace_requests.py,sha256=ADA8J_gbC3TwUo5LWQ3c_yTmCSZRaWzq1FC0iUOnst0,1370
 bella_openapi/console/__init__.py,sha256=uSfr5v6JLRSqTlftjK_ZU1pnbkEyxAPbuQbMyYX_phk,64
 bella_openapi/console/models.py,sha256=Hh1UuYHIxFtF9r5QK-pSJPFrSqbZUHv6spLvPbCeX08,1274
+bella_openapi/domtree/__init__.py,sha256=qxFd6d9WZ4ThWQAFtHAhSseo2h2FB2-5KZWyGOooAbo,107
+bella_openapi/domtree/standard_domtree.py,sha256=lIJP2gReyF2RgAxtyncSB6xl_XAzKCpOb-FTNaf-MIw,19952
+bella_openapi/domtree/utils.py,sha256=-ItZYh9Gj8QyOkZzjCC5xWPYU-FkzJllGC0oUO21Kp4,394
 bella_openapi/middleware/__init__.py,sha256=XWvZG1xO30ZXIn10YVYthmT1BV-9fonMEP_jVRZbAlQ,157
 bella_openapi/middleware/context_middleware.py,sha256=YawQyKAxMzvlDs_MxcuQKh90pP6VoMKzCBDS94qmlzQ,3870
-bella_openapi-1.0.2.1.dist-info/licenses/LICENSE,sha256=O-0zMbcEi6wXz1DiSdVgzMlQjJcNqNe5KDv08uYzqR0,1055
-bella_openapi-1.0.2.1.dist-info/METADATA,sha256=8_UzDGUg5KiWPERvi8bnPWsxpx0GDBBox3uxfgF01D4,9346
-bella_openapi-1.0.2.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-bella_openapi-1.0.2.1.dist-info/top_level.txt,sha256=EZuq3F6tKeF-vmZQi6_S2XzmES7SPW7HAbGN1Uv9vN8,14
-bella_openapi-1.0.2.1.dist-info/RECORD,,
+bella_openapi-1.0.2.3.dist-info/licenses/LICENSE,sha256=O-0zMbcEi6wXz1DiSdVgzMlQjJcNqNe5KDv08uYzqR0,1055
+bella_openapi-1.0.2.3.dist-info/METADATA,sha256=PbC6CexkWl5ZOR1_0dXPtD6HpydmJ9H_TsOMRH0Kp_E,9377
+bella_openapi-1.0.2.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+bella_openapi-1.0.2.3.dist-info/top_level.txt,sha256=EZuq3F6tKeF-vmZQi6_S2XzmES7SPW7HAbGN1Uv9vN8,14
+bella_openapi-1.0.2.3.dist-info/RECORD,,