PyPI - omni-split - Versions diffs - 0.0.3__py3-none-any.whl - Mend

omni-split 0.0.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (23) hide show

omni_split/__init__.py +16 -0
omni_split/base/__init__.py +0 -0
omni_split/base/chonkie_base.py +139 -0
omni_split/base/chonkie_tokenizer.py +285 -0
omni_split/base/chonkie_types.py +519 -0
omni_split/base/md2json_list.py +303 -0
omni_split/base/md_json_list2chunk.py +348 -0
omni_split/main.py +73 -0
omni_split/model/text_chunker_tokenizer/qwen_tokenizer.json +303282 -0
omni_split/omni_split.py +93 -0
omni_split/sub_chunker/__init__.py +0 -0
omni_split/sub_chunker/document_split.py +32 -0
omni_split/sub_chunker/markdown_split.py +47 -0
omni_split/sub_chunker/text_split.py +343 -0
omni_split/test.py +80 -0
omni_split/utils/__init__.py +0 -0
omni_split/utils/base_utils.py +181 -0
omni_split/utils/download_test_doc.py +61 -0
omni_split-0.0.3.dist-info/METADATA +147 -0
omni_split-0.0.3.dist-info/RECORD +23 -0
omni_split-0.0.3.dist-info/WHEEL +5 -0
omni_split-0.0.3.dist-info/licenses/LICENSE +21 -0
omni_split-0.0.3.dist-info/top_level.txt +1 -0

omni_split/base/md2json_list.py ADDED Viewed

@@ -0,0 +1,303 @@
+from mistletoe import Document
+from mistletoe.block_token import Heading, Paragraph, BlockCode, List, ListItem, Quote, Table, TableRow, TableCell, ThematicBreak, CodeFence
+from mistletoe.span_token import Strong, Emphasis, Link, Image, RawText
+import json
+import re
+# #     {
+#         "type": "text",
+#         "text": "Game-theoretic Workflow in this Paper  In this paper, we integrate game-theoretic principles into. the reasoning processes of LLMs prior to decision-making. By guiding the models to derive rational. strategies and make decisions based on these strategies, we aim to enhance their ability to perform effectively in strategic settings. ",
+#         "page_idx": 4
+#     },
+# {
+#     "type": "text",
+#     "text": "Game-theoretic LLM: Agent Workflow for Negotiation Games ",
+#     "text_level": 1,
+#     "page_idx": 0
+# },
+# {
+#     "type": "image",
+#     "img_path": "images/9d516d401a6bdc168c3000baaf9f12a86aac5e2b07edb16344678c15119aebb1.jpg",
+#     "img_caption": [
+#         "Figure 1: Game-theoretic Landscape Investigated in this Paper. "
+#     ],
+#     "img_footnote": [],
+#     "page_idx": 3
+# },
+# {
+#     "type": "table",
+#     "img_path": "images/f20e0b1b16500e94a492a40b553e7e3ae8ffe66d69de3c9252ee11b32982775b.jpg",
+#     "table_caption": [
+#         "Table 3b: Payoff matrix for Wait-Go Game "
+#     ],
+#     "table_footnote": [],
+#     "table_body": "\n\n<html><body><table><tr><td></td><td>Wait</td><td>Go</td></tr><tr><td>Wait</td><td>0,0</td><td>0, 2</td></tr><tr><td>Go</td><td>2,0</td><td>-4,-4</td></tr></table></body></html>\n\n",
+#     "page_idx": 7
+# },
+def is_markdown_equal(md_str):
+    md_str = md_str.strip()  # 去除前后空白
+    # 检查是否以 $$ 开头和结尾，并且中间没有 $$
+    return bool(re.fullmatch(r"^\$\$(.*?)\$\$", md_str, re.DOTALL))
+def is_markdown_table(md_str):
+    md_str = md_str.strip()
+    if md_str.startswith("<html><body><table>") and md_str.endswith("</table></body></html>"):
+        return True
+    lines = [line.strip() for line in md_str.split("\n") if line.strip()]
+    if not lines:
+        return False
+    # 检查所有行是否至少包含一个普通 |（非转义）
+    for line in lines:
+        if "|" not in line:
+            return False
+    # 单行内容不视为表格（除非是严格的单行表格，但通常 Markdown 表格需要分隔线）
+    if len(lines) == 1:
+        return False  # 直接返回 False，避免误判 LaTeX 等
+    # 多行表格：检查第二行是否是分隔线
+    separator_line = lines[1]
+    separator_parts = [part.strip() for part in separator_line.split("|") if part.strip()]
+    for part in separator_parts:
+        if not re.fullmatch(r"^:?-+:?$", part):
+            return False
+    return True  # 只有符合所有条件才返回 True
+def split_image_url_func(text):
+    parts = re.split(r"(!\[\]\([^)]+\))", text.strip('"'))
+    parts = [p for p in parts if p]
+    ret_parts = []
+    for item in parts:
+        if item != "":
+            ret_parts.append(item)
+    return ret_parts
+def md2json_list_func(md_content):
+    # 解析Markdown为AST
+    doc = Document(md_content)
+    result = []
+    # 遍历AST中的每个节点
+    for child in doc.children:
+        # 处理标题
+        if isinstance(child, Heading):
+            level = child.level
+            content = get_inline_md(child.children) if hasattr(child, "children") and child.children else ""
+            result.append({"text": content, "type": "text", "text_level": level, "page_idx": None})
+        # 处理段落
+        elif isinstance(child, Paragraph):
+            # 检查是否是独立的图片
+            ## todo
+            if hasattr(child, "children") and child.children and len(child.children) == 1 and isinstance(child.children[0], Image):
+                img = child.children[0]
+                result.append(
+                    {
+                        "type": "image",
+                        "img_path": img.src,
+                        "img_caption": [img.title if hasattr(img, "title") else ""],
+                        "img_footnote": [img.title if hasattr(img, "title") else ""],
+                        "page_idx": None,
+                    },
+                )
+                # result.append({"content": {"src": img.src if hasattr(img, "src") else "", "title": img.title if hasattr(img, "title") else "", "description": img.title if hasattr(img, "title") else ""}, "type": "image"})
+            else:
+                content = get_inline_md(child.children) if hasattr(child, "children") and child.children else ""
+                split_content_list = split_image_url_func(content)
+                for split_content in split_content_list:
+                    if split_content.startswith("![") and split_content.endswith(")"):
+                        ##note: 处理图片
+                        temp = {
+                            "type": "image",
+                            "img_path": split_content[4:-1],
+                            "img_caption": [""],
+                            "img_footnote": [""],
+                            "page_idx": None,
+                        }
+                    elif is_markdown_table(split_content):
+                        ##note:  处理表格
+                        ## 处理表格
+                        temp = {
+                            "type": "table",
+                            "img_path": None,
+                            "table_caption": [""],
+                            "table_footnote": [""],
+                            "table_body": f"{split_content}\n",
+                            "page_idx": None,
+                        }
+                    elif is_markdown_equal(split_content):
+                        ##note:  处理单行公式(非行内公式)
+                        temp = {
+                            "type": "equation",
+                            "text": f"{split_content}\n",
+                            "text_format": "latex",
+                            "page_idx": None,
+                        }
+                    else:
+                        ##note: 其他的按照标准text对待.
+                        temp = {
+                            "text": f"{split_content}\n",
+                            "type": "text",
+                            "page_idx": None,
+                        }
+                    result.append(temp)
+        # 处理代码块
+        elif isinstance(child, (BlockCode, CodeFence)):
+            if isinstance(child, BlockCode):
+                # BlockCode（缩进代码块）：手动添加 ``` 标记
+                code_content = child.children[0].content if hasattr(child, "children") and child.children else ""
+                language = ""
+                code_content = f"```\n{code_content}\n```"  # 添加 ``` 围栏
+            else:
+                # CodeFence（围栏代码块）：保留原始 ```language\ncontent\n```
+                code_content = child.children[0].content if hasattr(child, "children") and child.children else ""
+                language = getattr(child, "language", "")
+                fence_char = getattr(child, "fence_char", "`")  # 可能是 ` 或 ~
+                fence_length = getattr(child, "fence_length", 3)  # 通常是 3
+                fence = fence_char * fence_length
+                code_content = f"{fence}{language}\n{code_content}\n{fence}"
+            result.append({"content": code_content, "type": "code", "language": language})
+        # 处理列表
+        elif isinstance(child, List):
+            items = []
+            if hasattr(child, "children") and child.children:
+                for item in child.children:
+                    if isinstance(item, ListItem):
+                        item_content = get_inline_md(item.children) if hasattr(item, "children") and item.children else ""
+                        items.append(item_content)
+            # result.append({"content": items, "type": "list", "ordered": child.start is not None if hasattr(child, "start") else False})
+            temp_str = ""
+            for item in items:
+                temp_str += "- " + item + "\n"
+            temp = {
+                "text": temp_str,
+                "type": "text",
+                "page_idx": None,
+            }
+            result.append(temp)
+        # 处理引用
+        elif isinstance(child, Quote):
+            content = get_inline_md(child.children) if hasattr(child, "children") and child.children else ""
+            temp = {
+                "text": str(content)+"\n",
+                "type": "text",
+                "page_idx": None,
+            }
+            result.append(temp)
+            # result.append({"content": content, "type": "text", "page_idx": None})
+        # 处理表格
+        elif isinstance(child, Table):
+            table_content = ""
+            if hasattr(child, "children") and child.children:
+                # 重建表格的Markdown表示
+                if hasattr(child, "header") and child.header:
+                    header_row = child.header
+                    if hasattr(header_row, "children") and header_row.children:
+                        table_content += "| " + " | ".join(get_inline_md(cell.children) if hasattr(cell, "children") else "" for cell in header_row.children if isinstance(cell, TableCell)) + " |\n"
+                        table_content += "| " + " | ".join(["---"] * len(header_row.children)) + " |\n"
+                for row in child.children[1:] if hasattr(child, "children") else []:
+                    if isinstance(row, TableRow) and hasattr(row, "children") and row.children:
+                        table_content += "| " + " | ".join(get_inline_md(cell.children) if hasattr(cell, "children") else "" for cell in row.children if isinstance(cell, TableCell)) + " |\n"
+            result.append(
+                {
+                    "type": "table",
+                    "img_path": None,
+                    "table_caption": [""],
+                    "table_footnote": [""],
+                    "table_body": table_content.strip(),
+                    "page_idx": None,
+                }
+            )
+        # 处理分隔线
+        elif isinstance(child, ThematicBreak):
+            result.append(
+                {
+                    "content": "---\n",
+                    "type": "text",
+                    "page_idx": None,
+                }
+            )
+        # 处理数学公式块
+        elif is_math_block(child):
+            content = child.children[0].content.strip() if hasattr(child, "children") and child.children and len(child.children) > 0 else ""
+            result.append(
+                {
+                    "type": "equation",
+                    "text": content,
+                    "text_format": "latex",
+                    "page_idx": None,
+                }
+            )
+    return result
+def get_inline_md(tokens):
+    """获取行内元素的Markdown表示"""
+    if not tokens:
+        return ""
+    md = ""
+    for token in tokens:
+        if isinstance(token, RawText):
+            md += token.content if hasattr(token, "content") else ""
+        elif isinstance(token, Strong):
+            md += f"**{get_inline_md(token.children) if hasattr(token, 'children') else ''}**"
+        elif isinstance(token, Emphasis):
+            md += f"*{get_inline_md(token.children) if hasattr(token, 'children') else ''}*"
+        elif isinstance(token, Link):
+            md += f"[{get_inline_md(token.children) if hasattr(token, 'children') else ''}]({token.target if hasattr(token, 'target') else ''})"
+        elif isinstance(token, Image):
+            md += f"![{token.title if hasattr(token, 'title') else ''}]({token.src if hasattr(token, 'src') else ''})"
+        elif is_math_inline(token):
+            content = token.children[0].content if hasattr(token, "children") and token.children and len(token.children) > 0 else ""
+            md += f"${content}$"
+        elif hasattr(token, "children"):
+            md += get_inline_md(token.children) if token.children else ""
+    return md
+def is_math_block(token):
+    """检查是否是数学公式块"""
+    if isinstance(token, Paragraph) and hasattr(token, "children") and token.children and len(token.children) == 1:
+        content = token.children[0].content.strip() if hasattr(token.children[0], "content") else ""
+        return isinstance(token.children[0], RawText) and content.startswith("$$") and content.endswith("$$")
+    return False
+def is_math_inline(token):
+    """检查是否是行内数学公式"""
+    if isinstance(token, RawText):
+        content = token.content.strip() if hasattr(token, "content") else ""
+        return content.startswith("$") and content.endswith("$") and len(content) > 1
+    return False
+if __name__ == "__main__":
+    print("main function invoke")
+    with open("./test/c8d4614affc19ba92d7ba0671fd709803d0488a0c5a68bc237783a8af39fe32e/1c7fbb26-1012-4b03-894c-69ab2257985c_1743677710.4311144.md", "r") as f:
+        md_content = f.read()
+    json_list = md2json_list_func(md_content)
+    print(json.dumps(json_list, indent=4, ensure_ascii=False))

omni_split/base/md_json_list2chunk.py ADDED Viewed

@@ -0,0 +1,348 @@
+import json
+import re
+# 设置文本长度限制
+# 定义假性公式的标志
+# 定义占位符前缀
+element_placeholder = {"equation": "[EQUATION]", "image": "[IMAGE]", "table": "[TABLE]"}
+PSEUDO_EQUATION_FLAG = "[PSEUDO_EQUATION]"
+# 用于保存占位符映射内容，格式：(placeholder, element_type, content)
+# ---- 核心功能函数 ----
+def count_words(text, placeholder_size_map=None):
+    ## todo 使用tokenizer来计算字数
+    """更精确的字数统计方法（接近Word统计规则）"""
+    # 统计中文字符（含中文标点）
+    chinese_count = len(re.findall(r"[\u4e00-\u9fa5\u3000-\u303f\uff00-\uffef]", text))
+    # 统计英文单词（字母数字组合）
+    english_words = re.findall(r"\b[a-zA-Z0-9]+\b", text)
+    # 统计特殊符号（非中文、非字母数字的独立字符）
+    special_chars = re.findall(r"(?<!\S)[^\w\u4e00-\u9fa5](?!\S)", text)
+    base_count = chinese_count + len(english_words) + len(special_chars)
+    if placeholder_size_map:
+        # 查找文本中的占位符并累加其实际长度
+        # 占位符格式: [TYPE]_index, e.g., [TABLE]_0, [IMAGE]_1
+        placeholders = re.findall(r"\[(?:TABLE|IMAGE|EQUATION)\]_\d+", text)
+        for ph in placeholders:
+            if ph in placeholder_size_map:
+                # 加上实际内容的长度
+                base_count += placeholder_size_map[ph]
+                # 减去占位符本身被统计的长度（大致估算）
+                # 占位符通常会被统计为 1 个单词 (e.g. [TABLE]_0) 或几个特殊字符
+                # 这里简单处理：如果不扣除，误差很小且偏向保守（偏大），是安全的
+                # 若要精确，可以计算 ph 的 word count 并减去
+                base_count -= 1 # 简单减去1，假设占位符算作一个词
+    return base_count
+def split_text_by_words(text, max_words, soft_words, placeholder_size_map=None):
+    """智能分块算法，保留原始换行符"""
+    chunks = []
+    current_count = 0
+    buffer = []
+    # 按自然断点分割，增加换行符作为分隔符，以便处理连续的表格/图片
+    sentences = re.split(r"([。！？；\.\?!]|\n)", text)
+    sentences = [s for s in sentences if s] # 保留换行符，但去掉空字符串 (re.split might produce empty strings)
+    for i in range(0, len(sentences), 2):
+        sentence = sentences[i] + (sentences[i + 1] if i + 1 < len(sentences) else "")
+        sentence_word_count = count_words(sentence, placeholder_size_map)
+        # 强制分割条件
+        if current_count + sentence_word_count > max_words:
+            if buffer:
+                chunks.append("".join(buffer))  # 改为直接拼接，不添加额外换行
+                buffer = []
+                current_count = 0
+        buffer.append(sentence)
+        current_count += sentence_word_count
+        # 软分割条件
+        if current_count >= soft_words:
+            chunks.append("".join(buffer))
+            buffer = []
+            current_count = 0
+    if buffer:
+        chunks.append("".join(buffer))
+    ret_chunks = []
+    for item in chunks:
+        ret_chunks.append(item.replace("\n\n\n", "\n"))
+    return ret_chunks
+def find_balanced_split(text):
+    """寻找最佳均分点"""
+    mid = len(text) // 2
+    split_chars = ["\n", "。", ";", "!", "?", "；"]
+    # 向前查找
+    for i in range(mid, min(mid + 300, len(text))):
+        if text[i] in split_chars:
+            return i + 1  # 包含分割符
+    # 向后查找
+    for i in range(mid, max(mid - 300, 0), -1):
+        if text[i] in split_chars:
+            return i + 1
+    return mid
+# ---- 文档结构处理 ----
+def process_sections(data, MAX_CHUNK_WORDS, SOFT_CHUNK_WORDS, HARD_LIMIT, placeholder_size_map):
+    """处理章节结构"""
+    processed = []
+    current_title = None
+    accumulated = []
+    if len(data) ==1:
+        data[0]["text_level"] = 0
+    for item in data:
+        if is_section_title(item):
+            if current_title is not None:
+                flush_section(current_title, accumulated, processed, MAX_CHUNK_WORDS, SOFT_CHUNK_WORDS, HARD_LIMIT, placeholder_size_map)
+            current_title = item["text"]
+            accumulated = []
+        else:
+            accumulated.append(item["text"])
+    flush_section(current_title, accumulated, processed, MAX_CHUNK_WORDS, SOFT_CHUNK_WORDS, HARD_LIMIT, placeholder_size_map)
+    return processed
+def flush_section(title, parts, output, MAX_CHUNK_WORDS, SOFT_CHUNK_WORDS, HARD_LIMIT, placeholder_size_map):
+    """处理单个章节内容"""
+    full_text =title+ "\n".join(parts)
+    if not full_text.strip():
+        return
+    word_count = count_words(full_text, placeholder_size_map)
+    if word_count <= MAX_CHUNK_WORDS:
+        output.append(build_chunk(title, full_text))
+    elif MAX_CHUNK_WORDS < word_count <= HARD_LIMIT:
+        split_pos = find_balanced_split(full_text)
+        output.append(build_chunk(title, full_text[:split_pos]))
+        output.append(build_chunk(None, full_text[split_pos:]))
+    else:
+        chunks = split_text_by_words(full_text, MAX_CHUNK_WORDS, SOFT_CHUNK_WORDS, placeholder_size_map)
+        for i, chunk in enumerate(chunks):
+            output.append(build_chunk(title if i == 0 else None, chunk))
+def build_chunk(title, text):
+    """构建分块结构"""
+    if title:
+        return {"type": "text", "text": f"{title}\n{text.strip()}"}
+    return {"type": "text", "text": text.strip()}
+def is_section_title(item):
+    """判断是否为章节标题"""
+    return item.get("type") == "text" and item.get("text_level") == 1
+# 生成唯一占位符并记录映射
+def generate_placeholder(element_type, content, placeholder_map, placeholder_size_map):
+    placeholder = element_placeholder[element_type] + f"_{len(placeholder_map)}"
+    placeholder_map.append((placeholder, element_type, content))
+    # 计算 content 的字数并存入 map
+    placeholder_size_map[placeholder] = count_words(content)
+    # print(f"Generated placeholder: {placeholder} for {element_type} with content:\n{content}\n")
+    return placeholder, placeholder_map, placeholder_size_map
+# 根据占位符还原文本内容
+def restore_placeholders(text, placeholder_map):
+    for placeholder, element_type, content in placeholder_map:
+        if placeholder in text:
+            # 直接替换为之前生成的内容
+            text = text.replace(placeholder, content)
+    return text
+def replace_PSEUDO_EQUATION_FLAG(text, replace_text="\n"):
+    return text.replace(PSEUDO_EQUATION_FLAG, replace_text)
+def merge_element(prev, current, placeholder_map, placeholder_size_map):
+    """
+    * @description:
+    # 合并元素：
+       #   - 图像：用 Markdown 语法生成 "![](图片路径) 图片描述"，并与上文合并。
+       #   - 表格：将表格描述（如 table_caption）添加在表格内容上方，与上文合并。
+       #   - 公式：真公式与上下文合并（原逻辑不变）。假公式与下文合并
+    * @param  prev :
+    * @param  current :
+    * @return
+    """
+    if current["type"] in ["equation", "image", "table"]:
+        if current["type"] == "image":
+            # 针对图像，取 img_path 和 img_caption
+            img_path = current.get("img_path", "[Image path missing]")
+            img_cap = current.get("img_caption")
+            if isinstance(img_cap, list):
+                img_caption_text = " ".join(img_cap) if img_cap else ""
+            else:
+                img_caption_text = img_cap if img_cap else ""
+            # Markdown 格式：![](图片路径) 后接图片描述
+            placeholder_content = f"![]({img_path}) {img_caption_text}"
+        elif current["type"] == "table":
+            # 针对表格，先获取描述（table_caption），若没有则为空
+            table_caption = current.get("table_caption")
+            if isinstance(table_caption, list):
+                table_caption_text = " ".join(table_caption) if table_caption else ""
+            else:
+                table_caption_text = table_caption if table_caption else ""
+            # 表格内容取 table_body，如不存在则尝试 text 字段
+            table_body = current.get("table_body", current.get("text", "[Table content missing]"))
+            # 生成内容：在表格上方添加描述，再换行后显示表格内容
+            placeholder_content = f"{table_caption_text}\n{table_body}"
+        else:  # equation
+            placeholder_content = current.get("text", "[Equation]")
+        # 生成占位符并记录映射
+        placeholder, placeholder_map, placeholder_size_map = generate_placeholder(current["type"], placeholder_content, placeholder_map, placeholder_size_map)
+        # 合并规则：
+        # 公式与上下文合并；图像和表格只与上文合并
+        if current["type"] == "equation":
+            if prev and prev["type"] == "text":
+                # 合并公式到上一个文本段，并标记为假性公式
+                prev["text"] += PSEUDO_EQUATION_FLAG + placeholder
+                prev["is_pseudo_equation"] = True  # 标记为假性公式
+                return prev, None, placeholder_map, placeholder_size_map
+        else:
+            if prev and prev["type"] == "text":
+                prev["text"] += "\n" + placeholder + "\n"
+                return prev, None, placeholder_map, placeholder_size_map
+    return prev, current, placeholder_map, placeholder_size_map
+def pre_handle_func(data):
+    """
+    * @description: 预处理数据
+    * @param  data :
+    * @return
+    """
+    # 如果第一个元素不是文本，则在最前面插入一个空文本项，确保有上文可以合并
+    if data and data[0]["type"] != "text":
+      data.insert(0, {"type": "text", "text": "", "text_level": 1})
+    # 确保每个元素都有 "text" 和 "text_level" 键，避免 KeyError
+    for item in data:
+      if "text" not in item:
+        item["text"] = ""
+      if "text_level" not in item:
+        item["text_level"] = 0
+    # 过滤掉 "text" 为空的项，但保留 image、equation、table 类型的项以及 text_level 为 1 的项
+    filtered_data = [
+      item for item in data
+      if item.get("text") != "" or item["type"] in ["image", "equation", "table"] or item.get("text_level") == 1
+    ]
+    processed_data = []
+    previous_item = None
+    placeholder_map = []
+    placeholder_size_map = {} # 新增：用于记录占位符对应的实际字数
+    # 主处理流程：合并文本、拆分长文本等
+    for idx, item in enumerate(filtered_data):
+        if previous_item:
+            # 合并相邻的 "text_level": 1（标题连续），但如果文本中包含摘要或关键字，则不合并
+            if previous_item["type"] == "text" and item["type"] == "text":
+                if filtered_data[idx-1]==1 and item.get("text_level") == 1:
+                    previous_item["text"] += "-" + item["text"]
+                    continue
+            # 合并元素（图像、表格与上文，公式与上下文）
+            previous_item, item, placeholder_map, placeholder_size_map = merge_element(previous_item, item, placeholder_map, placeholder_size_map)
+            if item is None:
+                continue
+            # 处理假性公式的合并
+            if previous_item.get("is_pseudo_equation", False):
+                # 如果下一个文本段是标题，则不合并
+                if item.get("text_level") == 1:
+                    processed_data.append(previous_item)
+                    previous_item = item
+                    continue
+                # 如果当前文本段加上下一个文本段的长度超过 1200，则不合并
+                # 注意：这里使用简单的 len() 可能会低估包含占位符的文本实际长度，
+                # 但为了保持原有逻辑大致不变，且考虑到这里是初步合并，暂时保留。
+                # 理想情况应该用 count_words 结合 placeholder_size_map 判断。
+                # Update: 使用更准确的字数统计，以避免在预处理阶段就合并了过大的块，
+                # 导致后续 split_text_by_words 难以处理或产生非预期的巨大块。
+                # 虽然这里硬编码了 1200，但为了解决 "table chunk too long" 问题，
+                # 我们需要在这里也感知占位符的实际大小。
+                prev_len = count_words(previous_item["text"], placeholder_size_map)
+                curr_len = count_words(item["text"], placeholder_size_map)
+                if prev_len + curr_len > 1200:
+                    processed_data.append(previous_item)
+                    previous_item = item
+                    continue
+                # 否则，合并假性公式与下一个文本段
+                previous_item["text"] += item["text"]
+                previous_item["is_pseudo_equation"] = False  # 清除假性公式标志
+                continue
+            processed_data.append(previous_item)
+        previous_item = item
+    if previous_item:
+        processed_data.append(previous_item)
+    return processed_data, placeholder_map, placeholder_size_map
+def markdown_json_list2chunk_list(data, MAX_CHUNK_WORDS, SOFT_CHUNK_WORDS, HARD_LIMIT):
+    """
+    * @description: 主要流程, 整理
+    * @param  data :
+    * @param  MAX_CHUNK_WORDS :
+    * @param  SOFT_CHUNK_WORDS :
+    * @param  HARD_LIMIT :
+    * @return
+    """
+    pre_handle_data, placeholder_map, placeholder_size_map = pre_handle_func(data=data)
+    middle_handle_data = process_sections(pre_handle_data, MAX_CHUNK_WORDS, SOFT_CHUNK_WORDS, HARD_LIMIT, placeholder_size_map)
+    # 统一恢复占位符，确保所有文本中的占位符都被正确替换
+    for item in middle_handle_data:
+        if item["type"] == "text":
+            item["text"] = restore_placeholders(item["text"], placeholder_map)
+            item["text"] = replace_PSEUDO_EQUATION_FLAG(item["text"],"\n")
+    # 后处理：过滤掉空的文本元素
+    final_handle_data = [
+        item for item in middle_handle_data
+        if not (item.get("type") == "text" and not item.get("text", "").strip())
+    ]
+    return final_handle_data
+if __name__ == "__main__":
+    ## config:
+    MAX_CHUNK_WORDS = 1000
+    SOFT_CHUNK_WORDS = 400
+    HARD_LIMIT = 1400
+    ##
+    with open("temp.json", "r", encoding="utf-8") as file:
+        data = json.load(file)
+    final_handle_data = markdown_json_list2chunk_list(data, MAX_CHUNK_WORDS=MAX_CHUNK_WORDS, SOFT_CHUNK_WORDS=SOFT_CHUNK_WORDS, HARD_LIMIT=HARD_LIMIT)
+    # 保存更新后的 JSON 文件
+    with open("output4-yyb-en2.json", "w", encoding="utf-8") as file:
+        json.dump(final_handle_data, file, ensure_ascii=False, indent=4)