magic-pdf 0.10.0__py3-none-any.whl → 0.10.2__py3-none-any.whl

magic_pdf/post_proc/detect_para.py

@@ -1,3472 +0,0 @@
-import os
-import sys
-import json
-import re
-import math
-import unicodedata
-from collections import Counter
-
-
-import numpy as np
-from termcolor import cprint
-
-
-from magic_pdf.libs.commons import fitz
-from magic_pdf.libs.nlp_utils import NLPModels
-
-
-if sys.version_info[0] >= 3:
-    sys.stdout.reconfigure(encoding="utf-8")  # type: ignore
-
-
-def open_pdf(pdf_path):
-    try:
-        pdf_document = fitz.open(pdf_path)  # type: ignore
-        return pdf_document
-    except Exception as e:
-        print(f"无法打开PDF文件：{pdf_path}。原因是：{e}")
-        raise e
-
-
-def print_green_on_red(text):
-    cprint(text, "green", "on_red", attrs=["bold"], end="\n\n")
-
-
-def print_green(text):
-    print()
-    cprint(text, "green", attrs=["bold"], end="\n\n")
-
-
-def print_red(text):
-    print()
-    cprint(text, "red", attrs=["bold"], end="\n\n")
-
-
-def print_yellow(text):
-    print()
-    cprint(text, "yellow", attrs=["bold"], end="\n\n")
-
-
-def safe_get(dict_obj, key, default):
-    val = dict_obj.get(key)
-    if val is None:
-        return default
-    else:
-        return val
-
-
-def is_bbox_overlap(bbox1, bbox2):
-    """
-    This function checks if bbox1 and bbox2 overlap or not
-
-    Parameters
-    ----------
-    bbox1 : list
-        bbox1
-    bbox2 : list
-        bbox2
-
-    Returns
-    -------
-    bool
-        True if bbox1 and bbox2 overlap, else False
-    """
-    x0_1, y0_1, x1_1, y1_1 = bbox1
-    x0_2, y0_2, x1_2, y1_2 = bbox2
-
-    if x0_1 > x1_2 or x0_2 > x1_1:
-        return False
-    if y0_1 > y1_2 or y0_2 > y1_1:
-        return False
-
-    return True
-
-
-def is_in_bbox(bbox1, bbox2):
-    """
-    This function checks if bbox1 is in bbox2
-
-    Parameters
-    ----------
-    bbox1 : list
-        bbox1
-    bbox2 : list
-        bbox2
-
-    Returns
-    -------
-    bool
-        True if bbox1 is in bbox2, else False
-    """
-    x0_1, y0_1, x1_1, y1_1 = bbox1
-    x0_2, y0_2, x1_2, y1_2 = bbox2
-
-    if x0_1 >= x0_2 and y0_1 >= y0_2 and x1_1 <= x1_2 and y1_1 <= y1_2:
-        return True
-    else:
-        return False
-
-
-def calculate_para_bbox(lines):
-    """
-    This function calculates the minimum bbox of the paragraph
-
-    Parameters
-    ----------
-    lines : list
-        lines
-
-    Returns
-    -------
-    para_bbox : list
-        bbox of the paragraph
-    """
-    x0 = min(line["bbox"][0] for line in lines)
-    y0 = min(line["bbox"][1] for line in lines)
-    x1 = max(line["bbox"][2] for line in lines)
-    y1 = max(line["bbox"][3] for line in lines)
-    return [x0, y0, x1, y1]
-
-
-def is_line_right_aligned_from_neighbors(curr_line_bbox, prev_line_bbox, next_line_bbox, avg_char_width, direction=2):
-    """
-    This function checks if the line is right aligned from its neighbors
-
-    Parameters
-    ----------
-    curr_line_bbox : list
-        bbox of the current line
-    prev_line_bbox : list
-        bbox of the previous line
-    next_line_bbox : list
-        bbox of the next line
-    avg_char_width : float
-        average of char widths
-    direction : int
-        0 for prev, 1 for next, 2 for both
-
-    Returns
-    -------
-    bool
-        True if the line is right aligned from its neighbors, False otherwise.
-    """
-    horizontal_ratio = 0.5
-    horizontal_thres = horizontal_ratio * avg_char_width
-
-    _, _, x1, _ = curr_line_bbox
-    _, _, prev_x1, _ = prev_line_bbox if prev_line_bbox else (0, 0, 0, 0)
-    _, _, next_x1, _ = next_line_bbox if next_line_bbox else (0, 0, 0, 0)
-
-    if direction == 0:
-        return abs(x1 - prev_x1) < horizontal_thres
-    elif direction == 1:
-        return abs(x1 - next_x1) < horizontal_thres
-    elif direction == 2:
-        return abs(x1 - prev_x1) < horizontal_thres and abs(x1 - next_x1) < horizontal_thres
-    else:
-        return False
-
-
-def is_line_left_aligned_from_neighbors(curr_line_bbox, prev_line_bbox, next_line_bbox, avg_char_width, direction=2):
-    """
-    This function checks if the line is left aligned from its neighbors
-
-    Parameters
-    ----------
-    curr_line_bbox : list
-        bbox of the current line
-    prev_line_bbox : list
-        bbox of the previous line
-    next_line_bbox : list
-        bbox of the next line
-    avg_char_width : float
-        average of char widths
-    direction : int
-        0 for prev, 1 for next, 2 for both
-
-    Returns
-    -------
-    bool
-        True if the line is left aligned from its neighbors, False otherwise.
-    """
-    horizontal_ratio = 0.5
-    horizontal_thres = horizontal_ratio * avg_char_width
-
-    x0, _, _, _ = curr_line_bbox
-    prev_x0, _, _, _ = prev_line_bbox if prev_line_bbox else (0, 0, 0, 0)
-    next_x0, _, _, _ = next_line_bbox if next_line_bbox else (0, 0, 0, 0)
-
-    if direction == 0:
-        return abs(x0 - prev_x0) < horizontal_thres
-    elif direction == 1:
-        return abs(x0 - next_x0) < horizontal_thres
-    elif direction == 2:
-        return abs(x0 - prev_x0) < horizontal_thres and abs(x0 - next_x0) < horizontal_thres
-    else:
-        return False
-
-
-def end_with_punctuation(line_text):
-    """
-    This function checks if the line ends with punctuation marks
-    """
-
-    english_end_puncs = [".", "?", "!"]
-    chinese_end_puncs = ["。", "？", "！"]
-    end_puncs = english_end_puncs + chinese_end_puncs
-
-    last_non_space_char = None
-    for ch in line_text[::-1]:
-        if not ch.isspace():
-            last_non_space_char = ch
-            break
-
-    if last_non_space_char is None:
-        return False
-
-    return last_non_space_char in end_puncs
-
-
-def is_nested_list(lst):
-    if isinstance(lst, list):
-        return any(isinstance(sub, list) for sub in lst)
-    return False
-
-
-class DenseSingleLineBlockException(Exception):
-    """
-    This class defines the exception type for dense single line-block.
-    """
-
-    def __init__(self, message="DenseSingleLineBlockException"):
-        self.message = message
-        super().__init__(self.message)
-
-    def __str__(self):
-        return f"{self.message}"
-
-    def __repr__(self):
-        return f"{self.message}"
-
-
-class TitleDetectionException(Exception):
-    """
-    This class defines the exception type for title detection.
-    """
-
-    def __init__(self, message="TitleDetectionException"):
-        self.message = message
-        super().__init__(self.message)
-
-    def __str__(self):
-        return f"{self.message}"
-
-    def __repr__(self):
-        return f"{self.message}"
-
-
-class TitleLevelException(Exception):
-    """
-    This class defines the exception type for title level.
-    """
-
-    def __init__(self, message="TitleLevelException"):
-        self.message = message
-        super().__init__(self.message)
-
-    def __str__(self):
-        return f"{self.message}"
-
-    def __repr__(self):
-        return f"{self.message}"
-
-
-class ParaSplitException(Exception):
-    """
-    This class defines the exception type for paragraph splitting.
-    """
-
-    def __init__(self, message="ParaSplitException"):
-        self.message = message
-        super().__init__(self.message)
-
-    def __str__(self):
-        return f"{self.message}"
-
-    def __repr__(self):
-        return f"{self.message}"
-
-
-class ParaMergeException(Exception):
-    """
-    This class defines the exception type for paragraph merging.
-    """
-
-    def __init__(self, message="ParaMergeException"):
-        self.message = message
-        super().__init__(self.message)
-
-    def __str__(self):
-        return f"{self.message}"
-
-    def __repr__(self):
-        return f"{self.message}"
-
-
-class DiscardByException:
-    """
-    This class discards pdf files by exception
-    """
-
-    def __init__(self) -> None:
-        pass
-
-    def discard_by_single_line_block(self, pdf_dic, exception: DenseSingleLineBlockException):
-        """
-        This function discards pdf files by single line block exception
-
-        Parameters
-        ----------
-        pdf_dic : dict
-            pdf dictionary
-        exception : str
-            exception message
-
-        Returns
-        -------
-        error_message : str
-        """
-        exception_page_nums = 0
-        page_num = 0
-        for page_id, page in pdf_dic.items():
-            if page_id.startswith("page_"):
-                page_num += 1
-                if "preproc_blocks" in page.keys():
-                    preproc_blocks = page["preproc_blocks"]
-
-                    all_single_line_blocks = []
-                    for block in preproc_blocks:
-                        if len(block["lines"]) == 1:
-                            all_single_line_blocks.append(block)
-
-                    if len(preproc_blocks) > 0 and len(all_single_line_blocks) / len(preproc_blocks) > 0.9:
-                        exception_page_nums += 1
-
-        if page_num == 0:
-            return None
-
-        if exception_page_nums / page_num > 0.1:  # Low ratio means basically, whenever this is the case, it is discarded
-            return exception.message
-
-        return None
-
-    def discard_by_title_detection(self, pdf_dic, exception: TitleDetectionException):
-        """
-        This function discards pdf files by title detection exception
-
-        Parameters
-        ----------
-        pdf_dic : dict
-            pdf dictionary
-        exception : str
-            exception message
-
-        Returns
-        -------
-        error_message : str
-        """
-        # return exception.message
-        return None
-
-    def discard_by_title_level(self, pdf_dic, exception: TitleLevelException):
-        """
-        This function discards pdf files by title level exception
-
-        Parameters
-        ----------
-        pdf_dic : dict
-            pdf dictionary
-        exception : str
-            exception message
-
-        Returns
-        -------
-        error_message : str
-        """
-        # return exception.message
-        return None
-
-    def discard_by_split_para(self, pdf_dic, exception: ParaSplitException):
-        """
-        This function discards pdf files by split para exception
-
-        Parameters
-        ----------
-        pdf_dic : dict
-            pdf dictionary
-        exception : str
-            exception message
-
-        Returns
-        -------
-        error_message : str
-        """
-        # return exception.message
-        return None
-
-    def discard_by_merge_para(self, pdf_dic, exception: ParaMergeException):
-        """
-        This function discards pdf files by merge para exception
-
-        Parameters
-        ----------
-        pdf_dic : dict
-            pdf dictionary
-        exception : str
-            exception message
-
-        Returns
-        -------
-        error_message : str
-        """
-        # return exception.message
-        return None
-
-
-class LayoutFilterProcessor:
-    def __init__(self) -> None:
-        pass
-
-    def batch_process_blocks(self, pdf_dict):
-        """
-        This function processes the blocks in batch.
-
-        Parameters
-        ----------
-        self : object
-            The instance of the class.
-
-        pdf_dict : dict
-            pdf dictionary
-
-        Returns
-        -------
-        pdf_dict : dict
-            pdf dictionary
-        """
-        for page_id, blocks in pdf_dict.items():
-            if page_id.startswith("page_"):
-                if "layout_bboxes" in blocks.keys() and "para_blocks" in blocks.keys():
-                    layout_bbox_objs = blocks["layout_bboxes"]
-                    if layout_bbox_objs is None:
-                        continue
-                    layout_bboxes = [bbox_obj["layout_bbox"] for bbox_obj in layout_bbox_objs]
-
-                    # Enlarge each value of x0, y0, x1, y1 for each layout_bbox to prevent loss of text.
-                    layout_bboxes = [
-                        [math.ceil(x0), math.ceil(y0), math.ceil(x1), math.ceil(y1)] for x0, y0, x1, y1 in layout_bboxes
-                    ]
-
-                    para_blocks = blocks["para_blocks"]
-                    if para_blocks is None:
-                        continue
-
-                    for lb_bbox in layout_bboxes:
-                        for i, para_block in enumerate(para_blocks):
-                            para_bbox = para_block["bbox"]
-                            para_blocks[i]["in_layout"] = 0
-                            if is_in_bbox(para_bbox, lb_bbox):
-                                para_blocks[i]["in_layout"] = 1
-
-                    blocks["para_blocks"] = para_blocks
-
-        return pdf_dict
-
-
-class RawBlockProcessor:
-    def __init__(self) -> None:
-        self.y_tolerance = 2
-        self.pdf_dic = {}
-
-    def __span_flags_decomposer(self, span_flags):
-        """
-        Make font flags human readable.
-
-        Parameters
-        ----------
-        self : object
-            The instance of the class.
-
-        span_flags : int
-            span flags
-
-        Returns
-        -------
-        l : dict
-            decomposed flags
-        """
-
-        l = {
-            "is_superscript": False,
-            "is_italic": False,
-            "is_serifed": False,
-            "is_sans_serifed": False,
-            "is_monospaced": False,
-            "is_proportional": False,
-            "is_bold": False,
-        }
-
-        if span_flags & 2**0:
-            l["is_superscript"] = True  # 表示上标
-
-        if span_flags & 2**1:
-            l["is_italic"] = True  # 表示斜体
-
-        if span_flags & 2**2:
-            l["is_serifed"] = True  # 表示衬线字体
-        else:
-            l["is_sans_serifed"] = True  # 表示非衬线字体
-
-        if span_flags & 2**3:
-            l["is_monospaced"] = True  # 表示等宽字体
-        else:
-            l["is_proportional"] = True  # 表示比例字体
-
-        if span_flags & 2**4:
-            l["is_bold"] = True  # 表示粗体
-
-        return l
-
-    def __make_new_lines(self, raw_lines):
-        """
-        This function makes new lines.
-
-        Parameters
-        ----------
-        self : object
-            The instance of the class.
-
-        raw_lines : list
-            raw lines
-
-        Returns
-        -------
-        new_lines : list
-            new lines
-        """
-        new_lines = []
-        new_line = None
-
-        for raw_line in raw_lines:
-            raw_line_bbox = raw_line["bbox"]
-            raw_line_spans = raw_line["spans"]
-            raw_line_text = "".join([span["text"] for span in raw_line_spans])
-            raw_line_dir = raw_line.get("dir", None)
-
-            decomposed_line_spans = []
-            for span in raw_line_spans:
-                raw_flags = span["flags"]
-                decomposed_flags = self.__span_flags_decomposer(raw_flags)
-                span["decomposed_flags"] = decomposed_flags
-                decomposed_line_spans.append(span)
-
-            if new_line is None:  # Handle the first line
-                new_line = {
-                    "bbox": raw_line_bbox,
-                    "text": raw_line_text,
-                    "dir": raw_line_dir if raw_line_dir else (0, 0),
-                    "spans": decomposed_line_spans,
-                }
-            else:  # Handle the rest lines
-                if (
-                    abs(raw_line_bbox[1] - new_line["bbox"][1]) <= self.y_tolerance
-                    and abs(raw_line_bbox[3] - new_line["bbox"][3]) <= self.y_tolerance
-                ):
-                    new_line["bbox"] = (
-                        min(new_line["bbox"][0], raw_line_bbox[0]),  # left
-                        new_line["bbox"][1],  # top
-                        max(new_line["bbox"][2], raw_line_bbox[2]),  # right
-                        raw_line_bbox[3],  # bottom
-                    )
-                    new_line["text"] += raw_line_text
-                    new_line["spans"].extend(raw_line_spans)
-                    new_line["dir"] = (
-                        new_line["dir"][0] + raw_line_dir[0],
-                        new_line["dir"][1] + raw_line_dir[1],
-                    )
-                else:
-                    new_lines.append(new_line)
-                    new_line = {
-                        "bbox": raw_line_bbox,
-                        "text": raw_line_text,
-                        "dir": raw_line_dir if raw_line_dir else (0, 0),
-                        "spans": raw_line_spans,
-                    }
-        if new_line:
-            new_lines.append(new_line)
-
-        return new_lines
-
-    def __make_new_block(self, raw_block):
-        """
-        This function makes a new block.
-
-        Parameters
-        ----------
-        self : object
-            The instance of the class.
-        ----------
-        raw_block : dict
-            a raw block
-
-        Returns
-        -------
-        new_block : dict
-        """
-        new_block = {}
-
-        block_id = raw_block["number"]
-        block_bbox = raw_block["bbox"]
-        block_text = "".join(span["text"] for line in raw_block["lines"] for span in line["spans"])
-        raw_lines = raw_block["lines"]
-        block_lines = self.__make_new_lines(raw_lines)
-
-        new_block["block_id"] = block_id
-        new_block["bbox"] = block_bbox
-        new_block["text"] = block_text
-        new_block["lines"] = block_lines
-
-        return new_block
-
-    def batch_process_blocks(self, pdf_dic):
-        """
-        This function processes the blocks in batch.
-
-        Parameters
-        ----------
-        self : object
-            The instance of the class.
-        ----------
-        blocks : list
-            Input block is a list of raw blocks.
-
-        Returns
-        -------
-        result_dict : dict
-            result dictionary
-        """
-
-        for page_id, blocks in pdf_dic.items():
-            if page_id.startswith("page_"):
-                para_blocks = []
-                if "preproc_blocks" in blocks.keys():
-                    input_blocks = blocks["preproc_blocks"]
-                    for raw_block in input_blocks:
-                        new_block = self.__make_new_block(raw_block)
-                        para_blocks.append(new_block)
-
-                blocks["para_blocks"] = para_blocks
-
-        return pdf_dic
-
-
-class BlockStatisticsCalculator:
-    """
-    This class calculates the statistics of the block.
-    """
-
-    def __init__(self) -> None:
-        pass
-
-    def __calc_stats_of_new_lines(self, new_lines):
-        """
-        This function calculates the paragraph metrics
-
-        Parameters
-        ----------
-        combined_lines : list
-            combined lines
-
-        Returns
-        -------
-        X0 : float
-            Median of x0 values, which represents the left average boundary of the block
-        X1 : float
-            Median of x1 values, which represents the right average boundary of the block
-        avg_char_width : float
-            Average of char widths, which represents the average char width of the block
-        avg_char_height : float
-            Average of line heights, which represents the average line height of the block
-
-        """
-        x0_values = []
-        x1_values = []
-        char_widths = []
-        char_heights = []
-
-        block_font_types = []
-        block_font_sizes = []
-        block_directions = []
-
-        if len(new_lines) > 0:
-            for i, line in enumerate(new_lines):
-                line_bbox = line["bbox"]
-                line_text = line["text"]
-                line_spans = line["spans"]
-
-                num_chars = len([ch for ch in line_text if not ch.isspace()])
-
-                x0_values.append(line_bbox[0])
-                x1_values.append(line_bbox[2])
-
-                if num_chars > 0:
-                    char_width = (line_bbox[2] - line_bbox[0]) / num_chars
-                    char_widths.append(char_width)
-
-                for span in line_spans:
-                    block_font_types.append(span["font"])
-                    block_font_sizes.append(span["size"])
-
-                if "dir" in line:
-                    block_directions.append(line["dir"])
-
-                # line_font_types = [span["font"] for span in line_spans]
-                char_heights = [span["size"] for span in line_spans]
-
-        X0 = np.median(x0_values) if x0_values else 0
-        X1 = np.median(x1_values) if x1_values else 0
-        avg_char_width = sum(char_widths) / len(char_widths) if char_widths else 0
-        avg_char_height = sum(char_heights) / len(char_heights) if char_heights else 0
-
-        # max_freq_font_type = max(set(block_font_types), key=block_font_types.count) if block_font_types else None
-
-        max_span_length = 0
-        max_span_font_type = None
-        for line in new_lines:
-            line_spans = line["spans"]
-            for span in line_spans:
-                span_length = span["bbox"][2] - span["bbox"][0]
-                if span_length > max_span_length:
-                    max_span_length = span_length
-                    max_span_font_type = span["font"]
-
-        max_freq_font_type = max_span_font_type
-
-        avg_font_size = sum(block_font_sizes) / len(block_font_sizes) if block_font_sizes else None
-
-        avg_dir_horizontal = sum([dir[0] for dir in block_directions]) / len(block_directions) if block_directions else 0
-        avg_dir_vertical = sum([dir[1] for dir in block_directions]) / len(block_directions) if block_directions else 0
-
-        median_font_size = float(np.median(block_font_sizes)) if block_font_sizes else None
-
-        return (
-            X0,
-            X1,
-            avg_char_width,
-            avg_char_height,
-            max_freq_font_type,
-            avg_font_size,
-            (avg_dir_horizontal, avg_dir_vertical),
-            median_font_size,
-        )
-
-    def __make_new_block(self, input_block):
-        new_block = {}
-
-        raw_lines = input_block["lines"]
-        stats = self.__calc_stats_of_new_lines(raw_lines)
-
-        block_id = input_block["block_id"]
-        block_bbox = input_block["bbox"]
-        block_text = input_block["text"]
-        block_lines = raw_lines
-        block_avg_left_boundary = stats[0]
-        block_avg_right_boundary = stats[1]
-        block_avg_char_width = stats[2]
-        block_avg_char_height = stats[3]
-        block_font_type = stats[4]
-        block_font_size = stats[5]
-        block_direction = stats[6]
-        block_median_font_size = stats[7]
-
-        new_block["block_id"] = block_id
-        new_block["bbox"] = block_bbox
-        new_block["text"] = block_text
-        new_block["dir"] = block_direction
-        new_block["X0"] = block_avg_left_boundary
-        new_block["X1"] = block_avg_right_boundary
-        new_block["avg_char_width"] = block_avg_char_width
-        new_block["avg_char_height"] = block_avg_char_height
-        new_block["block_font_type"] = block_font_type
-        new_block["block_font_size"] = block_font_size
-        new_block["lines"] = block_lines
-        new_block["median_font_size"] = block_median_font_size
-
-        return new_block
-
-    def batch_process_blocks(self, pdf_dic):
-        """
-        This function processes the blocks in batch.
-
-        Parameters
-        ----------
-        self : object
-            The instance of the class.
-        ----------
-        blocks : list
-            Input block is a list of raw blocks.
-            Schema can refer to the value of key ""preproc_blocks".
-
-        Returns
-        -------
-        result_dict : dict
-            result dictionary
-        """
-
-        for page_id, blocks in pdf_dic.items():
-            if page_id.startswith("page_"):
-                para_blocks = []
-                if "para_blocks" in blocks.keys():
-                    input_blocks = blocks["para_blocks"]
-                    for input_block in input_blocks:
-                        new_block = self.__make_new_block(input_block)
-                        para_blocks.append(new_block)
-
-                blocks["para_blocks"] = para_blocks
-
-        return pdf_dic
-
-
-class DocStatisticsCalculator:
-    """
-    This class calculates the statistics of the document.
-    """
-
-    def __init__(self) -> None:
-        pass
-
-    def calc_stats_of_doc(self, pdf_dict):
-        """
-        This function computes the statistics of the document
-
-        Parameters
-        ----------
-        result_dict : dict
-            result dictionary
-
-        Returns
-        -------
-        statistics : dict
-            statistics of the document
-        """
-
-        total_text_length = 0
-        total_num_blocks = 0
-
-        for page_id, blocks in pdf_dict.items():
-            if page_id.startswith("page_"):
-                if "para_blocks" in blocks.keys():
-                    para_blocks = blocks["para_blocks"]
-                    for para_block in para_blocks:
-                        total_text_length += len(para_block["text"])
-                        total_num_blocks += 1
-
-        avg_text_length = total_text_length / total_num_blocks if total_num_blocks else 0
-
-        font_list = []
-
-        for page_id, blocks in pdf_dict.items():
-            if page_id.startswith("page_"):
-                if "para_blocks" in blocks.keys():
-                    input_blocks = blocks["para_blocks"]
-                    for input_block in input_blocks:
-                        block_text_length = len(input_block.get("text", ""))
-                        if block_text_length < avg_text_length * 0.5:
-                            continue
-                        block_font_type = safe_get(input_block, "block_font_type", "")
-                        block_font_size = safe_get(input_block, "block_font_size", 0)
-                        font_list.append((block_font_type, block_font_size))
-
-        font_counter = Counter(font_list)
-        most_common_font = font_counter.most_common(1)[0] if font_list else (("", 0), 0)
-        second_most_common_font = font_counter.most_common(2)[1] if len(font_counter) > 1 else (("", 0), 0)
-
-        statistics = {
-            "num_pages": 0,
-            "num_blocks": 0,
-            "num_paras": 0,
-            "num_titles": 0,
-            "num_header_blocks": 0,
-            "num_footer_blocks": 0,
-            "num_watermark_blocks": 0,
-            "num_vertical_margin_note_blocks": 0,
-            "most_common_font_type": most_common_font[0][0],
-            "most_common_font_size": most_common_font[0][1],
-            "number_of_most_common_font": most_common_font[1],
-            "second_most_common_font_type": second_most_common_font[0][0],
-            "second_most_common_font_size": second_most_common_font[0][1],
-            "number_of_second_most_common_font": second_most_common_font[1],
-            "avg_text_length": avg_text_length,
-        }
-
-        for page_id, blocks in pdf_dict.items():
-            if page_id.startswith("page_"):
-                blocks = pdf_dict[page_id]["para_blocks"]
-                statistics["num_pages"] += 1
-                for block_id, block_data in enumerate(blocks):
-                    statistics["num_blocks"] += 1
-
-                    if "paras" in block_data.keys():
-                        statistics["num_paras"] += len(block_data["paras"])
-
-                    for line in block_data["lines"]:
-                        if line.get("is_title", 0):
-                            statistics["num_titles"] += 1
-
-                    if block_data.get("is_header", 0):
-                        statistics["num_header_blocks"] += 1
-                    if block_data.get("is_footer", 0):
-                        statistics["num_footer_blocks"] += 1
-                    if block_data.get("is_watermark", 0):
-                        statistics["num_watermark_blocks"] += 1
-                    if block_data.get("is_vertical_margin_note", 0):
-                        statistics["num_vertical_margin_note_blocks"] += 1
-
-        pdf_dict["statistics"] = statistics
-
-        return pdf_dict
-
-
-class TitleProcessor:
-    """
-    This class processes the title.
-    """
-
-    def __init__(self, *doc_statistics) -> None:
-        if len(doc_statistics) > 0:
-            self.doc_statistics = doc_statistics[0]
-
-        self.nlp_model = NLPModels()
-        self.MAX_TITLE_LEVEL = 3
-        self.numbered_title_pattern = r"""
-            ^                                 # 行首
-            (                                 # 开始捕获组
-                [\(\（]\d+[\)\）]              # 括号内数字，支持中文和英文括号，例如：(1) 或 （1）
-                |\d+[\)\）]\s                  # 数字后跟右括号和空格，支持中文和英文括号，例如：2) 或 2）
-                |[\(\（][A-Z][\)\）]            # 括号内大写字母，支持中文和英文括号，例如：(A) 或 （A）
-                |[A-Z][\)\）]\s                # 大写字母后跟右括号和空格，例如：A) 或 A）
-                |[\(\（][IVXLCDM]+[\)\）]       # 括号内罗马数字，支持中文和英文括号，例如：(I) 或 （I）
-                |[IVXLCDM]+[\)\）]\s            # 罗马数字后跟右括号和空格，例如：I) 或 I）
-                |\d+(\.\d+)*\s                # 数字或复合数字编号后跟空格，例如：1. 或 3.2.1 
-                |[一二三四五六七八九十百千]+[、\s]       # 中文序号后跟顿号和空格，例如：一、
-                |[\（|\(][一二三四五六七八九十百千]+[\）|\)]\s*  # 中文括号内中文序号后跟空格，例如：（一）
-                |[A-Z]\.\d+(\.\d+)?\s         # 大写字母后跟点和数字，例如：A.1 或 A.1.1
-                |[\(\（][a-z][\)\）]            # 括号内小写字母，支持中文和英文括号，例如：(a) 或 （a）
-                |[a-z]\)\s                    # 小写字母后跟右括号和空格，例如：a) 
-                |[A-Z]-\s                     # 大写字母后跟短横线和空格，例如：A- 
-                |\w+:\s                       # 英文序号词后跟冒号和空格，例如：First: 
-                |第[一二三四五六七八九十百千]+[章节部分条款]\s # 以“第”开头的中文标题后跟空格
-                |[IVXLCDM]+\.                 # 罗马数字后跟点，例如：I.
-                |\d+\.\s                      # 单个数字后跟点和空格，例如：1. 
-            )                                 # 结束捕获组
-            .+                                # 标题的其余部分
-        """
-
-    def _is_potential_title(
-        self,
-        curr_line,
-        prev_line,
-        prev_line_is_title,
-        next_line,
-        avg_char_width,
-        avg_char_height,
-        median_font_size,
-    ):
-        """
-        This function checks if the line is a potential title.
-
-        Parameters
-        ----------
-        curr_line : dict
-            current line
-        prev_line : dict
-            previous line
-        next_line : dict
-            next line
-        avg_char_width : float
-            average of char widths
-        avg_char_height : float
-            average of line heights
-
-        Returns
-        -------
-        bool
-            True if the line is a potential title, False otherwise.
-        """
-
-        def __is_line_centered(line_bbox, page_bbox, avg_char_width):
-            """
-            This function checks if the line is centered on the page
-
-            Parameters
-            ----------
-            line_bbox : list
-                bbox of the line
-            page_bbox : list
-                bbox of the page
-            avg_char_width : float
-                average of char widths
-
-            Returns
-            -------
-            bool
-                True if the line is centered on the page, False otherwise.
-            """
-            horizontal_ratio = 0.5
-            horizontal_thres = horizontal_ratio * avg_char_width
-
-            x0, _, x1, _ = line_bbox
-            _, _, page_x1, _ = page_bbox
-
-            return abs((x0 + x1) / 2 - page_x1 / 2) < horizontal_thres
-
-        def __is_bold_font_line(line):
-            """
-            Check if a line contains any bold font style.
-            """
-
-            def _is_bold_span(span):
-                # if span text is empty or only contains space, return False
-                if not span["text"].strip():
-                    return False
-
-                return bool(span["flags"] & 2**4)  # Check if the font is bold
-
-            for span in line["spans"]:
-                if not _is_bold_span(span):
-                    return False
-
-            return True
-
-        def __is_italic_font_line(line):
-            """
-            Check if a line contains any italic font style.
-            """
-
-            def __is_italic_span(span):
-                return bool(span["flags"] & 2**1)  # Check if the font is italic
-
-            for span in line["spans"]:
-                if not __is_italic_span(span):
-                    return False
-
-            return True
-
-        def __is_punctuation_heavy(line_text):
-            """
-            Check if the line contains a high ratio of punctuation marks, which may indicate
-            that the line is not a title.
-
-            Parameters:
-            line_text (str): Text of the line.
-
-            Returns:
-            bool: True if the line is heavy with punctuation, False otherwise.
-            """
-            # Pattern for common title format like "X.Y. Title"
-            pattern = r"\b\d+\.\d+\..*\b"
-
-            # If the line matches the title format, return False
-            if re.match(pattern, line_text.strip()):
-                return False
-
-            # Find all punctuation marks in the line
-            punctuation_marks = re.findall(r"[^\w\s]", line_text)
-            number_of_punctuation_marks = len(punctuation_marks)
-
-            text_length = len(line_text)
-
-            if text_length == 0:
-                return False
-
-            punctuation_ratio = number_of_punctuation_marks / text_length
-            if punctuation_ratio >= 0.1:
-                return True
-
-            return False
-
-        def __has_mixed_font_styles(spans, strict_mode=False):
-            """
-            This function checks if the line has mixed font styles, the strict mode will compare the font types
-
-            Parameters
-            ----------
-            spans : list
-                spans of the line
-            strict_mode : bool
-                True for strict mode, the font types will be fully compared
-                False for non-strict mode, the font types will be compared by the most longest common prefix
-
-            Returns
-            -------
-            bool
-                True if the line has mixed font styles, False otherwise.
-            """
-            if strict_mode:
-                font_styles = set()
-                for span in spans:
-                    font_style = span["font"].lower()
-                    font_styles.add(font_style)
-
-                return len(font_styles) > 1
-
-            else:  # non-strict mode
-                font_styles = []
-                for span in spans:
-                    font_style = span["font"].lower()
-                    font_styles.append(font_style)
-
-                if len(font_styles) > 1:
-                    longest_common_prefix = os.path.commonprefix(font_styles)
-                    if len(longest_common_prefix) > 0:
-                        return False
-                    else:
-                        return True
-                else:
-                    return False
-
-        def __is_different_font_type_from_neighbors(curr_line_font_type, prev_line_font_type, next_line_font_type):
-            """
-            This function checks if the current line has a different font type from the previous and next lines
-
-            Parameters
-            ----------
-            curr_line_font_type : str
-                font type of the current line
-            prev_line_font_type : str
-                font type of the previous line
-            next_line_font_type : str
-                font type of the next line
-
-            Returns
-            -------
-            bool
-                True if the current line has a different font type from the previous and next lines, False otherwise.
-            """
-            return all(
-                curr_line_font_type != other_font_type.lower()
-                for other_font_type in [prev_line_font_type, next_line_font_type]
-                if other_font_type is not None
-            )
-
-        def __is_larger_font_size_from_neighbors(curr_line_font_size, prev_line_font_size, next_line_font_size):
-            """
-            This function checks if the current line has a larger font size than the previous and next lines
-
-            Parameters
-            ----------
-            curr_line_font_size : float
-                font size of the current line
-            prev_line_font_size : float
-                font size of the previous line
-            next_line_font_size : float
-                font size of the next line
-
-            Returns
-            -------
-            bool
-                True if the current line has a larger font size than the previous and next lines, False otherwise.
-            """
-            return all(
-                curr_line_font_size > other_font_size * 1.2
-                for other_font_size in [prev_line_font_size, next_line_font_size]
-                if other_font_size is not None
-            )
-
-        def __is_similar_to_pre_line(curr_line_font_type, prev_line_font_type, curr_line_font_size, prev_line_font_size):
-            """
-            This function checks if the current line is similar to the previous line
-
-            Parameters
-            ----------
-            curr_line : dict
-                current line
-            prev_line : dict
-                previous line
-
-            Returns
-            -------
-            bool
-                True if the current line is similar to the previous line, False otherwise.
-            """
-
-            if curr_line_font_type == prev_line_font_type and curr_line_font_size == prev_line_font_size:
-                return True
-            else:
-                return False
-
-        def __is_same_font_type_of_docAvg(curr_line_font_type):
-            """
-            This function checks if the current line has the same font type as the document average font type
-
-            Parameters
-            ----------
-            curr_line_font_type : str
-                font type of the current line
-
-            Returns
-            -------
-            bool
-                True if the current line has the same font type as the document average font type, False otherwise.
-            """
-            doc_most_common_font_type = safe_get(self.doc_statistics, "most_common_font_type", "").lower()
-            doc_second_most_common_font_type = safe_get(self.doc_statistics, "second_most_common_font_type", "").lower()
-
-            return curr_line_font_type.lower() in [doc_most_common_font_type, doc_second_most_common_font_type]
-
-        def __is_font_size_not_less_than_docAvg(curr_line_font_size, ratio: float = 1):
-            """
-            This function checks if the current line has a large enough font size
-
-            Parameters
-            ----------
-            curr_line_font_size : float
-                font size of the current line
-            ratio : float
-                ratio of the current line font size to the document average font size
-
-            Returns
-            -------
-            bool
-                True if the current line has a large enough font size, False otherwise.
-            """
-            doc_most_common_font_size = safe_get(self.doc_statistics, "most_common_font_size", 0)
-            doc_second_most_common_font_size = safe_get(self.doc_statistics, "second_most_common_font_size", 0)
-            doc_avg_font_size = min(doc_most_common_font_size, doc_second_most_common_font_size)
-
-            return curr_line_font_size >= doc_avg_font_size * ratio
-
-        def __is_sufficient_spacing_above_and_below(
-            curr_line_bbox,
-            prev_line_bbox,
-            next_line_bbox,
-            avg_char_height,
-            median_font_size,
-        ):
-            """
-            This function checks if the current line has sufficient spacing above and below
-
-            Parameters
-            ----------
-            curr_line_bbox : list
-                bbox of the current line
-            prev_line_bbox : list
-                bbox of the previous line
-            next_line_bbox : list
-                bbox of the next line
-            avg_char_width : float
-                average of char widths
-            avg_char_height : float
-                average of line heights
-
-            Returns
-            -------
-            bool
-                True if the current line has sufficient spacing above and below, False otherwise.
-            """
-            vertical_ratio = 1.25
-            vertical_thres = vertical_ratio * median_font_size
-
-            _, y0, _, y1 = curr_line_bbox
-
-            sufficient_spacing_above = False
-            if prev_line_bbox:
-                vertical_spacing_above = min(y0 - prev_line_bbox[1], y1 - prev_line_bbox[3])
-                sufficient_spacing_above = vertical_spacing_above > vertical_thres
-            else:
-                sufficient_spacing_above = True
-
-            sufficient_spacing_below = False
-            if next_line_bbox:
-                vertical_spacing_below = min(next_line_bbox[1] - y0, next_line_bbox[3] - y1)
-                sufficient_spacing_below = vertical_spacing_below > vertical_thres
-            else:
-                sufficient_spacing_below = True
-
-            return (sufficient_spacing_above, sufficient_spacing_below)
-
-        def __is_word_list_line_by_rules(curr_line_text):
-            """
-            This function checks if the current line is a word list
-
-            Parameters
-            ----------
-            curr_line_text : str
-                text of the current line
-
-            Returns
-            -------
-            bool
-                True if the current line is a name list, False otherwise.
-            """
-            # name_list_pattern = r"([a-zA-Z][a-zA-Z\s]{0,20}[a-zA-Z]|[\u4e00-\u9fa5·]{2,16})(?=[，,;；\s]|$)"
-            name_list_pattern = r"(?<![\u4e00-\u9fa5])([A-Z][a-z]{0,19}\s[A-Z][a-z]{0,19}|[\u4e00-\u9fa5]{2,6})(?=[，,;；\s]|$)"
-
-            compiled_pattern = re.compile(name_list_pattern)
-
-            if compiled_pattern.search(curr_line_text):
-                return True
-            else:
-                return False
-
-        def __get_text_catgr_by_nlp(curr_line_text):
-            """
-            This function checks if the current line is a name list using nlp model, such as spacy
-
-            Parameters
-            ----------
-            curr_line_text : str
-                text of the current line
-
-            Returns
-            -------
-            bool
-                True if the current line is a name list, False otherwise.
-            """
-
-            result = self.nlp_model.detect_entity_catgr_using_nlp(curr_line_text)
-
-            return result
-
-        def __is_numbered_title(curr_line_text):
-            """
-            This function checks if the current line is a numbered list
-
-            Parameters
-            ----------
-            curr_line_text : str
-                text of the current line
-
-            Returns
-            -------
-            bool
-                True if the current line is a numbered list, False otherwise.
-            """
-
-            compiled_pattern = re.compile(self.numbered_title_pattern, re.VERBOSE)
-
-            if compiled_pattern.search(curr_line_text):
-                return True
-            else:
-                return False
-
-        def __is_end_with_ending_puncs(line_text):
-            """
-            This function checks if the current line ends with a ending punctuation mark
-
-            Parameters
-            ----------
-            line_text : str
-                text of the current line
-
-            Returns
-            -------
-            bool
-                True if the current line ends with a punctuation mark, False otherwise.
-            """
-            end_puncs = [".", "?", "!", "。", "？", "！", "…"]
-
-            line_text = line_text.rstrip()
-            if line_text[-1] in end_puncs:
-                return True
-
-            return False
-
-        def __contains_only_no_meaning_symbols(line_text):
-            """
-            This function checks if the current line contains only symbols that have no meaning, if so, it is not a title.
-            Situation contains:
-            1. Only have punctuation marks
-            2. Only have other non-meaning symbols
-
-            Parameters
-            ----------
-            line_text : str
-                text of the current line
-
-            Returns
-            -------
-            bool
-                True if the current line contains only symbols that have no meaning, False otherwise.
-            """
-
-            punctuation_marks = re.findall(r"[^\w\s]", line_text)  # find all punctuation marks
-            number_of_punctuation_marks = len(punctuation_marks)
-
-            text_length = len(line_text)
-
-            if text_length == 0:
-                return False
-
-            punctuation_ratio = number_of_punctuation_marks / text_length
-            if punctuation_ratio >= 0.9:
-                return True
-
-            return False
-
-        def __is_equation(line_text):
-            """
-            This function checks if the current line is an equation.
-
-            Parameters
-            ----------
-            line_text : str
-
-            Returns
-            -------
-            bool
-                True if the current line is an equation, False otherwise.
-            """
-            equation_reg = r"\$.*?\\overline.*?\$"  # to match interline equations
-
-            if re.search(equation_reg, line_text):
-                return True
-            else:
-                return False
-
-        def __is_title_by_len(text, max_length=200):
-            """
-            This function checks if the current line is a title by length.
-
-            Parameters
-            ----------
-            text : str
-                text of the current line
-
-            max_length : int
-                max length of the title
-
-            Returns
-            -------
-            bool
-                True if the current line is a title, False otherwise.
-
-            """
-            text = text.strip()
-            return len(text) <= max_length
-
-        def __compute_line_font_type_and_size(curr_line):
-            """
-            This function computes the font type and font size of the line.
-
-            Parameters
-            ----------
-            line : dict
-                line
-
-            Returns
-            -------
-            font_type : str
-                font type of the line
-            font_size : float
-                font size of the line
-            """
-            spans = curr_line["spans"]
-            max_accumulated_length = 0
-            max_span_font_size = curr_line["spans"][0]["size"]  # default value, float type
-            max_span_font_type = curr_line["spans"][0]["font"].lower()  # default value, string type
-            for span in spans:
-                if span["text"].isspace():
-                    continue
-                span_length = span["bbox"][2] - span["bbox"][0]
-                if span_length > max_accumulated_length:
-                    max_accumulated_length = span_length
-                    max_span_font_size = span["size"]
-                    max_span_font_type = span["font"].lower()
-
-            return max_span_font_type, max_span_font_size
-
-        def __is_a_consistent_sub_title(pre_line, curr_line):
-            """
-            This function checks if the current line is a consistent sub title.
-
-            Parameters
-            ----------
-            pre_line : dict
-                previous line
-            curr_line : dict
-                current line
-
-            Returns
-            -------
-            bool
-                True if the current line is a consistent sub title, False otherwise.
-            """
-            if pre_line is None:
-                return False
-
-            start_letter_of_pre_line = pre_line["text"][0]
-            start_letter_of_curr_line = curr_line["text"][0]
-
-            has_same_prefix_digit = (
-                start_letter_of_pre_line.isdigit()
-                and start_letter_of_curr_line.isdigit()
-                and start_letter_of_pre_line == start_letter_of_curr_line
-            )
-
-            # prefix text of curr_line satisfies the following title format: x.x
-            prefix_text_pattern = r"^\d+\.\d+"
-            has_subtitle_format = re.match(prefix_text_pattern, curr_line["text"])
-
-            if has_same_prefix_digit or has_subtitle_format:
-                return True
-
-        """
-        Title detecting main Process.
-        """
-
-        """
-        Basic features about the current line.
-        """
-        curr_line_bbox = curr_line["bbox"]
-        curr_line_text = curr_line["text"]
-        curr_line_font_type, curr_line_font_size = __compute_line_font_type_and_size(curr_line)
-
-        if len(curr_line_text.strip()) == 0:  # skip empty lines
-            return False, False
-
-        prev_line_bbox = prev_line["bbox"] if prev_line else None
-        if prev_line:
-            prev_line_font_type, prev_line_font_size = __compute_line_font_type_and_size(prev_line)
-        else:
-            prev_line_font_type, prev_line_font_size = None, None
-
-        next_line_bbox = next_line["bbox"] if next_line else None
-        if next_line:
-            next_line_font_type, next_line_font_size = __compute_line_font_type_and_size(next_line)
-        else:
-            next_line_font_type, next_line_font_size = None, None
-
-        """
-        Aggregated features about the current line.
-        """
-        is_italc_font = __is_italic_font_line(curr_line)
-        is_bold_font = __is_bold_font_line(curr_line)
-
-        is_font_size_little_less_than_doc_avg = __is_font_size_not_less_than_docAvg(curr_line_font_size, ratio=0.8)
-        is_font_size_not_less_than_doc_avg = __is_font_size_not_less_than_docAvg(curr_line_font_size, ratio=1)
-        is_much_larger_font_than_doc_avg = __is_font_size_not_less_than_docAvg(curr_line_font_size, ratio=1.6)
-
-        is_not_same_font_type_of_docAvg = not __is_same_font_type_of_docAvg(curr_line_font_type)
-
-        is_potential_title_font = is_bold_font or is_font_size_not_less_than_doc_avg or is_not_same_font_type_of_docAvg
-
-        is_mix_font_styles_strict = __has_mixed_font_styles(curr_line["spans"], strict_mode=True)
-        is_mix_font_styles_loose = __has_mixed_font_styles(curr_line["spans"], strict_mode=False)
-
-        is_punctuation_heavy = __is_punctuation_heavy(curr_line_text)
-
-        is_word_list_line_by_rules = __is_word_list_line_by_rules(curr_line_text)
-        is_person_or_org_list_line_by_nlp = __get_text_catgr_by_nlp(curr_line_text) in ["PERSON", "GPE", "ORG"]
-
-        is_font_size_larger_than_neighbors = __is_larger_font_size_from_neighbors(
-            curr_line_font_size, prev_line_font_size, next_line_font_size
-        )
-
-        is_font_type_diff_from_neighbors = __is_different_font_type_from_neighbors(
-            curr_line_font_type, prev_line_font_type, next_line_font_type
-        )
-
-        has_sufficient_spaces_above, has_sufficient_spaces_below = __is_sufficient_spacing_above_and_below(
-            curr_line_bbox, prev_line_bbox, next_line_bbox, avg_char_height, median_font_size
-        )
-
-        is_similar_to_pre_line = __is_similar_to_pre_line(
-            curr_line_font_type, prev_line_font_type, curr_line_font_size, prev_line_font_size
-        )
-
-        is_consis_sub_title = __is_a_consistent_sub_title(prev_line, curr_line)
-
-        """
-        Further aggregated features about the current line.
-        
-        Attention:
-            Features that start with __ are for internal use.
-        """
-
-        __is_line_left_aligned_from_neighbors = is_line_left_aligned_from_neighbors(
-            curr_line_bbox, prev_line_bbox, next_line_bbox, avg_char_width
-        )
-        __is_font_diff_from_neighbors = is_font_size_larger_than_neighbors or is_font_type_diff_from_neighbors
-        is_a_left_inline_title = (
-            is_mix_font_styles_strict and __is_line_left_aligned_from_neighbors and __is_font_diff_from_neighbors
-        )
-
-        is_title_by_check_prev_line = prev_line is None and has_sufficient_spaces_above and is_potential_title_font
-        is_title_by_check_next_line = next_line is None and has_sufficient_spaces_below and is_potential_title_font
-
-        is_title_by_check_pre_and_next_line = (
-            (prev_line is not None or next_line is not None)
-            and has_sufficient_spaces_above
-            and has_sufficient_spaces_below
-            and is_potential_title_font
-        )
-
-        is_numbered_title = __is_numbered_title(curr_line_text) and (
-            (has_sufficient_spaces_above or prev_line is None) and (has_sufficient_spaces_below or next_line is None)
-        )
-
-        is_not_end_with_ending_puncs = not __is_end_with_ending_puncs(curr_line_text)
-
-        is_not_only_no_meaning_symbols = not __contains_only_no_meaning_symbols(curr_line_text)
-
-        is_equation = __is_equation(curr_line_text)
-
-        is_title_by_len = __is_title_by_len(curr_line_text)
-
-        """
-        Decide if the line is a title.
-        """
-
-        is_title = (
-            is_not_end_with_ending_puncs  # not end with ending punctuation marks
-            and is_not_only_no_meaning_symbols  # not only have no meaning symbols
-            and is_title_by_len  # is a title by length, default max length is 200
-            and not is_equation  # an interline equation should never be a title
-            and is_potential_title_font  # is a potential title font, which is bold or larger than the document average font size or not the same font type as the document average font type
-            and (
-                (is_not_same_font_type_of_docAvg and is_font_size_not_less_than_doc_avg)
-                or (is_bold_font and is_much_larger_font_than_doc_avg and is_not_same_font_type_of_docAvg)
-                or (
-                    is_much_larger_font_than_doc_avg
-                    and (is_title_by_check_prev_line or is_title_by_check_next_line or is_title_by_check_pre_and_next_line)
-                )
-                or (
-                    is_font_size_little_less_than_doc_avg
-                    and is_bold_font
-                    and (is_title_by_check_prev_line or is_title_by_check_next_line or is_title_by_check_pre_and_next_line)
-                )
-            )  # Consider the following situations: bold font, much larger font than doc avg, not same font type as doc avg, sufficient spacing above and below
-            and (
-                (
-                    not is_person_or_org_list_line_by_nlp
-                    and (
-                        is_much_larger_font_than_doc_avg
-                        or (is_not_same_font_type_of_docAvg and is_font_size_not_less_than_doc_avg)
-                    )
-                )
-                or (
-                    not (is_word_list_line_by_rules and is_person_or_org_list_line_by_nlp)
-                    and not is_a_left_inline_title
-                    and not is_punctuation_heavy
-                    and (is_title_by_check_prev_line or is_title_by_check_next_line or is_title_by_check_pre_and_next_line)
-                )
-                or (
-                    is_person_or_org_list_line_by_nlp
-                    and (is_bold_font and is_much_larger_font_than_doc_avg and is_not_same_font_type_of_docAvg)
-                    and (is_bold_font and is_much_larger_font_than_doc_avg and is_not_same_font_type_of_docAvg)
-                )
-                or (is_numbered_title and not is_a_left_inline_title)
-            )  # Exclude the following situations: person/org list
-        )
-        # ) or (prev_line_is_title and is_consis_sub_title)
-
-        is_name_or_org_list_to_be_removed = (
-            (is_person_or_org_list_line_by_nlp)
-            and is_punctuation_heavy
-            and (is_title_by_check_prev_line or is_title_by_check_next_line or is_title_by_check_pre_and_next_line)
-        ) and not is_title
-
-        if is_name_or_org_list_to_be_removed:
-            is_author_or_org_list = True
-        else:
-            is_author_or_org_list = False
-
-        # return is_title, is_author_or_org_list
-
-        """
-        # print reason why the line is a title
-        if is_title:
-            print_green("This line is a title.")
-            print_green("↓" * 10)
-            print()
-            print("curr_line_text: ", curr_line_text)
-            print()
-
-        # print reason why the line is not a title
-        line_text = curr_line_text.strip()
-        test_text = "Career/Personal Life"
-        text_content_condition = line_text == test_text
-        
-        if not is_title and text_content_condition: # Print specific line
-        # if not is_title: # Print each line
-            print_red("This line is not a title.")
-            print_red("↓" * 10)
-
-            print()
-            print("curr_line_text: ", curr_line_text)
-            print()
-
-            if is_not_end_with_ending_puncs:
-                print_green(f"is_not_end_with_ending_puncs")
-            else:
-                print_red(f"is_end_with_ending_puncs")
-
-            if is_not_only_no_meaning_symbols:
-                print_green(f"is_not_only_no_meaning_symbols")
-            else:
-                print_red(f"is_only_no_meaning_symbols")
-
-            if is_title_by_len:
-                print_green(f"is_title_by_len: {is_title_by_len}")
-            else:
-                print_red(f"is_not_title_by_len: {is_title_by_len}")
-
-            if is_equation:
-                print_red(f"is_equation")
-            else:
-                print_green(f"is_not_equation")
-
-            if is_potential_title_font:
-                print_green(f"is_potential_title_font")
-            else:
-                print_red(f"is_not_potential_title_font")
-
-            if is_punctuation_heavy:
-                print_red("is_punctuation_heavy")
-            else:
-                print_green("is_not_punctuation_heavy")
-
-            if is_bold_font:
-                print_green(f"is_bold_font")
-            else:
-                print_red(f"is_not_bold_font")
-
-            if is_font_size_not_less_than_doc_avg:
-                print_green(f"is_larger_font_than_doc_avg")
-            else:
-                print_red(f"is_not_larger_font_than_doc_avg")
-
-            if is_much_larger_font_than_doc_avg:
-                print_green(f"is_much_larger_font_than_doc_avg")
-            else:
-                print_red(f"is_not_much_larger_font_than_doc_avg")
-
-            if is_not_same_font_type_of_docAvg:
-                print_green(f"is_not_same_font_type_of_docAvg")
-            else:
-                print_red(f"is_same_font_type_of_docAvg")
-
-            if is_word_list_line_by_rules:
-                print_red("is_word_list_line_by_rules")
-            else:
-                print_green("is_not_name_list_by_rules")
-
-            if is_person_or_org_list_line_by_nlp:
-                print_red("is_person_or_org_list_line_by_nlp")
-            else:
-                print_green("is_not_person_or_org_list_line_by_nlp")
-
-            if not is_numbered_title:
-                print_red("is_not_numbered_title")
-            else:
-                print_green("is_numbered_title")
-
-            if is_a_left_inline_title:
-                print_red("is_a_left_inline_title")
-            else:
-                print_green("is_not_a_left_inline_title")
-
-            if not is_title_by_check_prev_line:
-                print_red("is_not_title_by_check_prev_line")
-            else:
-                print_green("is_title_by_check_prev_line")
-
-            if not is_title_by_check_next_line:
-                print_red("is_not_title_by_check_next_line")
-            else:
-                print_green("is_title_by_check_next_line")
-
-            if not is_title_by_check_pre_and_next_line:
-                print_red("is_not_title_by_check_pre_and_next_line")
-            else:
-                print_green("is_title_by_check_pre_and_next_line")
-
-        # print_green("Common features:")
-        # print_green("↓" * 10)
-
-        # print(f"    curr_line_font_type: {curr_line_font_type}")
-        # print(f"    curr_line_font_size: {curr_line_font_size}")
-        # print()
-
-        """
-
-        return is_title, is_author_or_org_list
-
-    def _detect_title(self, input_block):
-        """
-        Use the functions 'is_potential_title' to detect titles of each paragraph block.
-        If a line is a title, then the value of key 'is_title' of the line will be set to True.
-        """
-
-        raw_lines = input_block["lines"]
-
-        prev_line_is_title_flag = False
-
-        for i, curr_line in enumerate(raw_lines):
-            prev_line = raw_lines[i - 1] if i > 0 else None
-            next_line = raw_lines[i + 1] if i < len(raw_lines) - 1 else None
-
-            blk_avg_char_width = input_block["avg_char_width"]
-            blk_avg_char_height = input_block["avg_char_height"]
-            blk_media_font_size = input_block["median_font_size"]
-
-            is_title, is_author_or_org_list = self._is_potential_title(
-                curr_line,
-                prev_line,
-                prev_line_is_title_flag,
-                next_line,
-                blk_avg_char_width,
-                blk_avg_char_height,
-                blk_media_font_size,
-            )
-
-            if is_title:
-                curr_line["is_title"] = is_title
-                prev_line_is_title_flag = True
-            else:
-                curr_line["is_title"] = False
-                prev_line_is_title_flag = False
-
-            # print(f"curr_line['text']: {curr_line['text']}")
-            # print(f"curr_line['is_title']: {curr_line['is_title']}")
-            # print(f"prev_line['text']: {prev_line['text'] if prev_line else None}")
-            # print(f"prev_line_is_title_flag: {prev_line_is_title_flag}")
-            # print()
-
-            if is_author_or_org_list:
-                curr_line["is_author_or_org_list"] = is_author_or_org_list
-            else:
-                curr_line["is_author_or_org_list"] = False
-
-        return input_block
-
-    def batch_detect_titles(self, pdf_dic):
-        """
-        This function batch process the blocks to detect titles.
-
-        Parameters
-        ----------
-        pdf_dict : dict
-            result dictionary
-
-        Returns
-        -------
-        pdf_dict : dict
-            result dictionary
-        """
-        num_titles = 0
-
-        for page_id, blocks in pdf_dic.items():
-            if page_id.startswith("page_"):
-                para_blocks = []
-                if "para_blocks" in blocks.keys():
-                    para_blocks = blocks["para_blocks"]
-
-                    all_single_line_blocks = []
-                    for block in para_blocks:
-                        if len(block["lines"]) == 1:
-                            all_single_line_blocks.append(block)
-
-                    new_para_blocks = []
-                    if not len(all_single_line_blocks) == len(para_blocks):  # Not all blocks are single line blocks.
-                        for para_block in para_blocks:
-                            new_block = self._detect_title(para_block)
-                            new_para_blocks.append(new_block)
-                            num_titles += sum([line.get("is_title", 0) for line in new_block["lines"]])
-                    else:  # All blocks are single line blocks.
-                        for para_block in para_blocks:
-                            new_para_blocks.append(para_block)
-                            num_titles += sum([line.get("is_title", 0) for line in para_block["lines"]])
-                    para_blocks = new_para_blocks
-
-                blocks["para_blocks"] = para_blocks
-
-                for para_block in para_blocks:
-                    all_titles = all(safe_get(line, "is_title", False) for line in para_block["lines"])
-                    para_text_len = sum([len(line["text"]) for line in para_block["lines"]])
-                    if (
-                        all_titles and para_text_len < 200
-                    ):  # total length of the paragraph is less than 200, more than this should not be a title
-                        para_block["is_block_title"] = 1
-                    else:
-                        para_block["is_block_title"] = 0
-
-                    all_name_or_org_list_to_be_removed = all(
-                        safe_get(line, "is_author_or_org_list", False) for line in para_block["lines"]
-                    )
-                    if all_name_or_org_list_to_be_removed and page_id == "page_0":
-                        para_block["is_block_an_author_or_org_list"] = 1
-                    else:
-                        para_block["is_block_an_author_or_org_list"] = 0
-
-        pdf_dic["statistics"]["num_titles"] = num_titles
-
-        return pdf_dic
-
-    def _recog_title_level(self, title_blocks):
-        """
-        This function determines the title level based on the font size of the title.
-
-        Parameters
-        ----------
-        title_blocks : list
-
-        Returns
-        -------
-        title_blocks : list
-        """
-
-        font_sizes = np.array([safe_get(tb["block"], "block_font_size", 0) for tb in title_blocks])
-
-        # Use the mean and std of font sizes to remove extreme values
-        mean_font_size = np.mean(font_sizes)
-        std_font_size = np.std(font_sizes)
-        min_extreme_font_size = mean_font_size - std_font_size  # type: ignore
-        max_extreme_font_size = mean_font_size + std_font_size  # type: ignore
-
-        # Compute the threshold for title level
-        middle_font_sizes = font_sizes[(font_sizes > min_extreme_font_size) & (font_sizes < max_extreme_font_size)]
-        if middle_font_sizes.size > 0:
-            middle_mean_font_size = np.mean(middle_font_sizes)
-            level_threshold = middle_mean_font_size
-        else:
-            level_threshold = mean_font_size
-
-        for tb in title_blocks:
-            title_block = tb["block"]
-            title_font_size = safe_get(title_block, "block_font_size", 0)
-
-            current_level = 1  # Initialize title level, the biggest level is 1
-
-            # print(f"Before adjustment by font size, {current_level}")
-            if title_font_size >= max_extreme_font_size:
-                current_level = 1
-            elif title_font_size <= min_extreme_font_size:
-                current_level = 3
-            elif float(title_font_size) >= float(level_threshold):
-                current_level = 2
-            else:
-                current_level = 3
-            # print(f"After adjustment by font size, {current_level}")
-
-            title_block["block_title_level"] = current_level
-
-        return title_blocks
-
-    def batch_recog_title_level(self, pdf_dic):
-        """
-        This function batch process the blocks to recognize title level.
-
-        Parameters
-        ----------
-        pdf_dict : dict
-            result dictionary
-
-        Returns
-        -------
-        pdf_dict : dict
-            result dictionary
-        """
-        title_blocks = []
-
-        # Collect all titles
-        for page_id, blocks in pdf_dic.items():
-            if page_id.startswith("page_"):
-                para_blocks = blocks.get("para_blocks", [])
-                for block in para_blocks:
-                    if block.get("is_block_title"):
-                        title_obj = {"page_id": page_id, "block": block}
-                        title_blocks.append(title_obj)
-
-        # Determine title level
-        if title_blocks:
-            # Determine title level based on font size
-            title_blocks = self._recog_title_level(title_blocks)
-
-        return pdf_dic
-
-
-class BlockTerminationProcessor:
-    """
-    This class is used to process the block termination.
-    """
-
-    def __init__(self) -> None:
-        pass
-
-    def _is_consistent_lines(
-        self,
-        curr_line,
-        prev_line,
-        next_line,
-        consistent_direction,  # 0 for prev, 1 for next, 2 for both
-    ):
-        """
-        This function checks if the line is consistent with its neighbors
-
-        Parameters
-        ----------
-        curr_line : dict
-            current line
-        prev_line : dict
-            previous line
-        next_line : dict
-            next line
-        consistent_direction : int
-            0 for prev, 1 for next, 2 for both
-
-        Returns
-        -------
-        bool
-            True if the line is consistent with its neighbors, False otherwise.
-        """
-
-        curr_line_font_size = curr_line["spans"][0]["size"]
-        curr_line_font_type = curr_line["spans"][0]["font"].lower()
-
-        if consistent_direction == 0:
-            if prev_line:
-                prev_line_font_size = prev_line["spans"][0]["size"]
-                prev_line_font_type = prev_line["spans"][0]["font"].lower()
-                return curr_line_font_size == prev_line_font_size and curr_line_font_type == prev_line_font_type
-            else:
-                return False
-
-        elif consistent_direction == 1:
-            if next_line:
-                next_line_font_size = next_line["spans"][0]["size"]
-                next_line_font_type = next_line["spans"][0]["font"].lower()
-                return curr_line_font_size == next_line_font_size and curr_line_font_type == next_line_font_type
-            else:
-                return False
-
-        elif consistent_direction == 2:
-            if prev_line and next_line:
-                prev_line_font_size = prev_line["spans"][0]["size"]
-                prev_line_font_type = prev_line["spans"][0]["font"].lower()
-                next_line_font_size = next_line["spans"][0]["size"]
-                next_line_font_type = next_line["spans"][0]["font"].lower()
-                return (curr_line_font_size == prev_line_font_size and curr_line_font_type == prev_line_font_type) and (
-                    curr_line_font_size == next_line_font_size and curr_line_font_type == next_line_font_type
-                )
-            else:
-                return False
-
-        else:
-            return False
-
-    def _is_regular_line(self, curr_line_bbox, prev_line_bbox, next_line_bbox, avg_char_width, X0, X1, avg_line_height):
-        """
-        This function checks if the line is a regular line
-
-        Parameters
-        ----------
-        curr_line_bbox : list
-            bbox of the current line
-        prev_line_bbox : list
-            bbox of the previous line
-        next_line_bbox : list
-            bbox of the next line
-        avg_char_width : float
-            average of char widths
-        X0 : float
-            median of x0 values, which represents the left average boundary of the page
-        X1 : float
-            median of x1 values, which represents the right average boundary of the page
-        avg_line_height : float
-            average of line heights
-
-        Returns
-        -------
-        bool
-            True if the line is a regular line, False otherwise.
-        """
-        horizontal_ratio = 0.5
-        vertical_ratio = 0.5
-        horizontal_thres = horizontal_ratio * avg_char_width
-        vertical_thres = vertical_ratio * avg_line_height
-
-        x0, y0, x1, y1 = curr_line_bbox
-
-        x0_near_X0 = abs(x0 - X0) < horizontal_thres
-        x1_near_X1 = abs(x1 - X1) < horizontal_thres
-
-        prev_line_is_end_of_para = prev_line_bbox and (abs(prev_line_bbox[2] - X1) > avg_char_width)
-
-        sufficient_spacing_above = False
-        if prev_line_bbox:
-            vertical_spacing_above = y1 - prev_line_bbox[3]
-            sufficient_spacing_above = vertical_spacing_above > vertical_thres
-
-        sufficient_spacing_below = False
-        if next_line_bbox:
-            vertical_spacing_below = next_line_bbox[1] - y0
-            sufficient_spacing_below = vertical_spacing_below > vertical_thres
-
-        return (
-            (sufficient_spacing_above or sufficient_spacing_below)
-            or (not x0_near_X0 and not x1_near_X1)
-            or prev_line_is_end_of_para
-        )
-
-    def _is_possible_start_of_para(self, curr_line, prev_line, next_line, X0, X1, avg_char_width, avg_font_size):
-        """
-        This function checks if the line is a possible start of a paragraph
-
-        Parameters
-        ----------
-        curr_line : dict
-            current line
-        prev_line : dict
-            previous line
-        next_line : dict
-            next line
-        X0 : float
-            median of x0 values, which represents the left average boundary of the page
-        X1 : float
-            median of x1 values, which represents the right average boundary of the page
-        avg_char_width : float
-            average of char widths
-        avg_line_height : float
-            average of line heights
-
-        Returns
-        -------
-        bool
-            True if the line is a possible start of a paragraph, False otherwise.
-        """
-        start_confidence = 0.5  # Initial confidence of the line being a start of a paragraph
-        decision_path = []  # Record the decision path
-
-        curr_line_bbox = curr_line["bbox"]
-        prev_line_bbox = prev_line["bbox"] if prev_line else None
-        next_line_bbox = next_line["bbox"] if next_line else None
-
-        indent_ratio = 1
-
-        vertical_ratio = 1.5
-        vertical_thres = vertical_ratio * avg_font_size
-
-        left_horizontal_ratio = 0.5
-        left_horizontal_thres = left_horizontal_ratio * avg_char_width
-
-        right_horizontal_ratio = 2.5
-        right_horizontal_thres = right_horizontal_ratio * avg_char_width
-
-        x0, y0, x1, y1 = curr_line_bbox
-
-        indent_condition = x0 > X0 + indent_ratio * avg_char_width
-        if indent_condition:
-            start_confidence += 0.2
-            decision_path.append("indent_condition_met")
-
-        x0_near_X0 = abs(x0 - X0) < left_horizontal_thres
-        if x0_near_X0:
-            start_confidence += 0.1
-            decision_path.append("x0_near_X0")
-
-        x1_near_X1 = abs(x1 - X1) < right_horizontal_thres
-        if x1_near_X1:
-            start_confidence += 0.1
-            decision_path.append("x1_near_X1")
-
-        if prev_line is None:
-            prev_line_is_end_of_para = True
-            start_confidence += 0.2
-            decision_path.append("no_prev_line")
-        else:
-            prev_line_is_end_of_para, _, _ = self._is_possible_end_of_para(prev_line, next_line, X0, X1, avg_char_width)
-            if prev_line_is_end_of_para:
-                start_confidence += 0.1
-                decision_path.append("prev_line_is_end_of_para")
-
-        sufficient_spacing_above = False
-        if prev_line_bbox:
-            vertical_spacing_above = y1 - prev_line_bbox[3]
-            sufficient_spacing_above = vertical_spacing_above > vertical_thres
-            if sufficient_spacing_above:
-                start_confidence += 0.2
-                decision_path.append("sufficient_spacing_above")
-
-        sufficient_spacing_below = False
-        if next_line_bbox:
-            vertical_spacing_below = next_line_bbox[1] - y0
-            sufficient_spacing_below = vertical_spacing_below > vertical_thres
-            if sufficient_spacing_below:
-                start_confidence += 0.2
-                decision_path.append("sufficient_spacing_below")
-
-        is_regular_line = self._is_regular_line(
-            curr_line_bbox, prev_line_bbox, next_line_bbox, avg_char_width, X0, X1, avg_font_size
-        )
-        if is_regular_line:
-            start_confidence += 0.1
-            decision_path.append("is_regular_line")
-
-        is_start_of_para = (
-            (sufficient_spacing_above or sufficient_spacing_below)
-            or (indent_condition)
-            or (not indent_condition and x0_near_X0 and x1_near_X1 and not is_regular_line)
-            or prev_line_is_end_of_para
-        )
-        return (is_start_of_para, start_confidence, decision_path)
-
-    def _is_possible_end_of_para(self, curr_line, next_line, X0, X1, avg_char_width):
-        """
-        This function checks if the line is a possible end of a paragraph
-
-        Parameters
-        ----------
-        curr_line : dict
-            current line
-        next_line : dict
-            next line
-        X0 : float
-            median of x0 values, which represents the left average boundary of the page
-        X1 : float
-            median of x1 values, which represents the right average boundary of the page
-        avg_char_width : float
-            average of char widths
-
-        Returns
-        -------
-        bool
-            True if the line is a possible end of a paragraph, False otherwise.
-        """
-
-        end_confidence = 0.5  # Initial confidence of the line being a end of a paragraph
-        decision_path = []  # Record the decision path
-
-        curr_line_bbox = curr_line["bbox"]
-        next_line_bbox = next_line["bbox"] if next_line else None
-
-        left_horizontal_ratio = 0.5
-        right_horizontal_ratio = 0.5
-
-        x0, _, x1, y1 = curr_line_bbox
-        next_x0, next_y0, _, _ = next_line_bbox if next_line_bbox else (0, 0, 0, 0)
-
-        x0_near_X0 = abs(x0 - X0) < left_horizontal_ratio * avg_char_width
-        if x0_near_X0:
-            end_confidence += 0.1
-            decision_path.append("x0_near_X0")
-
-        x1_smaller_than_X1 = x1 < X1 - right_horizontal_ratio * avg_char_width
-        if x1_smaller_than_X1:
-            end_confidence += 0.1
-            decision_path.append("x1_smaller_than_X1")
-
-        next_line_is_start_of_para = (
-            next_line_bbox
-            and (next_x0 > X0 + left_horizontal_ratio * avg_char_width)
-            and (not is_line_left_aligned_from_neighbors(curr_line_bbox, None, next_line_bbox, avg_char_width, direction=1))
-        )
-        if next_line_is_start_of_para:
-            end_confidence += 0.2
-            decision_path.append("next_line_is_start_of_para")
-
-        is_line_left_aligned_from_neighbors_bool = is_line_left_aligned_from_neighbors(
-            curr_line_bbox, None, next_line_bbox, avg_char_width
-        )
-        if is_line_left_aligned_from_neighbors_bool:
-            end_confidence += 0.1
-            decision_path.append("line_is_left_aligned_from_neighbors")
-
-        is_line_right_aligned_from_neighbors_bool = is_line_right_aligned_from_neighbors(
-            curr_line_bbox, None, next_line_bbox, avg_char_width
-        )
-        if not is_line_right_aligned_from_neighbors_bool:
-            end_confidence += 0.1
-            decision_path.append("line_is_not_right_aligned_from_neighbors")
-
-        is_end_of_para = end_with_punctuation(curr_line["text"]) and (
-            (x0_near_X0 and x1_smaller_than_X1)
-            or (is_line_left_aligned_from_neighbors_bool and not is_line_right_aligned_from_neighbors_bool)
-        )
-
-        return (is_end_of_para, end_confidence, decision_path)
-
-    def _cut_paras_per_block(
-        self,
-        block,
-    ):
-        """
-        Processes a raw block from PyMuPDF and returns the processed block.
-
-        Parameters
-        ----------
-        raw_block : dict
-            A raw block from pymupdf.
-
-        Returns
-        -------
-        processed_block : dict
-
-        """
-
-        def _construct_para(lines, is_block_title, para_title_level):
-            """
-            Construct a paragraph from given lines.
-            """
-
-            font_sizes = [span["size"] for line in lines for span in line["spans"]]
-            avg_font_size = sum(font_sizes) / len(font_sizes) if font_sizes else 0
-
-            font_colors = [span["color"] for line in lines for span in line["spans"]]
-            most_common_font_color = max(set(font_colors), key=font_colors.count) if font_colors else None
-
-            font_type_lengths = {}
-            for line in lines:
-                for span in line["spans"]:
-                    font_type = span["font"]
-                    bbox_width = span["bbox"][2] - span["bbox"][0]
-                    if font_type in font_type_lengths:
-                        font_type_lengths[font_type] += bbox_width
-                    else:
-                        font_type_lengths[font_type] = bbox_width
-
-            # get the font type with the longest bbox width
-            most_common_font_type = max(font_type_lengths, key=font_type_lengths.get) if font_type_lengths else None  # type: ignore
-
-            para_bbox = calculate_para_bbox(lines)
-            para_text = " ".join(line["text"] for line in lines)
-
-            return {
-                "para_bbox": para_bbox,
-                "para_text": para_text,
-                "para_font_type": most_common_font_type,
-                "para_font_size": avg_font_size,
-                "para_font_color": most_common_font_color,
-                "is_para_title": is_block_title,
-                "para_title_level": para_title_level,
-            }
-
-        block_bbox = block["bbox"]
-        block_text = block["text"]
-        block_lines = block["lines"]
-
-        X0 = safe_get(block, "X0", 0)
-        X1 = safe_get(block, "X1", 0)
-        avg_char_width = safe_get(block, "avg_char_width", 0)
-        avg_char_height = safe_get(block, "avg_char_height", 0)
-        avg_font_size = safe_get(block, "avg_font_size", 0)
-
-        is_block_title = safe_get(block, "is_block_title", False)
-        para_title_level = safe_get(block, "block_title_level", 0)
-
-        # Segment into paragraphs
-        para_ranges = []
-        in_paragraph = False
-        start_idx_of_para = None
-
-        # Create the processed paragraphs
-        processed_paras = {}
-        para_bboxes = []
-        end_idx_of_para = 0
-
-        for line_index, line in enumerate(block_lines):
-            curr_line = line
-            prev_line = block_lines[line_index - 1] if line_index > 0 else None
-            next_line = block_lines[line_index + 1] if line_index < len(block_lines) - 1 else None
-
-            """
-            Start processing paragraphs.
-            """
-
-            # Check if the line is the start of a paragraph
-            is_start_of_para, start_confidence, decision_path = self._is_possible_start_of_para(
-                curr_line, prev_line, next_line, X0, X1, avg_char_width, avg_font_size
-            )
-            if not in_paragraph and is_start_of_para:
-                in_paragraph = True
-                start_idx_of_para = line_index
-
-                # print_green(">>> Start of a paragraph")
-                # print("    curr_line_text: ", curr_line["text"])
-                # print("    start_confidence: ", start_confidence)
-                # print("    decision_path: ", decision_path)
-
-            # Check if the line is the end of a paragraph
-            is_end_of_para, end_confidence, decision_path = self._is_possible_end_of_para(
-                curr_line, next_line, X0, X1, avg_char_width
-            )
-            if in_paragraph and (is_end_of_para or not next_line):
-                para_ranges.append((start_idx_of_para, line_index))
-                start_idx_of_para = None
-                in_paragraph = False
-
-                # print_red(">>> End of a paragraph")
-                # print("    curr_line_text: ", curr_line["text"])
-                # print("    end_confidence: ", end_confidence)
-                # print("    decision_path: ", decision_path)
-
-        # Add the last paragraph if it is not added
-        if in_paragraph and start_idx_of_para is not None:
-            para_ranges.append((start_idx_of_para, len(block_lines) - 1))
-
-        # Process the matched paragraphs
-        for para_index, (start_idx, end_idx) in enumerate(para_ranges):
-            matched_lines = block_lines[start_idx : end_idx + 1]
-            para_properties = _construct_para(matched_lines, is_block_title, para_title_level)
-            para_key = f"para_{len(processed_paras)}"
-            processed_paras[para_key] = para_properties
-            para_bboxes.append(para_properties["para_bbox"])
-            end_idx_of_para = end_idx + 1
-
-        # Deal with the remaining lines
-        if end_idx_of_para < len(block_lines):
-            unmatched_lines = block_lines[end_idx_of_para:]
-            unmatched_properties = _construct_para(unmatched_lines, is_block_title, para_title_level)
-            unmatched_key = f"para_{len(processed_paras)}"
-            processed_paras[unmatched_key] = unmatched_properties
-            para_bboxes.append(unmatched_properties["para_bbox"])
-
-        block["paras"] = processed_paras
-
-        return block
-
-    def batch_process_blocks(self, pdf_dict):
-        """
-        Parses the blocks of all pages.
-
-        Parameters
-        ----------
-        pdf_dict : dict
-            PDF dictionary.
-        filter_blocks : list
-            List of bounding boxes to filter.
-
-        Returns
-        -------
-        result_dict : dict
-            Result dictionary.
-
-        """
-
-        num_paras = 0
-
-        for page_id, page in pdf_dict.items():
-            if page_id.startswith("page_"):
-                para_blocks = []
-                if "para_blocks" in page.keys():
-                    input_blocks = page["para_blocks"]
-                    for input_block in input_blocks:
-                        new_block = self._cut_paras_per_block(input_block)
-                        para_blocks.append(new_block)
-                        num_paras += len(new_block["paras"])
-
-                page["para_blocks"] = para_blocks
-
-        pdf_dict["statistics"]["num_paras"] = num_paras
-        return pdf_dict
-
-
-class BlockContinuationProcessor:
-    """
-    This class is used to process the blocks to detect block continuations.
-    """
-
-    def __init__(self) -> None:
-        pass
-
-    def __is_similar_font_type(self, font_type_1, font_type_2, prefix_length_ratio=0.3):
-        """
-        This function checks if the two font types are similar.
-        Definition of similar font types: the two font types have a common prefix,
-        and the length of the common prefix is at least a certain ratio of the length of the shorter font type.
-
-        Parameters
-        ----------
-        font_type1 : str
-            font type 1
-        font_type2 : str
-            font type 2
-        prefix_length_ratio : float
-            minimum ratio of the common prefix length to the length of the shorter font type
-
-        Returns
-        -------
-        bool
-            True if the two font types are similar, False otherwise.
-        """
-
-        if isinstance(font_type_1, list):
-            font_type_1 = font_type_1[0] if font_type_1 else ""
-        if isinstance(font_type_2, list):
-            font_type_2 = font_type_2[0] if font_type_2 else ""
-
-        if font_type_1 == font_type_2:
-            return True
-
-        # Find the length of the common prefix
-        common_prefix_length = len(os.path.commonprefix([font_type_1, font_type_2]))
-
-        # Calculate the minimum prefix length based on the ratio
-        min_prefix_length = int(min(len(font_type_1), len(font_type_2)) * prefix_length_ratio)
-
-        return common_prefix_length >= min_prefix_length
-
-    def __is_same_block_font(self, block_1, block_2):
-        """
-        This function compares the font of block1 and block2
-
-        Parameters
-        ----------
-        block1 : dict
-            block1
-        block2 : dict
-            block2
-
-        Returns
-        -------
-        is_same : bool
-            True if block1 and block2 have the same font, else False
-        """
-        block_1_font_type = safe_get(block_1, "block_font_type", "")
-        block_1_font_size = safe_get(block_1, "block_font_size", 0)
-        block_1_avg_char_width = safe_get(block_1, "avg_char_width", 0)
-
-        block_2_font_type = safe_get(block_2, "block_font_type", "")
-        block_2_font_size = safe_get(block_2, "block_font_size", 0)
-        block_2_avg_char_width = safe_get(block_2, "avg_char_width", 0)
-
-        if isinstance(block_1_font_size, list):
-            block_1_font_size = block_1_font_size[0] if block_1_font_size else 0
-        if isinstance(block_2_font_size, list):
-            block_2_font_size = block_2_font_size[0] if block_2_font_size else 0
-
-        block_1_text = safe_get(block_1, "text", "")
-        block_2_text = safe_get(block_2, "text", "")
-
-        if block_1_avg_char_width == 0 or block_2_avg_char_width == 0:
-            return False
-
-        if not block_1_text or not block_2_text:
-            return False
-        else:
-            text_len_ratio = len(block_2_text) / len(block_1_text)
-            if text_len_ratio < 0.2:
-                avg_char_width_condition = (
-                    abs(block_1_avg_char_width - block_2_avg_char_width) / min(block_1_avg_char_width, block_2_avg_char_width)
-                    < 0.5
-                )
-            else:
-                avg_char_width_condition = (
-                    abs(block_1_avg_char_width - block_2_avg_char_width) / min(block_1_avg_char_width, block_2_avg_char_width)
-                    < 0.2
-                )
-
-        block_font_size_condition = abs(block_1_font_size - block_2_font_size) < 1
-
-        return (
-            self.__is_similar_font_type(block_1_font_type, block_2_font_type)
-            and avg_char_width_condition
-            and block_font_size_condition
-        )
-
-    def _is_alphabet_char(self, char):
-        if (char >= "\u0041" and char <= "\u005a") or (char >= "\u0061" and char <= "\u007a"):
-            return True
-        else:
-            return False
-
-    def _is_chinese_char(self, char):
-        if char >= "\u4e00" and char <= "\u9fa5":
-            return True
-        else:
-            return False
-
-    def _is_other_letter_char(self, char):
-        try:
-            cat = unicodedata.category(char)
-            if cat == "Lu" or cat == "Ll":
-                return not self._is_alphabet_char(char) and not self._is_chinese_char(char)
-        except TypeError:
-            print("The input to the function must be a single character.")
-        return False
-
-    def _is_year(self, s: str):
-        try:
-            number = int(s)
-            return 1900 <= number <= 2099
-        except ValueError:
-            return False
-
-    def _match_brackets(self, text):
-        # pattern = r"^[\(\)\[\]（）【】{}｛｝<>＜＞〔〕〘〙\"\'“”‘’]"
-        pattern = r"^[\(\)\]（）】{}｛｝>＞〕〙\"\'“”‘’]"
-        return bool(re.match(pattern, text))
-
-    def _is_para_font_consistent(self, para_1, para_2):
-        """
-        This function compares the font of para1 and para2
-
-        Parameters
-        ----------
-        para1 : dict
-            para1
-        para2 : dict
-            para2
-
-        Returns
-        -------
-        is_same : bool
-            True if para1 and para2 have the same font, else False
-        """
-        if para_1 is None or para_2 is None:
-            return False
-
-        para_1_font_type = safe_get(para_1, "para_font_type", "")
-        para_1_font_size = safe_get(para_1, "para_font_size", 0)
-        para_1_font_color = safe_get(para_1, "para_font_color", "")
-
-        para_2_font_type = safe_get(para_2, "para_font_type", "")
-        para_2_font_size = safe_get(para_2, "para_font_size", 0)
-        para_2_font_color = safe_get(para_2, "para_font_color", "")
-
-        if isinstance(para_1_font_type, list):  # get the most common font type
-            para_1_font_type = max(set(para_1_font_type), key=para_1_font_type.count)
-        if isinstance(para_2_font_type, list):
-            para_2_font_type = max(set(para_2_font_type), key=para_2_font_type.count)
-        if isinstance(para_1_font_size, list):  # compute average font type
-            para_1_font_size = sum(para_1_font_size) / len(para_1_font_size)
-        if isinstance(para_2_font_size, list):  # compute average font type
-            para_2_font_size = sum(para_2_font_size) / len(para_2_font_size)
-
-        return (
-            self.__is_similar_font_type(para_1_font_type, para_2_font_type)
-            and abs(para_1_font_size - para_2_font_size) < 1.5
-            # and para_font_color1 == para_font_color2
-        )
-
-    def _is_para_puncs_consistent(self, para_1, para_2):
-        """
-        This function determines whether para1 and para2 are originally from the same paragraph by checking the puncs of para1(former) and para2(latter)
-
-        Parameters
-        ----------
-        para1 : dict
-            para1
-        para2 : dict
-            para2
-
-        Returns
-        -------
-        is_same : bool
-            True if para1 and para2 are from the same paragraph by using the puncs, else False
-        """
-        para_1_text = safe_get(para_1, "para_text", "").strip()
-        para_2_text = safe_get(para_2, "para_text", "").strip()
-
-        para_1_bboxes = safe_get(para_1, "para_bbox", [])
-        para_1_font_sizes = safe_get(para_1, "para_font_size", 0)
-
-        para_2_bboxes = safe_get(para_2, "para_bbox", [])
-        para_2_font_sizes = safe_get(para_2, "para_font_size", 0)
-
-        # print_yellow("    Features of determine puncs_consistent:")
-        # print(f"    para_1_text: {para_1_text}")
-        # print(f"    para_2_text: {para_2_text}")
-        # print(f"    para_1_bboxes: {para_1_bboxes}")
-        # print(f"    para_2_bboxes: {para_2_bboxes}")
-        # print(f"    para_1_font_sizes: {para_1_font_sizes}")
-        # print(f"    para_2_font_sizes: {para_2_font_sizes}")
-
-        if is_nested_list(para_1_bboxes):
-            x0_1, y0_1, x1_1, y1_1 = para_1_bboxes[-1]
-        else:
-            x0_1, y0_1, x1_1, y1_1 = para_1_bboxes
-
-        if is_nested_list(para_2_bboxes):
-            x0_2, y0_2, x1_2, y1_2 = para_2_bboxes[0]
-            para_2_font_sizes = para_2_font_sizes[0]  # type: ignore
-        else:
-            x0_2, y0_2, x1_2, y1_2 = para_2_bboxes
-
-        right_align_threshold = 0.5 * (para_1_font_sizes + para_2_font_sizes) * 0.8
-        are_two_paras_right_aligned = abs(x1_1 - x1_2) < right_align_threshold
-
-        left_indent_threshold = 0.5 * (para_1_font_sizes + para_2_font_sizes) * 0.8
-        is_para1_left_indent_than_papa2 = x0_1 - x0_2 > left_indent_threshold
-        is_para2_left_indent_than_papa1 = x0_2 - x0_1 > left_indent_threshold
-
-        # Check if either para_text1 or para_text2 is empty
-        if not para_1_text or not para_2_text:
-            return False
-
-        # Define the end puncs for a sentence to end and hyphen
-        end_puncs = [".", "?", "!", "。", "？", "！", "…"]
-        hyphen = ["-", "—"]
-
-        # Check if para_text1 ends with either hyphen or non-end punctuation or spaces
-        para_1_end_with_hyphen = para_1_text and para_1_text[-1] in hyphen
-        para_1_end_with_end_punc = para_1_text and para_1_text[-1] in end_puncs
-        para_1_end_with_space = para_1_text and para_1_text[-1] == " "
-        para_1_not_end_with_end_punc = para_1_text and para_1_text[-1] not in end_puncs
-
-        # print_yellow(f"    para_1_end_with_hyphen: {para_1_end_with_hyphen}")
-        # print_yellow(f"    para_1_end_with_end_punc: {para_1_end_with_end_punc}")
-        # print_yellow(f"    para_1_not_end_with_end_punc: {para_1_not_end_with_end_punc}")
-        # print_yellow(f"    para_1_end_with_space: {para_1_end_with_space}")
-
-        if para_1_end_with_hyphen:  # If para_text1 ends with hyphen
-            # print_red(f"para_1 is end with hyphen.")
-            para_2_is_consistent = para_2_text and (
-                para_2_text[0] in hyphen
-                or (self._is_alphabet_char(para_2_text[0]) and para_2_text[0].islower())
-                or (self._is_chinese_char(para_2_text[0]))
-                or (self._is_other_letter_char(para_2_text[0]))
-            )
-            if para_2_is_consistent:
-                # print(f"para_2 is consistent.\n")
-                return True
-            else:
-                # print(f"para_2 is not consistent.\n")
-                pass
-
-        elif para_1_end_with_end_punc:  # If para_text1 ends with ending punctuations
-            # print_red(f"para_1 is end with end_punc.")
-            para_2_is_consistent = (
-                para_2_text
-                and (
-                    para_2_text[0]
-                    == " "
-                    # or (self._is_alphabet_char(para_2_text[0]) and para_2_text[0].isupper())
-                    # or (self._is_chinese_char(para_2_text[0]))
-                    # or (self._is_other_letter_char(para_2_text[0]))
-                )
-                and not is_para2_left_indent_than_papa1
-            )
-            if para_2_is_consistent:
-                # print(f"para_2 is consistent.\n")
-                return True
-            else:
-                # print(f"para_2 is not consistent.\n")
-                pass
-
-        elif para_1_not_end_with_end_punc:  # If para_text1 is not end with ending punctuations
-            # print_red(f"para_1 is NOT end with end_punc.")
-            para_2_is_consistent = para_2_text and (
-                para_2_text[0] == " "
-                or (self._is_alphabet_char(para_2_text[0]) and para_2_text[0].islower())
-                or (self._is_alphabet_char(para_2_text[0]))
-                or (self._is_year(para_2_text[0:4]))
-                or (are_two_paras_right_aligned or is_para1_left_indent_than_papa2)
-                or (self._is_chinese_char(para_2_text[0]))
-                or (self._is_other_letter_char(para_2_text[0]))
-                or (self._match_brackets(para_2_text[0]))
-            )
-            if para_2_is_consistent:
-                # print(f"para_2 is consistent.\n")
-                return True
-            else:
-                # print(f"para_2 is not consistent.\n")
-                pass
-
-        elif para_1_end_with_space:  # If para_text1 ends with space
-            # print_red(f"para_1 is end with space.")
-            para_2_is_consistent = para_2_text and (
-                para_2_text[0] == " "
-                or (self._is_alphabet_char(para_2_text[0]) and para_2_text[0].islower())
-                or (self._is_chinese_char(para_2_text[0]))
-                or (self._is_other_letter_char(para_2_text[0]))
-            )
-            if para_2_is_consistent:
-                # print(f"para_2 is consistent.\n")
-                return True
-            else:
-                pass
-                # print(f"para_2 is not consistent.\n")
-
-        return False
-
-    def _is_block_consistent(self, block_1, block_2):
-        """
-        This function determines whether block1 and block2 are originally from the same block
-
-        Parameters
-        ----------
-        block1 : dict
-            block1s
-        block2 : dict
-            block2
-
-        Returns
-        -------
-        is_same : bool
-            True if block1 and block2 are from the same block, else False
-        """
-        return self.__is_same_block_font(block_1, block_2)
-
-    def _is_para_continued(self, para_1, para_2):
-        """
-        This function determines whether para1 and para2 are originally from the same paragraph
-
-        Parameters
-        ----------
-        para1 : dict
-            para1
-        para2 : dict
-            para2
-
-        Returns
-        -------
-        is_same : bool
-            True if para1 and para2 are from the same paragraph, else False
-        """
-        is_para_font_consistent = self._is_para_font_consistent(para_1, para_2)
-        is_para_puncs_consistent = self._is_para_puncs_consistent(para_1, para_2)
-
-        return is_para_font_consistent and is_para_puncs_consistent
-
-    def _are_boundaries_of_block_consistent(self, block_1, block_2):
-        """
-        This function checks if the boundaries of block1 and block2 are consistent
-
-        Parameters
-        ----------
-        block1 : dict
-            block1
-
-        block2 : dict
-            block2
-
-        Returns
-        -------
-        is_consistent : bool
-            True if the boundaries of block1 and block2 are consistent, else False
-        """
-
-        last_line_of_block_1 = block_1["lines"][-1]
-        first_line_of_block_2 = block_2["lines"][0]
-
-        spans_of_last_line_of_block_1 = last_line_of_block_1["spans"]
-        spans_of_first_line_of_block_2 = first_line_of_block_2["spans"]
-
-        font_type_of_last_line_of_block_1 = spans_of_last_line_of_block_1[0]["font"].lower()
-        font_size_of_last_line_of_block_1 = spans_of_last_line_of_block_1[0]["size"]
-        font_color_of_last_line_of_block_1 = spans_of_last_line_of_block_1[0]["color"]
-        font_flags_of_last_line_of_block_1 = spans_of_last_line_of_block_1[0]["flags"]
-
-        font_type_of_first_line_of_block_2 = spans_of_first_line_of_block_2[0]["font"].lower()
-        font_size_of_first_line_of_block_2 = spans_of_first_line_of_block_2[0]["size"]
-        font_color_of_first_line_of_block_2 = spans_of_first_line_of_block_2[0]["color"]
-        font_flags_of_first_line_of_block_2 = spans_of_first_line_of_block_2[0]["flags"]
-
-        return (
-            self.__is_similar_font_type(font_type_of_last_line_of_block_1, font_type_of_first_line_of_block_2)
-            and abs(font_size_of_last_line_of_block_1 - font_size_of_first_line_of_block_2) < 1
-            # and font_color_of_last_line_of_block1 == font_color_of_first_line_of_block2
-            and font_flags_of_last_line_of_block_1 == font_flags_of_first_line_of_block_2
-        )
-
-    def should_merge_next_para(self, curr_para, next_para):
-        """
-        This function checks if the next_para should be merged into the curr_para.
-
-        Parameters
-        ----------
-        curr_para : dict
-            The current paragraph.
-        next_para : dict
-            The next paragraph.
-
-        Returns
-        -------
-        bool
-            True if the next_para should be merged into the curr_para, False otherwise.
-        """
-        if self._is_para_continued(curr_para, next_para):
-            return True
-        else:
-            return False
-
-    def batch_tag_paras(self, pdf_dict):
-        """
-        This function tags the paragraphs in the pdf_dict.
-
-        Parameters
-        ----------
-        pdf_dict : dict
-            PDF dictionary.
-
-        Returns
-        -------
-        pdf_dict : dict
-            PDF dictionary with tagged paragraphs.
-        """
-        the_last_page_id = len(pdf_dict) - 1
-
-        for curr_page_idx, (curr_page_id, curr_page_content) in enumerate(pdf_dict.items()):
-            if curr_page_id.startswith("page_") and curr_page_content.get("para_blocks", []):
-                para_blocks_of_curr_page = curr_page_content["para_blocks"]
-                next_page_idx = curr_page_idx + 1
-                next_page_id = f"page_{next_page_idx}"
-                next_page_content = pdf_dict.get(next_page_id, {})
-
-                for i, current_block in enumerate(para_blocks_of_curr_page):
-                    for para_id, curr_para in current_block["paras"].items():
-                        curr_para["curr_para_location"] = [
-                            curr_page_idx,
-                            current_block["block_id"],
-                            int(para_id.split("_")[-1]),
-                        ]
-                        curr_para["next_para_location"] = None  # 默认设置为None
-                        curr_para["merge_next_para"] = False  # 默认设置为False
-
-                    next_block = para_blocks_of_curr_page[i + 1] if i < len(para_blocks_of_curr_page) - 1 else None
-
-                    if next_block:
-                        curr_block_last_para_key = list(current_block["paras"].keys())[-1]
-                        curr_blk_last_para = current_block["paras"][curr_block_last_para_key]
-
-                        next_block_first_para_key = list(next_block["paras"].keys())[0]
-                        next_blk_first_para = next_block["paras"][next_block_first_para_key]
-
-                        if self.should_merge_next_para(curr_blk_last_para, next_blk_first_para):
-                            curr_blk_last_para["next_para_location"] = [
-                                curr_page_idx,
-                                next_block["block_id"],
-                                int(next_block_first_para_key.split("_")[-1]),
-                            ]
-                            curr_blk_last_para["merge_next_para"] = True
-                    else:
-                        # Handle the case where the next block is in a different page
-                        curr_block_last_para_key = list(current_block["paras"].keys())[-1]
-                        curr_blk_last_para = current_block["paras"][curr_block_last_para_key]
-
-                        while not next_page_content.get("para_blocks", []) and next_page_idx <= the_last_page_id:
-                            next_page_idx += 1
-                            next_page_id = f"page_{next_page_idx}"
-                            next_page_content = pdf_dict.get(next_page_id, {})
-
-                        if next_page_content.get("para_blocks", []):
-                            next_blk_first_para_key = list(next_page_content["para_blocks"][0]["paras"].keys())[0]
-                            next_blk_first_para = next_page_content["para_blocks"][0]["paras"][next_blk_first_para_key]
-
-                            if self.should_merge_next_para(curr_blk_last_para, next_blk_first_para):
-                                curr_blk_last_para["next_para_location"] = [
-                                    next_page_idx,
-                                    next_page_content["para_blocks"][0]["block_id"],
-                                    int(next_blk_first_para_key.split("_")[-1]),
-                                ]
-                                curr_blk_last_para["merge_next_para"] = True
-
-        return pdf_dict
-
-    def find_block_by_id(self, para_blocks, block_id):
-        """
-        This function finds a block by its id.
-
-        Parameters
-        ----------
-        para_blocks : list
-            List of blocks.
-        block_id : int
-            Id of the block to find.
-
-        Returns
-        -------
-        block : dict
-            The block with the given id.
-        """
-        for blk_idx, block in enumerate(para_blocks):
-            if block.get("block_id") == block_id:
-                return block
-        return None
-
-    def batch_merge_paras(self, pdf_dict):
-        """
-        This function merges the paragraphs in the pdf_dict.
-
-        Parameters
-        ----------
-        pdf_dict : dict
-            PDF dictionary.
-
-        Returns
-        -------
-        pdf_dict : dict
-            PDF dictionary with merged paragraphs.
-        """
-        for page_id, page_content in pdf_dict.items():
-            if page_id.startswith("page_") and page_content.get("para_blocks", []):
-                para_blocks_of_page = page_content["para_blocks"]
-
-                for i in range(len(para_blocks_of_page)):
-                    current_block = para_blocks_of_page[i]
-                    paras = current_block["paras"]
-
-                    for para_id, curr_para in list(paras.items()):
-                        # print(f"current para_id: {para_id}")
-                        # 跳过标题段落
-                        if curr_para.get("is_para_title"):
-                            continue
-
-                        while curr_para.get("merge_next_para"):
-                            curr_para_location = curr_para.get("curr_para_location")
-                            next_para_location = curr_para.get("next_para_location")
-
-                            # print(f"curr_para_location: {curr_para_location}, next_para_location: {next_para_location}")
-                            
-                            if not next_para_location:
-                                break
-
-                            if curr_para_location == next_para_location:
-                                # print_red("The next para is in the same block as the current para.")
-                                curr_para["merge_next_para"] = False
-                                break
-
-                            next_page_idx, next_block_id, next_para_id = next_para_location
-                            next_page_id = f"page_{next_page_idx}"
-                            next_page_content = pdf_dict.get(next_page_id)
-                            if not next_page_content:
-                                break
-
-                            next_block = self.find_block_by_id(next_page_content.get("para_blocks", []), next_block_id)
-
-                            if not next_block:
-                                break
-
-                            next_para = next_block["paras"].get(f"para_{next_para_id}")
-
-                            if not next_para or next_para.get("is_para_title"):
-                                break
-
-                            # 合并段落文本
-                            curr_para_text = curr_para.get("para_text", "")
-                            next_para_text = next_para.get("para_text", "")
-                            curr_para["para_text"] = curr_para_text + " " + next_para_text
-
-                            # 更新 next_para_location
-                            curr_para["next_para_location"] = next_para.get("next_para_location")
-
-                            # 将下一个段落文本置为空，表示已被合并
-                            next_para["para_text"] = ""
-
-                            # 更新 merge_next_para 标记
-                            curr_para["merge_next_para"] = next_para.get("merge_next_para", False)
-
-        return pdf_dict
-
-
-class DrawAnnos:
-    """
-    This class draws annotations on the pdf file
-
-    ----------------------------------------
-                Color Code
-    ----------------------------------------
-        Red: (1, 0, 0)
-        Green: (0, 1, 0)
-        Blue: (0, 0, 1)
-        Yellow: (1, 1, 0) - mix of red and green
-        Cyan: (0, 1, 1) - mix of green and blue
-        Magenta: (1, 0, 1) - mix of red and blue
-        White: (1, 1, 1) - red, green and blue full intensity
-        Black: (0, 0, 0) - no color component whatsoever
-        Gray: (0.5, 0.5, 0.5) - equal and medium intensity of red, green and blue color components
-        Orange: (1, 0.65, 0) - maximum intensity of red, medium intensity of green, no blue component
-    """
-
-    def __init__(self) -> None:
-        pass
-
-    def __is_nested_list(self, lst):
-        """
-        This function returns True if the given list is a nested list of any degree.
-        """
-        if isinstance(lst, list):
-            return any(self.__is_nested_list(i) for i in lst) or any(isinstance(i, list) for i in lst)
-        return False
-
-    def __valid_rect(self, bbox):
-        # Ensure that the rectangle is not empty or invalid
-        if isinstance(bbox[0], list):
-            return False  # It's a nested list, hence it can't be valid rect
-        else:
-            return bbox[0] < bbox[2] and bbox[1] < bbox[3]
-
-    def __draw_nested_boxes(self, page, nested_bbox, color=(0, 1, 1)):
-        """
-        This function draws the nested boxes
-
-        Parameters
-        ----------
-        page : fitz.Page
-            page
-        nested_bbox : list
-            nested bbox
-        color : tuple
-            color, by default (0, 1, 1)    # draw with cyan color for combined paragraph
-        """
-        if self.__is_nested_list(nested_bbox):  # If it's a nested list
-            for bbox in nested_bbox:
-                self.__draw_nested_boxes(page, bbox, color)  # Recursively call the function
-        elif self.__valid_rect(nested_bbox):  # If valid rectangle
-            para_rect = fitz.Rect(nested_bbox)
-            para_anno = page.add_rect_annot(para_rect)
-            para_anno.set_colors(stroke=color)  # draw with cyan color for combined paragraph
-            para_anno.set_border(width=1)
-            para_anno.update()
-
-    def draw_annos(self, input_pdf_path, pdf_dic, output_pdf_path):
-        """
-        This function draws annotations on the pdf file.
-
-        Parameters
-        ----------
-        input_pdf_path : str
-            path to the input pdf file
-        pdf_dic : dict
-            pdf dictionary
-        output_pdf_path : str
-            path to the output pdf file
-
-        pdf_dic : dict
-            pdf dictionary
-        """
-        pdf_doc = open_pdf(input_pdf_path)
-
-        if pdf_dic is None:
-            pdf_dic = {}
-
-        if output_pdf_path is None:
-            output_pdf_path = input_pdf_path.replace(".pdf", "_anno.pdf")
-
-        for page_id, page in enumerate(pdf_doc):  # type: ignore
-            page_key = f"page_{page_id}"
-            for ele_key, ele_data in pdf_dic[page_key].items():
-                if ele_key == "para_blocks":
-                    para_blocks = ele_data
-                    for para_block in para_blocks:
-                        if "paras" in para_block.keys():
-                            paras = para_block["paras"]
-                            for para_key, para_content in paras.items():
-                                para_bbox = para_content["para_bbox"]
-                                # print(f"para_bbox: {para_bbox}")
-                                # print(f"is a nested list: {self.__is_nested_list(para_bbox)}")
-                                if self.__is_nested_list(para_bbox) and len(para_bbox) > 1:
-                                    color = (0, 1, 1)
-                                    self.__draw_nested_boxes(
-                                        page, para_bbox, color
-                                    )  # draw with cyan color for combined paragraph
-                                else:
-                                    if self.__valid_rect(para_bbox):
-                                        para_rect = fitz.Rect(para_bbox)
-                                        para_anno = page.add_rect_annot(para_rect)
-                                        para_anno.set_colors(stroke=(0, 1, 0))  # draw with green color for normal paragraph
-                                        para_anno.set_border(width=0.5)
-                                        para_anno.update()
-
-                                is_para_title = para_content["is_para_title"]
-                                if is_para_title:
-                                    if self.__is_nested_list(para_content["para_bbox"]) and len(para_content["para_bbox"]) > 1:
-                                        color = (0, 0, 1)
-                                        self.__draw_nested_boxes(
-                                            page, para_content["para_bbox"], color
-                                        )  # draw with cyan color for combined title
-                                    else:
-                                        if self.__valid_rect(para_content["para_bbox"]):
-                                            para_rect = fitz.Rect(para_content["para_bbox"])
-                                            if self.__valid_rect(para_content["para_bbox"]):
-                                                para_anno = page.add_rect_annot(para_rect)
-                                                para_anno.set_colors(stroke=(0, 0, 1))  # draw with blue color for normal title
-                                                para_anno.set_border(width=0.5)
-                                                para_anno.update()
-
-        pdf_doc.save(output_pdf_path)
-        pdf_doc.close()
-
-
-class ParaProcessPipeline:
-    def __init__(self) -> None:
-        pass
-
-    def para_process_pipeline(self, pdf_info_dict, para_debug_mode=None, input_pdf_path=None, output_pdf_path=None):
-        """
-        This function processes the paragraphs, including:
-        1. Read raw input json file into pdf_dic
-        2. Detect and replace equations
-        3. Combine spans into a natural line
-        4. Check if the paragraphs are inside bboxes passed from "layout_bboxes" key
-        5. Compute statistics for each block
-        6. Detect titles in the document
-        7. Detect paragraphs inside each block
-        8. Divide the level of the titles
-        9. Detect and combine paragraphs from different blocks into one paragraph
-        10. Check whether the final results after checking headings, dividing paragraphs within blocks, and merging paragraphs between blocks are plausible and reasonable.
-        11. Draw annotations on the pdf file
-
-        Parameters
-        ----------
-        pdf_dic_json_fpath : str
-            path to the pdf dictionary json file.
-            Notice: data noises, including overlap blocks, header, footer, watermark, vertical margin note have been removed already.
-        input_pdf_doc : str
-            path to the input pdf file
-        output_pdf_path : str
-            path to the output pdf file
-
-        Returns
-        -------
-        pdf_dict : dict
-            result dictionary
-        """
-
-        error_info = None
-
-        output_json_file = ""
-        output_dir = ""
-
-        if input_pdf_path is not None:
-            input_pdf_path = os.path.abspath(input_pdf_path)
-
-            # print_green_on_red(f">>>>>>>>>>>>>>>>>>> Process the paragraphs of {input_pdf_path}")
-
-        if output_pdf_path is not None:
-            output_dir = os.path.dirname(output_pdf_path)
-            output_json_file = f"{output_dir}/pdf_dic.json"
-
-        def __save_pdf_dic(pdf_dic, output_pdf_path, stage="0", para_debug_mode=para_debug_mode):
-            """
-            Save the pdf_dic to a json file
-            """
-            output_pdf_file_name = os.path.basename(output_pdf_path)
-            # output_dir = os.path.dirname(output_pdf_path)
-            output_dir = "\\tmp\\pdf_parse"
-            output_pdf_file_name = output_pdf_file_name.replace(".pdf", f"_stage_{stage}.json")
-            pdf_dic_json_fpath = os.path.join(output_dir, output_pdf_file_name)
-
-            if not os.path.exists(output_dir):
-                os.makedirs(output_dir)
-
-            if para_debug_mode == "full":
-                with open(pdf_dic_json_fpath, "w", encoding="utf-8") as f:
-                    json.dump(pdf_dic, f, indent=2, ensure_ascii=False)
-
-            # Validate the output already exists
-            if not os.path.exists(pdf_dic_json_fpath):
-                print_red(f"Failed to save the pdf_dic to {pdf_dic_json_fpath}")
-                return None
-            else:
-                print_green(f"Succeed to save the pdf_dic to {pdf_dic_json_fpath}")
-
-            return pdf_dic_json_fpath
-
-        """
-        Preprocess the lines of block
-        """
-        # Combine spans into a natural line
-        rawBlockProcessor = RawBlockProcessor()
-        pdf_dic = rawBlockProcessor.batch_process_blocks(pdf_info_dict)
-        # print(f"pdf_dic['page_0']['para_blocks'][0]: {pdf_dic['page_0']['para_blocks'][0]}", end="\n\n")
-
-        # Check if the paragraphs are inside bboxes passed from "layout_bboxes" key
-        layoutFilter = LayoutFilterProcessor()
-        pdf_dic = layoutFilter.batch_process_blocks(pdf_dic)
-
-        # Compute statistics for each block
-        blockStatisticsCalculator = BlockStatisticsCalculator()
-        pdf_dic = blockStatisticsCalculator.batch_process_blocks(pdf_dic)
-        # print(f"pdf_dic['page_0']['para_blocks'][0]: {pdf_dic['page_0']['para_blocks'][0]}", end="\n\n")
-
-        # Compute statistics for all blocks(namely this pdf document)
-        docStatisticsCalculator = DocStatisticsCalculator()
-        pdf_dic = docStatisticsCalculator.calc_stats_of_doc(pdf_dic)
-        # print(f"pdf_dic['statistics']: {pdf_dic['statistics']}", end="\n\n")
-
-        # Dump the first three stages of pdf_dic to a json file
-        if para_debug_mode == "full":
-            pdf_dic_json_fpath = __save_pdf_dic(pdf_dic, output_pdf_path, stage="0", para_debug_mode=para_debug_mode)
-
-        """
-        Detect titles in the document
-        """
-        doc_statistics = pdf_dic["statistics"]
-        titleProcessor = TitleProcessor(doc_statistics)
-        pdf_dic = titleProcessor.batch_detect_titles(pdf_dic)
-
-        if para_debug_mode == "full":
-            pdf_dic_json_fpath = __save_pdf_dic(pdf_dic, output_pdf_path, stage="1", para_debug_mode=para_debug_mode)
-
-        """
-        Detect and divide the level of the titles
-        """
-        titleProcessor = TitleProcessor()
-
-        pdf_dic = titleProcessor.batch_recog_title_level(pdf_dic)
-
-        if para_debug_mode == "full":
-            pdf_dic_json_fpath = __save_pdf_dic(pdf_dic, output_pdf_path, stage="2", para_debug_mode=para_debug_mode)
-
-        """
-        Detect and split paragraphs inside each block
-        """
-        blockInnerParasProcessor = BlockTerminationProcessor()
-
-        pdf_dic = blockInnerParasProcessor.batch_process_blocks(pdf_dic)
-
-        if para_debug_mode == "full":
-            pdf_dic_json_fpath = __save_pdf_dic(pdf_dic, output_pdf_path, stage="3", para_debug_mode=para_debug_mode)
-
-        # pdf_dic_json_fpath = __save_pdf_dic(pdf_dic, output_pdf_path, stage="3", para_debug_mode="full")
-        # print_green(f"pdf_dic_json_fpath: {pdf_dic_json_fpath}")
-
-        """
-        Detect and combine paragraphs from different blocks into one paragraph
-        """
-        blockContinuationProcessor = BlockContinuationProcessor()
-
-        pdf_dic = blockContinuationProcessor.batch_tag_paras(pdf_dic)
-        pdf_dic = blockContinuationProcessor.batch_merge_paras(pdf_dic)
-
-        if para_debug_mode == "full":
-            pdf_dic_json_fpath = __save_pdf_dic(pdf_dic, output_pdf_path, stage="4", para_debug_mode=para_debug_mode)
-
-        # pdf_dic_json_fpath = __save_pdf_dic(pdf_dic, output_pdf_path, stage="4", para_debug_mode="full")
-        # print_green(f"pdf_dic_json_fpath: {pdf_dic_json_fpath}")
-
-        """
-        Discard pdf files by checking exceptions and return the error info to the caller
-        """
-        discardByException = DiscardByException()
-
-        is_discard_by_single_line_block = discardByException.discard_by_single_line_block(
-            pdf_dic, exception=DenseSingleLineBlockException()
-        )
-        is_discard_by_title_detection = discardByException.discard_by_title_detection(
-            pdf_dic, exception=TitleDetectionException()
-        )
-        is_discard_by_title_level = discardByException.discard_by_title_level(pdf_dic, exception=TitleLevelException())
-        is_discard_by_split_para = discardByException.discard_by_split_para(pdf_dic, exception=ParaSplitException())
-        is_discard_by_merge_para = discardByException.discard_by_merge_para(pdf_dic, exception=ParaMergeException())
-
-        if is_discard_by_single_line_block is not None:
-            error_info = is_discard_by_single_line_block
-        elif is_discard_by_title_detection is not None:
-            error_info = is_discard_by_title_detection
-        elif is_discard_by_title_level is not None:
-            error_info = is_discard_by_title_level
-        elif is_discard_by_split_para is not None:
-            error_info = is_discard_by_split_para
-        elif is_discard_by_merge_para is not None:
-            error_info = is_discard_by_merge_para
-
-        if error_info is not None:
-            return pdf_dic, error_info
-
-        """
-        Dump the final pdf_dic to a json file
-        """
-        if para_debug_mode is not None:
-            with open(output_json_file, "w", encoding="utf-8") as f:
-                json.dump(pdf_info_dict, f, ensure_ascii=False, indent=4)
-
-        """
-        Draw the annotations
-        """
-        if para_debug_mode is not None:
-            drawAnnos = DrawAnnos()
-            drawAnnos.draw_annos(input_pdf_path, pdf_dic, output_pdf_path)
-
-        """
-        Remove the intermediate files which are generated in the process of paragraph processing if debug_mode is simple
-        """
-        if para_debug_mode is not None:
-            for fpath in os.listdir(output_dir):
-                if fpath.endswith(".json") and "stage" in fpath:
-                    os.remove(os.path.join(output_dir, fpath))
-
-        return pdf_dic, error_info
-
-
-"""
-Run this script to test the function with Command: 
-
-python detect_para.py [pdf_path] [output_pdf_path]
-
-Params:
-- pdf_path: the path of the pdf file
-- output_pdf_path: the path of the output pdf file
-"""
-
-if __name__ == "__main__":
-    DEFAULT_PDF_PATH = (
-        "app/pdf_toolbox/tests/assets/paper/paper.pdf" if os.name != "nt" else "app\\pdf_toolbox\\tests\\assets\\paper\\paper.pdf"
-    )
-    input_pdf_path = sys.argv[1] if len(sys.argv) > 1 else DEFAULT_PDF_PATH
-    output_pdf_path = sys.argv[2] if len(sys.argv) > 2 else input_pdf_path.split(".")[0] + "_recogPara.pdf"
-    output_json_path = sys.argv[3] if len(sys.argv) > 3 else input_pdf_path.split(".")[0] + "_recogPara.json"
-
-    import stat
-
-    # Remove existing output file if it exists
-    if os.path.exists(output_pdf_path):
-        os.chmod(output_pdf_path, stat.S_IWRITE)
-        os.remove(output_pdf_path)
-
-    input_pdf_doc = open_pdf(input_pdf_path)
-
-    # postprocess the paragraphs
-    paraProcessPipeline = ParaProcessPipeline()
-
-    # parse paragraph and save to json file
-    pdf_dic = {}
-
-    blockInnerParasProcessor = BlockTerminationProcessor()
-
-    """
-    Construct the pdf dictionary.
-    """
-
-    for page_id, page in enumerate(input_pdf_doc):  # type: ignore
-        # print(f"Processing page {page_id}")
-        # print(f"page: {page}")
-        raw_blocks = page.get_text("dict")["blocks"]
-
-        # Save text blocks to "preproc_blocks"
-        preproc_blocks = []
-        for block in raw_blocks:
-            if block["type"] == 0:
-                preproc_blocks.append(block)
-
-        layout_bboxes = []
-
-        # Construct the pdf dictionary as schema above
-        page_dict = {
-            "para_blocks": None,
-            "preproc_blocks": preproc_blocks,
-            "images": None,
-            "tables": None,
-            "interline_equations": None,
-            "inline_equations": None,
-            "layout_bboxes": None,
-            "pymu_raw_blocks": None,
-            "global_statistic": None,
-            "droped_text_block": None,
-            "droped_image_block": None,
-            "droped_table_block": None,
-            "image_backup": None,
-            "table_backup": None,
-        }
-
-        pdf_dic[f"page_{page_id}"] = page_dict
-
-    # print(f"pdf_dic: {pdf_dic}")
-
-    with open(output_json_path, "w", encoding="utf-8") as f:
-        json.dump(pdf_dic, f, ensure_ascii=False, indent=4)
-
-    pdf_dic = paraProcessPipeline.para_process_pipeline(output_json_path, input_pdf_doc, output_pdf_path)