magic-pdf 0.10.1__py3-none-any.whl → 0.10.2__py3-none-any.whl

magic_pdf/para/title_processor.py

@@ -1,1014 +0,0 @@
-import os
-import re
-import numpy as np
-
-from magic_pdf.libs.nlp_utils import NLPModels
-
-from magic_pdf.para.commons import *
-
-if sys.version_info[0] >= 3:
-    sys.stdout.reconfigure(encoding="utf-8")  # type: ignore
-
-
-class TitleProcessor:
-    def __init__(self, *doc_statistics) -> None:
-        if len(doc_statistics) > 0:
-            self.doc_statistics = doc_statistics[0]
-
-        self.nlp_model = NLPModels()
-        self.MAX_TITLE_LEVEL = 3
-        self.numbered_title_pattern = r"""
-            ^                                 # 行首
-            (                                 # 开始捕获组
-                [\(\（]\d+[\)\）]              # 括号内数字，支持中文和英文括号，例如：(1) 或 （1）
-                |\d+[\)\）]\s                  # 数字后跟右括号和空格，支持中文和英文括号，例如：2) 或 2）
-                |[\(\（][A-Z][\)\）]            # 括号内大写字母，支持中文和英文括号，例如：(A) 或 （A）
-                |[A-Z][\)\）]\s                # 大写字母后跟右括号和空格，例如：A) 或 A）
-                |[\(\（][IVXLCDM]+[\)\）]       # 括号内罗马数字，支持中文和英文括号，例如：(I) 或 （I）
-                |[IVXLCDM]+[\)\）]\s            # 罗马数字后跟右括号和空格，例如：I) 或 I）
-                |\d+(\.\d+)*\s                # 数字或复合数字编号后跟空格，例如：1. 或 3.2.1 
-                |[一二三四五六七八九十百千]+[、\s]       # 中文序号后跟顿号和空格，例如：一、
-                |[\（|\(][一二三四五六七八九十百千]+[\）|\)]\s*  # 中文括号内中文序号后跟空格，例如：（一）
-                |[A-Z]\.\d+(\.\d+)?\s         # 大写字母后跟点和数字，例如：A.1 或 A.1.1
-                |[\(\（][a-z][\)\）]            # 括号内小写字母，支持中文和英文括号，例如：(a) 或 （a）
-                |[a-z]\)\s                    # 小写字母后跟右括号和空格，例如：a) 
-                |[A-Z]-\s                     # 大写字母后跟短横线和空格，例如：A- 
-                |\w+:\s                       # 英文序号词后跟冒号和空格，例如：First: 
-                |第[一二三四五六七八九十百千]+[章节部分条款]\s # 以“第”开头的中文标题后跟空格
-                |[IVXLCDM]+\.                 # 罗马数字后跟点，例如：I.
-                |\d+\.\s                      # 单个数字后跟点和空格，例如：1. 
-            )                                 # 结束捕获组
-            .+                                # 标题的其余部分
-        """
-
-    def _is_potential_title(
-        self,
-        curr_line,
-        prev_line,
-        prev_line_is_title,
-        next_line,
-        avg_char_width,
-        avg_char_height,
-        median_font_size,
-    ):
-        """
-        This function checks if the line is a potential title.
-
-        Parameters
-        ----------
-        curr_line : dict
-            current line
-        prev_line : dict
-            previous line
-        next_line : dict
-            next line
-        avg_char_width : float
-            average of char widths
-        avg_char_height : float
-            average of line heights
-
-        Returns
-        -------
-        bool
-            True if the line is a potential title, False otherwise.
-        """
-
-        def __is_line_centered(line_bbox, page_bbox, avg_char_width):
-            """
-            This function checks if the line is centered on the page
-
-            Parameters
-            ----------
-            line_bbox : list
-                bbox of the line
-            page_bbox : list
-                bbox of the page
-            avg_char_width : float
-                average of char widths
-
-            Returns
-            -------
-            bool
-                True if the line is centered on the page, False otherwise.
-            """
-            horizontal_ratio = 0.5
-            horizontal_thres = horizontal_ratio * avg_char_width
-
-            x0, _, x1, _ = line_bbox
-            _, _, page_x1, _ = page_bbox
-
-            return abs((x0 + x1) / 2 - page_x1 / 2) < horizontal_thres
-
-        def __is_bold_font_line(line):
-            """
-            Check if a line contains any bold font style.
-            """
-
-            def _is_bold_span(span):
-                # if span text is empty or only contains space, return False
-                if not span["text"].strip():
-                    return False
-
-                return bool(span["flags"] & 2**4)  # Check if the font is bold
-
-            for span in line["spans"]:
-                if not _is_bold_span(span):
-                    return False
-
-            return True
-
-        def __is_italic_font_line(line):
-            """
-            Check if a line contains any italic font style.
-            """
-
-            def __is_italic_span(span):
-                return bool(span["flags"] & 2**1)  # Check if the font is italic
-
-            for span in line["spans"]:
-                if not __is_italic_span(span):
-                    return False
-
-            return True
-
-        def __is_punctuation_heavy(line_text):
-            """
-            Check if the line contains a high ratio of punctuation marks, which may indicate
-            that the line is not a title.
-
-            Parameters:
-            line_text (str): Text of the line.
-
-            Returns:
-            bool: True if the line is heavy with punctuation, False otherwise.
-            """
-            # Pattern for common title format like "X.Y. Title"
-            pattern = r"\b\d+\.\d+\..*\b"
-
-            # If the line matches the title format, return False
-            if re.match(pattern, line_text.strip()):
-                return False
-
-            # Find all punctuation marks in the line
-            punctuation_marks = re.findall(r"[^\w\s]", line_text)
-            number_of_punctuation_marks = len(punctuation_marks)
-
-            text_length = len(line_text)
-
-            if text_length == 0:
-                return False
-
-            punctuation_ratio = number_of_punctuation_marks / text_length
-            if punctuation_ratio >= 0.1:
-                return True
-
-            return False
-
-        def __has_mixed_font_styles(spans, strict_mode=False):
-            """
-            This function checks if the line has mixed font styles, the strict mode will compare the font types
-
-            Parameters
-            ----------
-            spans : list
-                spans of the line
-            strict_mode : bool
-                True for strict mode, the font types will be fully compared
-                False for non-strict mode, the font types will be compared by the most longest common prefix
-
-            Returns
-            -------
-            bool
-                True if the line has mixed font styles, False otherwise.
-            """
-            if strict_mode:
-                font_styles = set()
-                for span in spans:
-                    font_style = span["font"].lower()
-                    font_styles.add(font_style)
-
-                return len(font_styles) > 1
-
-            else:  # non-strict mode
-                font_styles = []
-                for span in spans:
-                    font_style = span["font"].lower()
-                    font_styles.append(font_style)
-
-                if len(font_styles) > 1:
-                    longest_common_prefix = os.path.commonprefix(font_styles)
-                    if len(longest_common_prefix) > 0:
-                        return False
-                    else:
-                        return True
-                else:
-                    return False
-
-        def __is_different_font_type_from_neighbors(curr_line_font_type, prev_line_font_type, next_line_font_type):
-            """
-            This function checks if the current line has a different font type from the previous and next lines
-
-            Parameters
-            ----------
-            curr_line_font_type : str
-                font type of the current line
-            prev_line_font_type : str
-                font type of the previous line
-            next_line_font_type : str
-                font type of the next line
-
-            Returns
-            -------
-            bool
-                True if the current line has a different font type from the previous and next lines, False otherwise.
-            """
-            return all(
-                curr_line_font_type != other_font_type.lower()
-                for other_font_type in [prev_line_font_type, next_line_font_type]
-                if other_font_type is not None
-            )
-
-        def __is_larger_font_size_from_neighbors(curr_line_font_size, prev_line_font_size, next_line_font_size):
-            """
-            This function checks if the current line has a larger font size than the previous and next lines
-
-            Parameters
-            ----------
-            curr_line_font_size : float
-                font size of the current line
-            prev_line_font_size : float
-                font size of the previous line
-            next_line_font_size : float
-                font size of the next line
-
-            Returns
-            -------
-            bool
-                True if the current line has a larger font size than the previous and next lines, False otherwise.
-            """
-            return all(
-                curr_line_font_size > other_font_size * 1.2
-                for other_font_size in [prev_line_font_size, next_line_font_size]
-                if other_font_size is not None
-            )
-
-        def __is_similar_to_pre_line(curr_line_font_type, prev_line_font_type, curr_line_font_size, prev_line_font_size):
-            """
-            This function checks if the current line is similar to the previous line
-
-            Parameters
-            ----------
-            curr_line : dict
-                current line
-            prev_line : dict
-                previous line
-
-            Returns
-            -------
-            bool
-                True if the current line is similar to the previous line, False otherwise.
-            """
-
-            if curr_line_font_type == prev_line_font_type and curr_line_font_size == prev_line_font_size:
-                return True
-            else:
-                return False
-
-        def __is_same_font_type_of_docAvg(curr_line_font_type):
-            """
-            This function checks if the current line has the same font type as the document average font type
-
-            Parameters
-            ----------
-            curr_line_font_type : str
-                font type of the current line
-
-            Returns
-            -------
-            bool
-                True if the current line has the same font type as the document average font type, False otherwise.
-            """
-            doc_most_common_font_type = safe_get(self.doc_statistics, "most_common_font_type", "").lower()
-            doc_second_most_common_font_type = safe_get(self.doc_statistics, "second_most_common_font_type", "").lower()
-
-            return curr_line_font_type.lower() in [doc_most_common_font_type, doc_second_most_common_font_type]
-
-        def __is_font_size_not_less_than_docAvg(curr_line_font_size, ratio: float = 1):
-            """
-            This function checks if the current line has a large enough font size
-
-            Parameters
-            ----------
-            curr_line_font_size : float
-                font size of the current line
-            ratio : float
-                ratio of the current line font size to the document average font size
-
-            Returns
-            -------
-            bool
-                True if the current line has a large enough font size, False otherwise.
-            """
-            doc_most_common_font_size = safe_get(self.doc_statistics, "most_common_font_size", 0)
-            doc_second_most_common_font_size = safe_get(self.doc_statistics, "second_most_common_font_size", 0)
-            doc_avg_font_size = min(doc_most_common_font_size, doc_second_most_common_font_size)
-
-            return curr_line_font_size >= doc_avg_font_size * ratio
-
-        def __is_sufficient_spacing_above_and_below(
-            curr_line_bbox,
-            prev_line_bbox,
-            next_line_bbox,
-            avg_char_height,
-            median_font_size,
-        ):
-            """
-            This function checks if the current line has sufficient spacing above and below
-
-            Parameters
-            ----------
-            curr_line_bbox : list
-                bbox of the current line
-            prev_line_bbox : list
-                bbox of the previous line
-            next_line_bbox : list
-                bbox of the next line
-            avg_char_width : float
-                average of char widths
-            avg_char_height : float
-                average of line heights
-
-            Returns
-            -------
-            bool
-                True if the current line has sufficient spacing above and below, False otherwise.
-            """
-            vertical_ratio = 1.25
-            vertical_thres = vertical_ratio * median_font_size
-
-            _, y0, _, y1 = curr_line_bbox
-
-            sufficient_spacing_above = False
-            if prev_line_bbox:
-                vertical_spacing_above = min(y0 - prev_line_bbox[1], y1 - prev_line_bbox[3])
-                sufficient_spacing_above = vertical_spacing_above > vertical_thres
-            else:
-                sufficient_spacing_above = True
-
-            sufficient_spacing_below = False
-            if next_line_bbox:
-                vertical_spacing_below = min(next_line_bbox[1] - y0, next_line_bbox[3] - y1)
-                sufficient_spacing_below = vertical_spacing_below > vertical_thres
-            else:
-                sufficient_spacing_below = True
-
-            return (sufficient_spacing_above, sufficient_spacing_below)
-
-        def __is_word_list_line_by_rules(curr_line_text):
-            """
-            This function checks if the current line is a word list
-
-            Parameters
-            ----------
-            curr_line_text : str
-                text of the current line
-
-            Returns
-            -------
-            bool
-                True if the current line is a name list, False otherwise.
-            """
-            # name_list_pattern = r"([a-zA-Z][a-zA-Z\s]{0,20}[a-zA-Z]|[\u4e00-\u9fa5·]{2,16})(?=[，,;；\s]|$)"
-            name_list_pattern = r"(?<![\u4e00-\u9fa5])([A-Z][a-z]{0,19}\s[A-Z][a-z]{0,19}|[\u4e00-\u9fa5]{2,6})(?=[，,;；\s]|$)"
-
-            compiled_pattern = re.compile(name_list_pattern)
-
-            if compiled_pattern.search(curr_line_text):
-                return True
-            else:
-                return False
-
-        # """
-        def __get_text_catgr_by_nlp(curr_line_text):
-            """
-            This function checks if the current line is a name list using nlp model, such as spacy
-
-            Parameters
-            ----------
-            curr_line_text : str
-                text of the current line
-
-            Returns
-            -------
-            bool
-                True if the current line is a name list, False otherwise.
-            """
-
-            result = self.nlp_model.detect_entity_catgr_using_nlp(curr_line_text)
-
-            return result
-
-        # """
-
-        def __is_numbered_title(curr_line_text):
-            """
-            This function checks if the current line is a numbered list
-
-            Parameters
-            ----------
-            curr_line_text : str
-                text of the current line
-
-            Returns
-            -------
-            bool
-                True if the current line is a numbered list, False otherwise.
-            """
-
-            compiled_pattern = re.compile(self.numbered_title_pattern, re.VERBOSE)
-
-            if compiled_pattern.search(curr_line_text):
-                return True
-            else:
-                return False
-
-        def __is_end_with_ending_puncs(line_text):
-            """
-            This function checks if the current line ends with a ending punctuation mark
-
-            Parameters
-            ----------
-            line_text : str
-                text of the current line
-
-            Returns
-            -------
-            bool
-                True if the current line ends with a punctuation mark, False otherwise.
-            """
-            end_puncs = [".", "?", "!", "。", "？", "！", "…"]
-
-            line_text = line_text.rstrip()
-            if line_text[-1] in end_puncs:
-                return True
-
-            return False
-
-        def __contains_only_no_meaning_symbols(line_text):
-            """
-            This function checks if the current line contains only symbols that have no meaning, if so, it is not a title.
-            Situation contains:
-            1. Only have punctuation marks
-            2. Only have other non-meaning symbols
-
-            Parameters
-            ----------
-            line_text : str
-                text of the current line
-
-            Returns
-            -------
-            bool
-                True if the current line contains only symbols that have no meaning, False otherwise.
-            """
-
-            punctuation_marks = re.findall(r"[^\w\s]", line_text)  # find all punctuation marks
-            number_of_punctuation_marks = len(punctuation_marks)
-
-            text_length = len(line_text)
-
-            if text_length == 0:
-                return False
-
-            punctuation_ratio = number_of_punctuation_marks / text_length
-            if punctuation_ratio >= 0.9:
-                return True
-
-            return False
-
-        def __is_equation(line_text):
-            """
-            This function checks if the current line is an equation.
-
-            Parameters
-            ----------
-            line_text : str
-
-            Returns
-            -------
-            bool
-                True if the current line is an equation, False otherwise.
-            """
-            equation_reg = r"\$.*?\\overline.*?\$"  # to match interline equations
-
-            if re.search(equation_reg, line_text):
-                return True
-            else:
-                return False
-
-        def __is_title_by_len(text, max_length=200):
-            """
-            This function checks if the current line is a title by length.
-
-            Parameters
-            ----------
-            text : str
-                text of the current line
-
-            max_length : int
-                max length of the title
-
-            Returns
-            -------
-            bool
-                True if the current line is a title, False otherwise.
-
-            """
-            text = text.strip()
-            return len(text) <= max_length
-
-        def __compute_line_font_type_and_size(curr_line):
-            """
-            This function computes the font type and font size of the line.
-
-            Parameters
-            ----------
-            line : dict
-                line
-
-            Returns
-            -------
-            font_type : str
-                font type of the line
-            font_size : float
-                font size of the line
-            """
-            spans = curr_line["spans"]
-            max_accumulated_length = 0
-            max_span_font_size = curr_line["spans"][0]["size"]  # default value, float type
-            max_span_font_type = curr_line["spans"][0]["font"].lower()  # default value, string type
-            for span in spans:
-                if span["text"].isspace():
-                    continue
-                span_length = span["bbox"][2] - span["bbox"][0]
-                if span_length > max_accumulated_length:
-                    max_accumulated_length = span_length
-                    max_span_font_size = span["size"]
-                    max_span_font_type = span["font"].lower()
-
-            return max_span_font_type, max_span_font_size
-
-        """
-        Title detecting main Process.
-        """
-
-        """
-        Basic features about the current line.
-        """
-        curr_line_bbox = curr_line["bbox"]
-        curr_line_text = curr_line["text"]
-        curr_line_font_type, curr_line_font_size = __compute_line_font_type_and_size(curr_line)
-
-        if len(curr_line_text.strip()) == 0:  # skip empty lines
-            return False
-
-        prev_line_bbox = prev_line["bbox"] if prev_line else None
-        if prev_line:
-            prev_line_font_type, prev_line_font_size = __compute_line_font_type_and_size(prev_line)
-        else:
-            prev_line_font_type, prev_line_font_size = None, None
-
-        next_line_bbox = next_line["bbox"] if next_line else None
-        if next_line:
-            next_line_font_type, next_line_font_size = __compute_line_font_type_and_size(next_line)
-        else:
-            next_line_font_type, next_line_font_size = None, None
-
-        """
-        Aggregated features about the current line.
-        """
-        is_italc_font = __is_italic_font_line(curr_line)
-        is_bold_font = __is_bold_font_line(curr_line)
-
-        is_font_size_little_less_than_doc_avg = __is_font_size_not_less_than_docAvg(curr_line_font_size, ratio=0.8)
-        is_font_size_not_less_than_doc_avg = __is_font_size_not_less_than_docAvg(curr_line_font_size, ratio=1)
-        is_much_larger_font_than_doc_avg = __is_font_size_not_less_than_docAvg(curr_line_font_size, ratio=1.6)
-
-        is_not_same_font_type_of_docAvg = not __is_same_font_type_of_docAvg(curr_line_font_type)
-
-        is_potential_title_font = is_bold_font or is_font_size_not_less_than_doc_avg or is_not_same_font_type_of_docAvg
-
-        is_mix_font_styles_strict = __has_mixed_font_styles(curr_line["spans"], strict_mode=True)
-        is_mix_font_styles_loose = __has_mixed_font_styles(curr_line["spans"], strict_mode=False)
-
-        is_punctuation_heavy = __is_punctuation_heavy(curr_line_text)
-
-        is_word_list_line_by_rules = __is_word_list_line_by_rules(curr_line_text)
-        is_person_or_org_list_line_by_nlp = __get_text_catgr_by_nlp(curr_line_text) in ["PERSON", "GPE", "ORG"]
-
-        is_font_size_larger_than_neighbors = __is_larger_font_size_from_neighbors(
-            curr_line_font_size, prev_line_font_size, next_line_font_size
-        )
-
-        is_font_type_diff_from_neighbors = __is_different_font_type_from_neighbors(
-            curr_line_font_type, prev_line_font_type, next_line_font_type
-        )
-
-        has_sufficient_spaces_above, has_sufficient_spaces_below = __is_sufficient_spacing_above_and_below(
-            curr_line_bbox, prev_line_bbox, next_line_bbox, avg_char_height, median_font_size
-        )
-
-        is_similar_to_pre_line = __is_similar_to_pre_line(
-            curr_line_font_type, prev_line_font_type, curr_line_font_size, prev_line_font_size
-        )
-
-        """
-        Further aggregated features about the current line.
-        
-        Attention:
-            Features that start with __ are for internal use.
-        """
-
-        __is_line_left_aligned_from_neighbors = is_line_left_aligned_from_neighbors(
-            curr_line_bbox, prev_line_bbox, next_line_bbox, avg_char_width
-        )
-        __is_font_diff_from_neighbors = is_font_size_larger_than_neighbors or is_font_type_diff_from_neighbors
-        is_a_left_inline_title = (
-            is_mix_font_styles_strict and __is_line_left_aligned_from_neighbors and __is_font_diff_from_neighbors
-        )
-
-        is_title_by_check_prev_line = prev_line is None and has_sufficient_spaces_above and is_potential_title_font
-        is_title_by_check_next_line = next_line is None and has_sufficient_spaces_below and is_potential_title_font
-
-        is_title_by_check_pre_and_next_line = (
-            (prev_line is not None or next_line is not None)
-            and has_sufficient_spaces_above
-            and has_sufficient_spaces_below
-            and is_potential_title_font
-        )
-
-        is_numbered_title = __is_numbered_title(curr_line_text) and (
-            (has_sufficient_spaces_above or prev_line is None) and (has_sufficient_spaces_below or next_line is None)
-        )
-
-        is_not_end_with_ending_puncs = not __is_end_with_ending_puncs(curr_line_text)
-
-        is_not_only_no_meaning_symbols = not __contains_only_no_meaning_symbols(curr_line_text)
-
-        is_equation = __is_equation(curr_line_text)
-
-        is_title_by_len = __is_title_by_len(curr_line_text)
-
-        """
-        Decide if the line is a title.
-        """
-        # is_title = False
-        # if prev_line_is_title:
-
-        is_title = (
-            is_not_end_with_ending_puncs  # not end with ending punctuation marks
-            and is_not_only_no_meaning_symbols  # not only have no meaning symbols
-            and is_title_by_len  # is a title by length, default max length is 200
-            and not is_equation  # an interline equation should never be a title
-            and is_potential_title_font  # is a potential title font, which is bold or larger than the document average font size or not the same font type as the document average font type
-            and (
-                (is_not_same_font_type_of_docAvg and is_font_size_not_less_than_doc_avg)
-                or (is_bold_font and is_much_larger_font_than_doc_avg and is_not_same_font_type_of_docAvg)
-                or (
-                    is_much_larger_font_than_doc_avg
-                    and (is_title_by_check_prev_line or is_title_by_check_next_line or is_title_by_check_pre_and_next_line)
-                )
-                or (
-                    is_font_size_little_less_than_doc_avg
-                    and is_bold_font
-                    and (is_title_by_check_prev_line or is_title_by_check_next_line or is_title_by_check_pre_and_next_line)
-                )
-            )  # not the same font type as the document average font type, which includes the most common font type and the second most common font type
-            and (
-                (
-                    not is_person_or_org_list_line_by_nlp
-                    and (
-                        is_much_larger_font_than_doc_avg
-                        or (is_not_same_font_type_of_docAvg and is_font_size_not_less_than_doc_avg)
-                    )
-                )
-                or (
-                    not (is_word_list_line_by_rules and is_person_or_org_list_line_by_nlp)
-                    and not is_a_left_inline_title
-                    and not is_punctuation_heavy
-                    and (is_title_by_check_prev_line or is_title_by_check_next_line or is_title_by_check_pre_and_next_line)
-                )
-                or (
-                    is_person_or_org_list_line_by_nlp
-                    and (is_bold_font and is_much_larger_font_than_doc_avg and is_not_same_font_type_of_docAvg)
-                    and (is_bold_font and is_much_larger_font_than_doc_avg and is_not_same_font_type_of_docAvg)
-                )
-                or (is_numbered_title and not is_a_left_inline_title)
-            )
-        )
-        # ) or (is_similar_to_pre_line and prev_line_is_title)
-
-        is_name_or_org_list_to_be_removed = (
-            (is_person_or_org_list_line_by_nlp)
-            and is_punctuation_heavy
-            and (is_title_by_check_prev_line or is_title_by_check_next_line or is_title_by_check_pre_and_next_line)
-        ) and not is_title
-
-        if is_name_or_org_list_to_be_removed:
-            is_author_or_org_list = True
-            # print curr_line_text to check
-            # print_yellow(f"Text of is_author_or_org_list: {curr_line_text}")
-        else:
-            is_author_or_org_list = False
-        """
-        # print reason why the line is a title
-        if is_title:
-            print_green("This line is a title.")
-            print_green("↓" * 10)
-            print()
-            print("curr_line_text: ", curr_line_text)
-            print()
-
-        # print reason why the line is not a title
-        line_text = curr_line_text.strip()
-        test_text = "Career/Personal Life"
-        text_content_condition = line_text == test_text
-        
-        if not is_title and text_content_condition: # Print specific line
-        # if not is_title: # Print each line
-            print_red("This line is not a title.")
-            print_red("↓" * 10)
-
-            print()
-            print("curr_line_text: ", curr_line_text)
-            print()
-
-            if is_not_end_with_ending_puncs:
-                print_green(f"is_not_end_with_ending_puncs")
-            else:
-                print_red(f"is_end_with_ending_puncs")
-
-            if is_not_only_no_meaning_symbols:
-                print_green(f"is_not_only_no_meaning_symbols")
-            else:
-                print_red(f"is_only_no_meaning_symbols")
-
-            if is_title_by_len:
-                print_green(f"is_title_by_len: {is_title_by_len}")
-            else:
-                print_red(f"is_not_title_by_len: {is_title_by_len}")
-
-            if is_equation:
-                print_red(f"is_equation")
-            else:
-                print_green(f"is_not_equation")
-
-            if is_potential_title_font:
-                print_green(f"is_potential_title_font")
-            else:
-                print_red(f"is_not_potential_title_font")
-
-            if is_punctuation_heavy:
-                print_red("is_punctuation_heavy")
-            else:
-                print_green("is_not_punctuation_heavy")
-
-            if is_bold_font:
-                print_green(f"is_bold_font")
-            else:
-                print_red(f"is_not_bold_font")
-
-            if is_font_size_not_less_than_doc_avg:
-                print_green(f"is_larger_font_than_doc_avg")
-            else:
-                print_red(f"is_not_larger_font_than_doc_avg")
-
-            if is_much_larger_font_than_doc_avg:
-                print_green(f"is_much_larger_font_than_doc_avg")
-            else:
-                print_red(f"is_not_much_larger_font_than_doc_avg")
-
-            if is_not_same_font_type_of_docAvg:
-                print_green(f"is_not_same_font_type_of_docAvg")
-            else:
-                print_red(f"is_same_font_type_of_docAvg")
-
-            if is_word_list_line_by_rules:
-                print_red("is_word_list_line_by_rules")
-            else:
-                print_green("is_not_name_list_by_rules")
-
-            if is_person_or_org_list_line_by_nlp:
-                print_red("is_person_or_org_list_line_by_nlp")
-            else:
-                print_green("is_not_person_or_org_list_line_by_nlp")
-
-            if not is_numbered_title:
-                print_red("is_not_numbered_title")
-            else:
-                print_green("is_numbered_title")
-
-            if is_a_left_inline_title:
-                print_red("is_a_left_inline_title")
-            else:
-                print_green("is_not_a_left_inline_title")
-
-            if not is_title_by_check_prev_line:
-                print_red("is_not_title_by_check_prev_line")
-            else:
-                print_green("is_title_by_check_prev_line")
-
-            if not is_title_by_check_next_line:
-                print_red("is_not_title_by_check_next_line")
-            else:
-                print_green("is_title_by_check_next_line")
-
-            if not is_title_by_check_pre_and_next_line:
-                print_red("is_not_title_by_check_pre_and_next_line")
-            else:
-                print_green("is_title_by_check_pre_and_next_line")
-
-        # print_green("Common features:")
-        # print_green("↓" * 10)
-
-        # print(f"    curr_line_font_type: {curr_line_font_type}")
-        # print(f"    curr_line_font_size: {curr_line_font_size}")
-        # print()
-
-        """
-
-        return is_title, is_author_or_org_list
-
-    def _detect_block_title(self, input_block):
-        """
-        Use the functions 'is_potential_title' to detect titles of each paragraph block.
-        If a line is a title, then the value of key 'is_title' of the line will be set to True.
-        """
-
-        raw_lines = input_block["lines"]
-
-        prev_line_is_title_flag = False
-
-        for i, curr_line in enumerate(raw_lines):
-            prev_line = raw_lines[i - 1] if i > 0 else None
-            next_line = raw_lines[i + 1] if i < len(raw_lines) - 1 else None
-
-            blk_avg_char_width = input_block["avg_char_width"]
-            blk_avg_char_height = input_block["avg_char_height"]
-            blk_media_font_size = input_block["median_font_size"]
-
-            is_title, is_author_or_org_list = self._is_potential_title(
-                curr_line,
-                prev_line,
-                prev_line_is_title_flag,
-                next_line,
-                blk_avg_char_width,
-                blk_avg_char_height,
-                blk_media_font_size,
-            )
-
-            if is_title:
-                curr_line["is_title"] = is_title
-                prev_line_is_title_flag = True
-            else:
-                curr_line["is_title"] = False
-                prev_line_is_title_flag = False
-
-            if is_author_or_org_list:
-                curr_line["is_author_or_org_list"] = is_author_or_org_list
-            else:
-                curr_line["is_author_or_org_list"] = False
-
-        return input_block
-
-    def batch_process_blocks_detect_titles(self, pdf_dic):
-        """
-        This function batch process the blocks to detect titles.
-
-        Parameters
-        ----------
-        pdf_dict : dict
-            result dictionary
-
-        Returns
-        -------
-        pdf_dict : dict
-            result dictionary
-        """
-        num_titles = 0
-
-        for page_id, blocks in pdf_dic.items():
-            if page_id.startswith("page_"):
-                para_blocks = []
-                if "para_blocks" in blocks.keys():
-                    para_blocks = blocks["para_blocks"]
-
-                    all_single_line_blocks = []
-                    for block in para_blocks:
-                        if len(block["lines"]) == 1:
-                            all_single_line_blocks.append(block)
-
-                    new_para_blocks = []
-                    if not len(all_single_line_blocks) == len(para_blocks):  # Not all blocks are single line blocks.
-                        for para_block in para_blocks:
-                            new_block = self._detect_block_title(para_block)
-                            new_para_blocks.append(new_block)
-                            num_titles += sum([line.get("is_title", 0) for line in new_block["lines"]])
-                    else:  # All blocks are single line blocks.
-                        for para_block in para_blocks:
-                            new_para_blocks.append(para_block)
-                            num_titles += sum([line.get("is_title", 0) for line in para_block["lines"]])
-                    para_blocks = new_para_blocks
-
-                blocks["para_blocks"] = para_blocks
-
-                for para_block in para_blocks:
-                    all_titles = all(safe_get(line, "is_title", False) for line in para_block["lines"])
-                    para_text_len = sum([len(line["text"]) for line in para_block["lines"]])
-                    if (
-                        all_titles and para_text_len < 200
-                    ):  # total length of the paragraph is less than 200, more than this should not be a title
-                        para_block["is_block_title"] = 1
-                    else:
-                        para_block["is_block_title"] = 0
-
-                    all_name_or_org_list_to_be_removed = all(
-                        safe_get(line, "is_author_or_org_list", False) for line in para_block["lines"]
-                    )
-                    if all_name_or_org_list_to_be_removed and page_id == "page_0":
-                        para_block["is_block_an_author_or_org_list"] = 1
-                    else:
-                        para_block["is_block_an_author_or_org_list"] = 0
-
-        pdf_dic["statistics"]["num_titles"] = num_titles
-
-        return pdf_dic
-
-    def __determine_size_based_level(self, title_blocks):
-        """
-        This function determines the title level based on the font size of the title.
-
-        Parameters
-        ----------
-        title_blocks : list
-
-        Returns
-        -------
-        title_blocks : list
-        """
-
-        font_sizes = np.array([safe_get(tb["block"], "block_font_size", 0) for tb in title_blocks])
-
-        # Use the mean and std of font sizes to remove extreme values
-        mean_font_size = np.mean(font_sizes)
-        std_font_size = np.std(font_sizes)
-        min_extreme_font_size = mean_font_size - std_font_size  # type: ignore
-        max_extreme_font_size = mean_font_size + std_font_size  # type: ignore
-
-        # Compute the threshold for title level
-        middle_font_sizes = font_sizes[(font_sizes > min_extreme_font_size) & (font_sizes < max_extreme_font_size)]
-        if middle_font_sizes.size > 0:
-            middle_mean_font_size = np.mean(middle_font_sizes)
-            level_threshold = middle_mean_font_size
-        else:
-            level_threshold = mean_font_size
-
-        for tb in title_blocks:
-            title_block = tb["block"]
-            title_font_size = safe_get(title_block, "block_font_size", 0)
-
-            current_level = 1  # Initialize title level, the biggest level is 1
-
-            # print(f"Before adjustment by font size, {current_level}")
-            if title_font_size >= max_extreme_font_size:
-                current_level = 1
-            elif title_font_size <= min_extreme_font_size:
-                current_level = 3
-            elif float(title_font_size) >= float(level_threshold):
-                current_level = 2
-            else:
-                current_level = 3
-            # print(f"After adjustment by font size, {current_level}")
-
-            title_block["block_title_level"] = current_level
-
-        return title_blocks
-
-    def batch_process_blocks_recog_title_level(self, pdf_dic):
-        title_blocks = []
-
-        # Collect all titles
-        for page_id, blocks in pdf_dic.items():
-            if page_id.startswith("page_"):
-                para_blocks = blocks.get("para_blocks", [])
-                for block in para_blocks:
-                    if block.get("is_block_title"):
-                        title_obj = {"page_id": page_id, "block": block}
-                        title_blocks.append(title_obj)
-
-        # Determine title level
-        if title_blocks:
-            # Determine title level based on font size
-            title_blocks = self.__determine_size_based_level(title_blocks)
-
-        return pdf_dic