docling 2.26.0__py3-none-any.whl → 2.27.0__py3-none-any.whl

docling/backend/asciidoc_backend.py

@@ -380,7 +380,7 @@ class AsciiDocBackend(DeclarativeDocumentBackend):
                     end_row_offset_idx=row_idx + row_span,
                     start_col_offset_idx=col_idx,
                     end_col_offset_idx=col_idx + col_span,
-                    col_header=False,
+                    column_header=row_idx == 0,
                     row_header=False,
                 )
                 data.table_cells.append(cell)

docling/backend/csv_backend.py

@@ -111,7 +111,7 @@ class CsvDocumentBackend(DeclarativeDocumentBackend):
                             end_row_offset_idx=row_idx + 1,
                             start_col_offset_idx=col_idx,
                             end_col_offset_idx=col_idx + 1,
-                            col_header=row_idx == 0,  # First row as header
+                            column_header=row_idx == 0,  # First row as header
                             row_header=False,
                         )
                         table_data.table_cells.append(cell)

docling/backend/docling_parse_backend.py

@@ -6,12 +6,12 @@ from typing import Iterable, List, Optional, Union
 
 import pypdfium2 as pdfium
 from docling_core.types.doc import BoundingBox, CoordOrigin, Size
+from docling_core.types.doc.page import BoundingRectangle, SegmentedPdfPage, TextCell
 from docling_parse.pdf_parsers import pdf_parser_v1
 from PIL import Image, ImageDraw
 from pypdfium2 import PdfPage
 
 from docling.backend.pdf_backend import PdfDocumentBackend, PdfPageBackend
-from docling.datamodel.base_models import Cell
 from docling.datamodel.document import InputDocument
 
 _log = logging.getLogger(__name__)
@@ -68,8 +68,11 @@ class DoclingParsePageBackend(PdfPageBackend):
 
         return text_piece
 
-    def get_text_cells(self) -> Iterable[Cell]:
-        cells: List[Cell] = []
+    def get_segmented_page(self) -> Optional[SegmentedPdfPage]:
+        return None
+
+    def get_text_cells(self) -> Iterable[TextCell]:
+        cells: List[TextCell] = []
         cell_counter = 0
 
         if not self.valid:
@@ -91,19 +94,24 @@ class DoclingParsePageBackend(PdfPageBackend):
 
             text_piece = self._dpage["cells"][i]["content"]["rnormalized"]
             cells.append(
-                Cell(
-                    id=cell_counter,
+                TextCell(
+                    index=cell_counter,
                     text=text_piece,
-                    bbox=BoundingBox(
-                        # l=x0, b=y0, r=x1, t=y1,
-                        l=x0 * page_size.width / parser_width,
-                        b=y0 * page_size.height / parser_height,
-                        r=x1 * page_size.width / parser_width,
-                        t=y1 * page_size.height / parser_height,
-                        coord_origin=CoordOrigin.BOTTOMLEFT,
+                    orig=text_piece,
+                    from_ocr=False,
+                    rect=BoundingRectangle.from_bounding_box(
+                        BoundingBox(
+                            # l=x0, b=y0, r=x1, t=y1,
+                            l=x0 * page_size.width / parser_width,
+                            b=y0 * page_size.height / parser_height,
+                            r=x1 * page_size.width / parser_width,
+                            t=y1 * page_size.height / parser_height,
+                            coord_origin=CoordOrigin.BOTTOMLEFT,
+                        )
                     ).to_top_left_origin(page_size.height),
                 )
             )
+
             cell_counter += 1
 
         def draw_clusters_and_cells():
@@ -112,7 +120,7 @@ class DoclingParsePageBackend(PdfPageBackend):
             )  # make new image to avoid drawing on the saved ones
             draw = ImageDraw.Draw(image)
             for c in cells:
-                x0, y0, x1, y1 = c.bbox.as_tuple()
+                x0, y0, x1, y1 = c.rect.to_bounding_box().as_tuple()
                 cell_color = (
                     random.randint(30, 140),
                     random.randint(30, 140),

docling/backend/docling_parse_v2_backend.py

@@ -6,12 +6,13 @@ from typing import TYPE_CHECKING, Iterable, List, Optional, Union
 
 import pypdfium2 as pdfium
 from docling_core.types.doc import BoundingBox, CoordOrigin
+from docling_core.types.doc.page import BoundingRectangle, SegmentedPdfPage, TextCell
 from docling_parse.pdf_parsers import pdf_parser_v2
 from PIL import Image, ImageDraw
 from pypdfium2 import PdfPage
 
 from docling.backend.pdf_backend import PdfDocumentBackend, PdfPageBackend
-from docling.datamodel.base_models import Cell, Size
+from docling.datamodel.base_models import Size
 from docling.utils.locks import pypdfium2_lock
 
 if TYPE_CHECKING:
@@ -78,8 +79,11 @@ class DoclingParseV2PageBackend(PdfPageBackend):
 
         return text_piece
 
-    def get_text_cells(self) -> Iterable[Cell]:
-        cells: List[Cell] = []
+    def get_segmented_page(self) -> Optional[SegmentedPdfPage]:
+        return None
+
+    def get_text_cells(self) -> Iterable[TextCell]:
+        cells: List[TextCell] = []
         cell_counter = 0
 
         if not self.valid:
@@ -106,16 +110,20 @@ class DoclingParseV2PageBackend(PdfPageBackend):
 
             text_piece = cell_data[cells_header.index("text")]
             cells.append(
-                Cell(
-                    id=cell_counter,
+                TextCell(
+                    index=cell_counter,
                     text=text_piece,
-                    bbox=BoundingBox(
-                        # l=x0, b=y0, r=x1, t=y1,
-                        l=x0 * page_size.width / parser_width,
-                        b=y0 * page_size.height / parser_height,
-                        r=x1 * page_size.width / parser_width,
-                        t=y1 * page_size.height / parser_height,
-                        coord_origin=CoordOrigin.BOTTOMLEFT,
+                    orig=text_piece,
+                    from_ocr=False,
+                    rect=BoundingRectangle.from_bounding_box(
+                        BoundingBox(
+                            # l=x0, b=y0, r=x1, t=y1,
+                            l=x0 * page_size.width / parser_width,
+                            b=y0 * page_size.height / parser_height,
+                            r=x1 * page_size.width / parser_width,
+                            t=y1 * page_size.height / parser_height,
+                            coord_origin=CoordOrigin.BOTTOMLEFT,
+                        )
                     ).to_top_left_origin(page_size.height),
                 )
             )

docling/backend/docling_parse_v4_backend.py

@@ -0,0 +1,185 @@
+import logging
+import random
+from io import BytesIO
+from pathlib import Path
+from typing import TYPE_CHECKING, Iterable, List, Optional, Union
+
+import pypdfium2 as pdfium
+from docling_core.types.doc import BoundingBox, CoordOrigin
+from docling_core.types.doc.page import SegmentedPdfPage, TextCell
+from docling_parse.pdf_parser import DoclingPdfParser, PdfDocument
+from PIL import Image, ImageDraw
+from pypdfium2 import PdfPage
+
+from docling.backend.pdf_backend import PdfDocumentBackend, PdfPageBackend
+from docling.datamodel.base_models import Size
+from docling.utils.locks import pypdfium2_lock
+
+if TYPE_CHECKING:
+    from docling.datamodel.document import InputDocument
+
+_log = logging.getLogger(__name__)
+
+
+class DoclingParseV4PageBackend(PdfPageBackend):
+    def __init__(self, parsed_page: SegmentedPdfPage, page_obj: PdfPage):
+        self._ppage = page_obj
+        self._dpage = parsed_page
+        self.valid = parsed_page is not None
+
+    def is_valid(self) -> bool:
+        return self.valid
+
+    def get_text_in_rect(self, bbox: BoundingBox) -> str:
+        # Find intersecting cells on the page
+        text_piece = ""
+        page_size = self.get_size()
+
+        scale = (
+            1  # FIX - Replace with param in get_text_in_rect across backends (optional)
+        )
+
+        for i, cell in enumerate(self._dpage.textline_cells):
+            cell_bbox = (
+                cell.rect.to_bounding_box()
+                .to_top_left_origin(page_height=page_size.height)
+                .scaled(scale)
+            )
+
+            overlap_frac = cell_bbox.intersection_area_with(bbox) / cell_bbox.area()
+
+            if overlap_frac > 0.5:
+                if len(text_piece) > 0:
+                    text_piece += " "
+                text_piece += cell.text
+
+        return text_piece
+
+    def get_segmented_page(self) -> Optional[SegmentedPdfPage]:
+        return self._dpage
+
+    def get_text_cells(self) -> Iterable[TextCell]:
+        page_size = self.get_size()
+
+        [tc.to_top_left_origin(page_size.height) for tc in self._dpage.textline_cells]
+
+        # for cell in self._dpage.textline_cells:
+        #     rect = cell.rect
+        #
+        #     assert (
+        #         rect.to_bounding_box().l <= rect.to_bounding_box().r
+        #     ), f"left is > right on bounding box {rect.to_bounding_box()} of rect {rect}"
+        #     assert (
+        #         rect.to_bounding_box().t <= rect.to_bounding_box().b
+        #     ), f"top is > bottom on bounding box {rect.to_bounding_box()} of rect {rect}"
+
+        return self._dpage.textline_cells
+
+    def get_bitmap_rects(self, scale: float = 1) -> Iterable[BoundingBox]:
+        AREA_THRESHOLD = 0  # 32 * 32
+
+        images = self._dpage.bitmap_resources
+
+        for img in images:
+            cropbox = img.rect.to_bounding_box().to_top_left_origin(
+                self.get_size().height
+            )
+
+            if cropbox.area() > AREA_THRESHOLD:
+                cropbox = cropbox.scaled(scale=scale)
+
+                yield cropbox
+
+    def get_page_image(
+        self, scale: float = 1, cropbox: Optional[BoundingBox] = None
+    ) -> Image.Image:
+
+        page_size = self.get_size()
+
+        if not cropbox:
+            cropbox = BoundingBox(
+                l=0,
+                r=page_size.width,
+                t=0,
+                b=page_size.height,
+                coord_origin=CoordOrigin.TOPLEFT,
+            )
+            padbox = BoundingBox(
+                l=0, r=0, t=0, b=0, coord_origin=CoordOrigin.BOTTOMLEFT
+            )
+        else:
+            padbox = cropbox.to_bottom_left_origin(page_size.height).model_copy()
+            padbox.r = page_size.width - padbox.r
+            padbox.t = page_size.height - padbox.t
+
+        image = (
+            self._ppage.render(
+                scale=scale * 1.5,
+                rotation=0,  # no additional rotation
+                crop=padbox.as_tuple(),
+            )
+            .to_pil()
+            .resize(size=(round(cropbox.width * scale), round(cropbox.height * scale)))
+        )  # We resize the image from 1.5x the given scale to make it sharper.
+
+        return image
+
+    def get_size(self) -> Size:
+        return Size(
+            width=self._dpage.dimension.width,
+            height=self._dpage.dimension.height,
+        )
+
+    def unload(self):
+        self._ppage = None
+        self._dpage = None
+
+
+class DoclingParseV4DocumentBackend(PdfDocumentBackend):
+    def __init__(self, in_doc: "InputDocument", path_or_stream: Union[BytesIO, Path]):
+        super().__init__(in_doc, path_or_stream)
+
+        with pypdfium2_lock:
+            self._pdoc = pdfium.PdfDocument(self.path_or_stream)
+        self.parser = DoclingPdfParser(loglevel="fatal")
+        self.dp_doc: PdfDocument = self.parser.load(path_or_stream=self.path_or_stream)
+        success = self.dp_doc is not None
+
+        if not success:
+            raise RuntimeError(
+                f"docling-parse v4 could not load document {self.document_hash}."
+            )
+
+    def page_count(self) -> int:
+        # return len(self._pdoc)  # To be replaced with docling-parse API
+
+        len_1 = len(self._pdoc)
+        len_2 = self.dp_doc.number_of_pages()
+
+        if len_1 != len_2:
+            _log.error(f"Inconsistent number of pages: {len_1}!={len_2}")
+
+        return len_2
+
+    def load_page(
+        self, page_no: int, create_words: bool = True, create_textlines: bool = True
+    ) -> DoclingParseV4PageBackend:
+        with pypdfium2_lock:
+            return DoclingParseV4PageBackend(
+                self.dp_doc.get_page(
+                    page_no + 1,
+                    create_words=create_words,
+                    create_textlines=create_textlines,
+                ),
+                self._pdoc[page_no],
+            )
+
+    def is_valid(self) -> bool:
+        return self.page_count() > 0
+
+    def unload(self):
+        super().unload()
+        self.dp_doc.unload()
+        with pypdfium2_lock:
+            self._pdoc.close()
+        self._pdoc = None

docling/backend/docx/latex/latex_dict.py

@@ -0,0 +1,271 @@
+# -*- coding: utf-8 -*-
+
+"""
+Adapted from https://github.com/xiilei/dwml/blob/master/dwml/latex_dict.py
+On 23/01/2025
+"""
+
+from __future__ import unicode_literals
+
+CHARS = ("{", "}", "_", "^", "#", "&", "$", "%", "~")
+
+BLANK = ""
+BACKSLASH = "\\"
+ALN = "&"
+
+CHR = {
+    # Unicode : Latex Math Symbols
+    # Top accents
+    "\u0300": "\\grave{{{0}}}",
+    "\u0301": "\\acute{{{0}}}",
+    "\u0302": "\\hat{{{0}}}",
+    "\u0303": "\\tilde{{{0}}}",
+    "\u0304": "\\bar{{{0}}}",
+    "\u0305": "\\overbar{{{0}}}",
+    "\u0306": "\\breve{{{0}}}",
+    "\u0307": "\\dot{{{0}}}",
+    "\u0308": "\\ddot{{{0}}}",
+    "\u0309": "\\ovhook{{{0}}}",
+    "\u030a": "\\ocirc{{{0}}}}",
+    "\u030c": "\\check{{{0}}}}",
+    "\u0310": "\\candra{{{0}}}",
+    "\u0312": "\\oturnedcomma{{{0}}}",
+    "\u0315": "\\ocommatopright{{{0}}}",
+    "\u031a": "\\droang{{{0}}}",
+    "\u0338": "\\not{{{0}}}",
+    "\u20d0": "\\leftharpoonaccent{{{0}}}",
+    "\u20d1": "\\rightharpoonaccent{{{0}}}",
+    "\u20d2": "\\vertoverlay{{{0}}}",
+    "\u20d6": "\\overleftarrow{{{0}}}",
+    "\u20d7": "\\vec{{{0}}}",
+    "\u20db": "\\dddot{{{0}}}",
+    "\u20dc": "\\ddddot{{{0}}}",
+    "\u20e1": "\\overleftrightarrow{{{0}}}",
+    "\u20e7": "\\annuity{{{0}}}",
+    "\u20e9": "\\widebridgeabove{{{0}}}",
+    "\u20f0": "\\asteraccent{{{0}}}",
+    # Bottom accents
+    "\u0330": "\\wideutilde{{{0}}}",
+    "\u0331": "\\underbar{{{0}}}",
+    "\u20e8": "\\threeunderdot{{{0}}}",
+    "\u20ec": "\\underrightharpoondown{{{0}}}",
+    "\u20ed": "\\underleftharpoondown{{{0}}}",
+    "\u20ee": "\\underledtarrow{{{0}}}",
+    "\u20ef": "\\underrightarrow{{{0}}}",
+    # Over | group
+    "\u23b4": "\\overbracket{{{0}}}",
+    "\u23dc": "\\overparen{{{0}}}",
+    "\u23de": "\\overbrace{{{0}}}",
+    # Under| group
+    "\u23b5": "\\underbracket{{{0}}}",
+    "\u23dd": "\\underparen{{{0}}}",
+    "\u23df": "\\underbrace{{{0}}}",
+}
+
+CHR_BO = {
+    # Big operators,
+    "\u2140": "\\Bbbsum",
+    "\u220f": "\\prod",
+    "\u2210": "\\coprod",
+    "\u2211": "\\sum",
+    "\u222b": "\\int",
+    "\u22c0": "\\bigwedge",
+    "\u22c1": "\\bigvee",
+    "\u22c2": "\\bigcap",
+    "\u22c3": "\\bigcup",
+    "\u2a00": "\\bigodot",
+    "\u2a01": "\\bigoplus",
+    "\u2a02": "\\bigotimes",
+}
+
+T = {
+    "\u2192": "\\rightarrow ",
+    # Greek letters
+    "\U0001d6fc": "\\alpha ",
+    "\U0001d6fd": "\\beta ",
+    "\U0001d6fe": "\\gamma ",
+    "\U0001d6ff": "\\theta ",
+    "\U0001d700": "\\epsilon ",
+    "\U0001d701": "\\zeta ",
+    "\U0001d702": "\\eta ",
+    "\U0001d703": "\\theta ",
+    "\U0001d704": "\\iota ",
+    "\U0001d705": "\\kappa ",
+    "\U0001d706": "\\lambda ",
+    "\U0001d707": "\\m ",
+    "\U0001d708": "\\n ",
+    "\U0001d709": "\\xi ",
+    "\U0001d70a": "\\omicron ",
+    "\U0001d70b": "\\pi ",
+    "\U0001d70c": "\\rho ",
+    "\U0001d70d": "\\varsigma ",
+    "\U0001d70e": "\\sigma ",
+    "\U0001d70f": "\\ta ",
+    "\U0001d710": "\\upsilon ",
+    "\U0001d711": "\\phi ",
+    "\U0001d712": "\\chi ",
+    "\U0001d713": "\\psi ",
+    "\U0001d714": "\\omega ",
+    "\U0001d715": "\\partial ",
+    "\U0001d716": "\\varepsilon ",
+    "\U0001d717": "\\vartheta ",
+    "\U0001d718": "\\varkappa ",
+    "\U0001d719": "\\varphi ",
+    "\U0001d71a": "\\varrho ",
+    "\U0001d71b": "\\varpi ",
+    # Relation symbols
+    "\u2190": "\\leftarrow ",
+    "\u2191": "\\uparrow ",
+    "\u2192": "\\rightarrow ",
+    "\u2193": "\\downright ",
+    "\u2194": "\\leftrightarrow ",
+    "\u2195": "\\updownarrow ",
+    "\u2196": "\\nwarrow ",
+    "\u2197": "\\nearrow ",
+    "\u2198": "\\searrow ",
+    "\u2199": "\\swarrow ",
+    "\u22ee": "\\vdots ",
+    "\u22ef": "\\cdots ",
+    "\u22f0": "\\adots ",
+    "\u22f1": "\\ddots ",
+    "\u2260": "\\ne ",
+    "\u2264": "\\leq ",
+    "\u2265": "\\geq ",
+    "\u2266": "\\leqq ",
+    "\u2267": "\\geqq ",
+    "\u2268": "\\lneqq ",
+    "\u2269": "\\gneqq ",
+    "\u226a": "\\ll ",
+    "\u226b": "\\gg ",
+    "\u2208": "\\in ",
+    "\u2209": "\\notin ",
+    "\u220b": "\\ni ",
+    "\u220c": "\\nni ",
+    # Ordinary symbols
+    "\u221e": "\\infty ",
+    # Binary relations
+    "\u00b1": "\\pm ",
+    "\u2213": "\\mp ",
+    # Italic, Latin, uppercase
+    "\U0001d434": "A",
+    "\U0001d435": "B",
+    "\U0001d436": "C",
+    "\U0001d437": "D",
+    "\U0001d438": "E",
+    "\U0001d439": "F",
+    "\U0001d43a": "G",
+    "\U0001d43b": "H",
+    "\U0001d43c": "I",
+    "\U0001d43d": "J",
+    "\U0001d43e": "K",
+    "\U0001d43f": "L",
+    "\U0001d440": "M",
+    "\U0001d441": "N",
+    "\U0001d442": "O",
+    "\U0001d443": "P",
+    "\U0001d444": "Q",
+    "\U0001d445": "R",
+    "\U0001d446": "S",
+    "\U0001d447": "T",
+    "\U0001d448": "U",
+    "\U0001d449": "V",
+    "\U0001d44a": "W",
+    "\U0001d44b": "X",
+    "\U0001d44c": "Y",
+    "\U0001d44d": "Z",
+    # Italic, Latin, lowercase
+    "\U0001d44e": "a",
+    "\U0001d44f": "b",
+    "\U0001d450": "c",
+    "\U0001d451": "d",
+    "\U0001d452": "e",
+    "\U0001d453": "f",
+    "\U0001d454": "g",
+    "\U0001d456": "i",
+    "\U0001d457": "j",
+    "\U0001d458": "k",
+    "\U0001d459": "l",
+    "\U0001d45a": "m",
+    "\U0001d45b": "n",
+    "\U0001d45c": "o",
+    "\U0001d45d": "p",
+    "\U0001d45e": "q",
+    "\U0001d45f": "r",
+    "\U0001d460": "s",
+    "\U0001d461": "t",
+    "\U0001d462": "u",
+    "\U0001d463": "v",
+    "\U0001d464": "w",
+    "\U0001d465": "x",
+    "\U0001d466": "y",
+    "\U0001d467": "z",
+}
+
+FUNC = {
+    "sin": "\\sin({fe})",
+    "cos": "\\cos({fe})",
+    "tan": "\\tan({fe})",
+    "arcsin": "\\arcsin({fe})",
+    "arccos": "\\arccos({fe})",
+    "arctan": "\\arctan({fe})",
+    "arccot": "\\arccot({fe})",
+    "sinh": "\\sinh({fe})",
+    "cosh": "\\cosh({fe})",
+    "tanh": "\\tanh({fe})",
+    "coth": "\\coth({fe})",
+    "sec": "\\sec({fe})",
+    "csc": "\\csc({fe})",
+}
+
+FUNC_PLACE = "{fe}"
+
+BRK = "\\\\"
+
+CHR_DEFAULT = {
+    "ACC_VAL": "\\hat{{{0}}}",
+}
+
+POS = {
+    "top": "\\overline{{{0}}}",  # not sure
+    "bot": "\\underline{{{0}}}",
+}
+
+POS_DEFAULT = {
+    "BAR_VAL": "\\overline{{{0}}}",
+}
+
+SUB = "_{{{0}}}"
+
+SUP = "^{{{0}}}"
+
+F = {
+    "bar": "\\frac{{{num}}}{{{den}}}",
+    "skw": r"^{{{num}}}/_{{{den}}}",
+    "noBar": "\\genfrac{{}}{{}}{{0pt}}{{}}{{{num}}}{{{den}}}",
+    "lin": "{{{num}}}/{{{den}}}",
+}
+F_DEFAULT = "\\frac{{{num}}}{{{den}}}"
+
+D = "\\left{left}{text}\\right{right}"
+
+D_DEFAULT = {
+    "left": "(",
+    "right": ")",
+    "null": ".",
+}
+
+RAD = "\\sqrt[{deg}]{{{text}}}"
+RAD_DEFAULT = "\\sqrt{{{text}}}"
+ARR = "{text}"
+
+LIM_FUNC = {
+    "lim": "\\lim_{{{lim}}}",
+    "max": "\\max_{{{lim}}}",
+    "min": "\\min_{{{lim}}}",
+}
+
+LIM_TO = ("\\rightarrow", "\\to")
+
+LIM_UPP = "\\overset{{{lim}}}{{{text}}}"
+
+M = "\\begin{{matrix}}{text}\\end{{matrix}}"