docling 2.9.0__py3-none-any.whl → 2.11.0__py3-none-any.whl

docling/backend/docling_parse_backend.py

@@ -6,7 +6,7 @@ from typing import Iterable, List, Optional, Union
 
 import pypdfium2 as pdfium
 from docling_core.types.doc import BoundingBox, CoordOrigin, Size
-from docling_parse.docling_parse import pdf_parser_v1
+from docling_parse.pdf_parsers import pdf_parser_v1
 from PIL import Image, ImageDraw
 from pypdfium2 import PdfPage
 

docling/backend/docling_parse_v2_backend.py

@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Iterable, List, Optional, Union
 
 import pypdfium2 as pdfium
 from docling_core.types.doc import BoundingBox, CoordOrigin
-from docling_parse.docling_parse import pdf_parser_v2
+from docling_parse.pdf_parsers import pdf_parser_v2
 from PIL import Image, ImageDraw
 from pypdfium2 import PdfPage
 
@@ -210,12 +210,14 @@ class DoclingParseV2DocumentBackend(PdfDocumentBackend):
         self.parser = pdf_parser_v2("fatal")
 
         success = False
-        if isinstance(path_or_stream, BytesIO):
+        if isinstance(self.path_or_stream, BytesIO):
             success = self.parser.load_document_from_bytesio(
-                self.document_hash, path_or_stream
+                self.document_hash, self.path_or_stream
+            )
+        elif isinstance(self.path_or_stream, Path):
+            success = self.parser.load_document(
+                self.document_hash, str(self.path_or_stream)
             )
-        elif isinstance(path_or_stream, Path):
-            success = self.parser.load_document(self.document_hash, str(path_or_stream))
 
         if not success:
             raise RuntimeError(

docling/cli/main.py

@@ -27,8 +27,10 @@ from docling.datamodel.base_models import (
 from docling.datamodel.document import ConversionResult
 from docling.datamodel.pipeline_options import (
     EasyOcrOptions,
+    OcrEngine,
     OcrMacOptions,
     OcrOptions,
+    PdfBackend,
     PdfPipelineOptions,
     RapidOcrOptions,
     TableFormerMode,
@@ -68,22 +70,6 @@ def version_callback(value: bool):
         raise typer.Exit()
 
 
-# Define an enum for the backend options
-class PdfBackend(str, Enum):
-    PYPDFIUM2 = "pypdfium2"
-    DLPARSE_V1 = "dlparse_v1"
-    DLPARSE_V2 = "dlparse_v2"
-
-
-# Define an enum for the ocr engines
-class OcrEngine(str, Enum):
-    EASYOCR = "easyocr"
-    TESSERACT_CLI = "tesseract_cli"
-    TESSERACT = "tesseract"
-    OCRMAC = "ocrmac"
-    RAPIDOCR = "rapidocr"
-
-
 def export_documents(
     conv_results: Iterable[ConversionResult],
     output_dir: Path,
@@ -208,7 +194,7 @@ def convert(
     ] = None,
     pdf_backend: Annotated[
         PdfBackend, typer.Option(..., help="The PDF backend to use.")
-    ] = PdfBackend.DLPARSE_V1,
+    ] = PdfBackend.DLPARSE_V2,
     table_mode: Annotated[
         TableFormerMode,
         typer.Option(..., help="The mode to use in the table structure model."),
@@ -264,6 +250,13 @@ def convert(
             help="Show version information.",
         ),
     ] = None,
+    document_timeout: Annotated[
+        Optional[float],
+        typer.Option(
+            ...,
+            help="The timeout for processing each document, in seconds.",
+        ),
+    ] = None,
 ):
     if verbose == 0:
         logging.basicConfig(level=logging.WARNING)
@@ -347,6 +340,7 @@ def convert(
             do_ocr=ocr,
             ocr_options=ocr_options,
             do_table_structure=True,
+            document_timeout=document_timeout,
         )
         pipeline_options.table_structure_options.do_cell_matching = (
             True  # do_cell_matching
@@ -372,11 +366,13 @@ def convert(
         else:
             raise RuntimeError(f"Unexpected PDF backend type {pdf_backend}")
 
+        pdf_format_option = PdfFormatOption(
+            pipeline_options=pipeline_options,
+            backend=backend,  # pdf_backend
+        )
         format_options: Dict[InputFormat, FormatOption] = {
-            InputFormat.PDF: PdfFormatOption(
-                pipeline_options=pipeline_options,
-                backend=backend,  # pdf_backend
-            )
+            InputFormat.PDF: pdf_format_option,
+            InputFormat.IMAGE: pdf_format_option,
         }
         doc_converter = DocumentConverter(
             allowed_formats=from_formats,

docling/datamodel/base_models.py

@@ -19,12 +19,12 @@ if TYPE_CHECKING:
 
 
 class ConversionStatus(str, Enum):
-    PENDING = auto()
-    STARTED = auto()
-    FAILURE = auto()
-    SUCCESS = auto()
-    PARTIAL_SUCCESS = auto()
-    SKIPPED = auto()
+    PENDING = "pending"
+    STARTED = "started"
+    FAILURE = "failure"
+    SUCCESS = "success"
+    PARTIAL_SUCCESS = "partial_success"
+    SKIPPED = "skipped"
 
 
 class InputFormat(str, Enum):
@@ -89,15 +89,15 @@ MimeTypeToFormat = {
 
 
 class DocInputType(str, Enum):
-    PATH = auto()
-    STREAM = auto()
+    PATH = "path"
+    STREAM = "stream"
 
 
 class DoclingComponentType(str, Enum):
-    DOCUMENT_BACKEND = auto()
-    MODEL = auto()
-    DOC_ASSEMBLER = auto()
-    USER_INPUT = auto()
+    DOCUMENT_BACKEND = "document_backend"
+    MODEL = "model"
+    DOC_ASSEMBLER = "doc_assembler"
+    USER_INPUT = "user_input"
 
 
 class ErrorItem(BaseModel):

docling/datamodel/document.py

@@ -33,6 +33,7 @@ from docling_core.types.legacy_doc.document import (
 from docling_core.types.legacy_doc.document import CCSFileInfoObject as DsFileInfoObject
 from docling_core.types.legacy_doc.document import ExportedCCSDocument as DsDocument
 from docling_core.utils.file import resolve_source_to_stream
+from docling_core.utils.legacy import docling_document_to_legacy
 from pydantic import BaseModel
 from typing_extensions import deprecated
 
@@ -189,259 +190,7 @@ class ConversionResult(BaseModel):
     @property
     @deprecated("Use document instead.")
     def legacy_document(self):
-        reverse_label_mapping = {
-            DocItemLabel.CAPTION.value: "Caption",
-            DocItemLabel.FOOTNOTE.value: "Footnote",
-            DocItemLabel.FORMULA.value: "Formula",
-            DocItemLabel.LIST_ITEM.value: "List-item",
-            DocItemLabel.PAGE_FOOTER.value: "Page-footer",
-            DocItemLabel.PAGE_HEADER.value: "Page-header",
-            DocItemLabel.PICTURE.value: "Picture",  # low threshold adjust to capture chemical structures for examples.
-            DocItemLabel.SECTION_HEADER.value: "Section-header",
-            DocItemLabel.TABLE.value: "Table",
-            DocItemLabel.TEXT.value: "Text",
-            DocItemLabel.TITLE.value: "Title",
-            DocItemLabel.DOCUMENT_INDEX.value: "Document Index",
-            DocItemLabel.CODE.value: "Code",
-            DocItemLabel.CHECKBOX_SELECTED.value: "Checkbox-Selected",
-            DocItemLabel.CHECKBOX_UNSELECTED.value: "Checkbox-Unselected",
-            DocItemLabel.FORM.value: "Form",
-            DocItemLabel.KEY_VALUE_REGION.value: "Key-Value Region",
-            DocItemLabel.PARAGRAPH.value: "paragraph",
-        }
-
-        title = ""
-        desc = DsDocumentDescription(logs=[])
-
-        page_hashes = [
-            PageReference(
-                hash=create_hash(self.input.document_hash + ":" + str(p.page_no - 1)),
-                page=p.page_no,
-                model="default",
-            )
-            for p in self.document.pages.values()
-        ]
-
-        file_info = DsFileInfoObject(
-            filename=self.input.file.name,
-            document_hash=self.input.document_hash,
-            num_pages=self.input.page_count,
-            page_hashes=page_hashes,
-        )
-
-        main_text = []
-        tables = []
-        figures = []
-        equations = []
-        footnotes = []
-        page_headers = []
-        page_footers = []
-
-        embedded_captions = set()
-        for ix, (item, level) in enumerate(
-            self.document.iterate_items(self.document.body)
-        ):
-
-            if isinstance(item, (TableItem, PictureItem)) and len(item.captions) > 0:
-                caption = item.caption_text(self.document)
-                if caption:
-                    embedded_captions.add(caption)
-
-        for item, level in self.document.iterate_items():
-            if isinstance(item, DocItem):
-                item_type = item.label
-
-                if isinstance(item, (TextItem, ListItem, SectionHeaderItem)):
-
-                    if isinstance(item, ListItem) and item.marker:
-                        text = f"{item.marker} {item.text}"
-                    else:
-                        text = item.text
-
-                    # Can be empty.
-                    prov = [
-                        Prov(
-                            bbox=p.bbox.as_tuple(),
-                            page=p.page_no,
-                            span=[0, len(item.text)],
-                        )
-                        for p in item.prov
-                    ]
-                    main_text.append(
-                        BaseText(
-                            text=text,
-                            obj_type=layout_label_to_ds_type.get(item.label),
-                            name=reverse_label_mapping[item.label],
-                            prov=prov,
-                        )
-                    )
-
-                    # skip captions of they are embedded in the actual
-                    # floating object
-                    if item_type == DocItemLabel.CAPTION and text in embedded_captions:
-                        continue
-
-                elif isinstance(item, TableItem) and item.data:
-                    index = len(tables)
-                    ref_str = f"#/tables/{index}"
-                    main_text.append(
-                        Ref(
-                            name=reverse_label_mapping[item.label],
-                            obj_type=layout_label_to_ds_type.get(item.label),
-                            ref=ref_str,
-                        ),
-                    )
-
-                    # Initialise empty table data grid (only empty cells)
-                    table_data = [
-                        [
-                            TableCell(
-                                text="",
-                                # bbox=[0,0,0,0],
-                                spans=[[i, j]],
-                                obj_type="body",
-                            )
-                            for j in range(item.data.num_cols)
-                        ]
-                        for i in range(item.data.num_rows)
-                    ]
-
-                    # Overwrite cells in table data for which there is actual cell content.
-                    for cell in item.data.table_cells:
-                        for i in range(
-                            min(cell.start_row_offset_idx, item.data.num_rows),
-                            min(cell.end_row_offset_idx, item.data.num_rows),
-                        ):
-                            for j in range(
-                                min(cell.start_col_offset_idx, item.data.num_cols),
-                                min(cell.end_col_offset_idx, item.data.num_cols),
-                            ):
-                                celltype = "body"
-                                if cell.column_header:
-                                    celltype = "col_header"
-                                elif cell.row_header:
-                                    celltype = "row_header"
-                                elif cell.row_section:
-                                    celltype = "row_section"
-
-                                def make_spans(cell):
-                                    for rspan in range(
-                                        min(
-                                            cell.start_row_offset_idx,
-                                            item.data.num_rows,
-                                        ),
-                                        min(
-                                            cell.end_row_offset_idx, item.data.num_rows
-                                        ),
-                                    ):
-                                        for cspan in range(
-                                            min(
-                                                cell.start_col_offset_idx,
-                                                item.data.num_cols,
-                                            ),
-                                            min(
-                                                cell.end_col_offset_idx,
-                                                item.data.num_cols,
-                                            ),
-                                        ):
-                                            yield [rspan, cspan]
-
-                                spans = list(make_spans(cell))
-                                table_data[i][j] = GlmTableCell(
-                                    text=cell.text,
-                                    bbox=(
-                                        cell.bbox.as_tuple()
-                                        if cell.bbox is not None
-                                        else None
-                                    ),  # check if this is bottom-left
-                                    spans=spans,
-                                    obj_type=celltype,
-                                    col=j,
-                                    row=i,
-                                    row_header=cell.row_header,
-                                    row_section=cell.row_section,
-                                    col_header=cell.column_header,
-                                    row_span=[
-                                        cell.start_row_offset_idx,
-                                        cell.end_row_offset_idx,
-                                    ],
-                                    col_span=[
-                                        cell.start_col_offset_idx,
-                                        cell.end_col_offset_idx,
-                                    ],
-                                )
-
-                    # Compute the caption
-                    caption = item.caption_text(self.document)
-
-                    tables.append(
-                        DsSchemaTable(
-                            text=caption,
-                            num_cols=item.data.num_cols,
-                            num_rows=item.data.num_rows,
-                            obj_type=layout_label_to_ds_type.get(item.label),
-                            data=table_data,
-                            prov=[
-                                Prov(
-                                    bbox=p.bbox.as_tuple(),
-                                    page=p.page_no,
-                                    span=[0, 0],
-                                )
-                                for p in item.prov
-                            ],
-                        )
-                    )
-
-                elif isinstance(item, PictureItem):
-                    index = len(figures)
-                    ref_str = f"#/figures/{index}"
-                    main_text.append(
-                        Ref(
-                            name=reverse_label_mapping[item.label],
-                            obj_type=layout_label_to_ds_type.get(item.label),
-                            ref=ref_str,
-                        ),
-                    )
-
-                    # Compute the caption
-                    caption = item.caption_text(self.document)
-
-                    figures.append(
-                        Figure(
-                            prov=[
-                                Prov(
-                                    bbox=p.bbox.as_tuple(),
-                                    page=p.page_no,
-                                    span=[0, len(caption)],
-                                )
-                                for p in item.prov
-                            ],
-                            obj_type=layout_label_to_ds_type.get(item.label),
-                            text=caption,
-                            # data=[[]],
-                        )
-                    )
-
-        page_dimensions = [
-            PageDimensions(page=p.page_no, height=p.size.height, width=p.size.width)
-            for p in self.document.pages.values()
-        ]
-
-        ds_doc = DsDocument(
-            name=title,
-            description=desc,
-            file_info=file_info,
-            main_text=main_text,
-            equations=equations,
-            footnotes=footnotes,
-            page_headers=page_headers,
-            page_footers=page_footers,
-            tables=tables,
-            figures=figures,
-            page_dimensions=page_dimensions,
-        )
-
-        return ds_doc
+        return docling_document_to_legacy(self.document)
 
 
 class _DummyBackend(AbstractDocumentBackend):

docling/datamodel/pipeline_options.py

@@ -126,12 +126,33 @@ class OcrMacOptions(OcrOptions):
     )
 
 
+# Define an enum for the backend options
+class PdfBackend(str, Enum):
+    """Enum of valid PDF backends."""
+
+    PYPDFIUM2 = "pypdfium2"
+    DLPARSE_V1 = "dlparse_v1"
+    DLPARSE_V2 = "dlparse_v2"
+
+
+# Define an enum for the ocr engines
+class OcrEngine(str, Enum):
+    """Enum of valid OCR engines."""
+
+    EASYOCR = "easyocr"
+    TESSERACT_CLI = "tesseract_cli"
+    TESSERACT = "tesseract"
+    OCRMAC = "ocrmac"
+    RAPIDOCR = "rapidocr"
+
+
 class PipelineOptions(BaseModel):
     """Base pipeline options."""
 
     create_legacy_output: bool = (
-        True  # This defautl will be set to False on a future version of docling
+        True  # This default will be set to False on a future version of docling
     )
+    document_timeout: Optional[float] = None
 
 
 class PdfPipelineOptions(PipelineOptions):
@@ -143,7 +164,11 @@ class PdfPipelineOptions(PipelineOptions):
 
     table_structure_options: TableStructureOptions = TableStructureOptions()
     ocr_options: Union[
-        EasyOcrOptions, TesseractCliOcrOptions, TesseractOcrOptions, OcrMacOptions
+        EasyOcrOptions,
+        TesseractCliOcrOptions,
+        TesseractOcrOptions,
+        OcrMacOptions,
+        RapidOcrOptions,
     ] = Field(EasyOcrOptions(), discriminator="kind")
 
     images_scale: float = 1.0

docling/document_converter.py

@@ -9,7 +9,7 @@ from pydantic import BaseModel, ConfigDict, model_validator, validate_call
 
 from docling.backend.abstract_backend import AbstractDocumentBackend
 from docling.backend.asciidoc_backend import AsciiDocBackend
-from docling.backend.docling_parse_backend import DoclingParseDocumentBackend
+from docling.backend.docling_parse_v2_backend import DoclingParseV2DocumentBackend
 from docling.backend.html_backend import HTMLDocumentBackend
 from docling.backend.md_backend import MarkdownDocumentBackend
 from docling.backend.msexcel_backend import MsExcelDocumentBackend
@@ -84,12 +84,12 @@ class HTMLFormatOption(FormatOption):
 
 class PdfFormatOption(FormatOption):
     pipeline_cls: Type = StandardPdfPipeline
-    backend: Type[AbstractDocumentBackend] = DoclingParseDocumentBackend
+    backend: Type[AbstractDocumentBackend] = DoclingParseV2DocumentBackend
 
 
 class ImageFormatOption(FormatOption):
     pipeline_cls: Type = StandardPdfPipeline
-    backend: Type[AbstractDocumentBackend] = DoclingParseDocumentBackend
+    backend: Type[AbstractDocumentBackend] = DoclingParseV2DocumentBackend
 
 
 def _get_default_option(format: InputFormat) -> FormatOption:
@@ -113,10 +113,10 @@ def _get_default_option(format: InputFormat) -> FormatOption:
             pipeline_cls=SimplePipeline, backend=HTMLDocumentBackend
         ),
         InputFormat.IMAGE: FormatOption(
-            pipeline_cls=StandardPdfPipeline, backend=DoclingParseDocumentBackend
+            pipeline_cls=StandardPdfPipeline, backend=DoclingParseV2DocumentBackend
         ),
         InputFormat.PDF: FormatOption(
-            pipeline_cls=StandardPdfPipeline, backend=DoclingParseDocumentBackend
+            pipeline_cls=StandardPdfPipeline, backend=DoclingParseV2DocumentBackend
         ),
     }
     if (options := format_to_default_options.get(format)) is not None:

docling/models/ds_glm_model.py

@@ -3,9 +3,7 @@ import random
 from pathlib import Path
 from typing import List, Union
 
-from deepsearch_glm.nlp_utils import init_nlp_model
-from deepsearch_glm.utils.doc_utils import to_docling_document
-from deepsearch_glm.utils.load_pretrained_models import load_pretrained_nlp_models
+from deepsearch_glm.andromeda_nlp import nlp_model
 from docling_core.types.doc import BoundingBox, CoordOrigin, DoclingDocument
 from docling_core.types.legacy_doc.base import BoundingBox as DsBoundingBox
 from docling_core.types.legacy_doc.base import (
@@ -29,6 +27,7 @@ from pydantic import BaseModel, ConfigDict
 from docling.datamodel.base_models import Cluster, FigureElement, Table, TextElement
 from docling.datamodel.document import ConversionResult, layout_label_to_ds_type
 from docling.datamodel.settings import settings
+from docling.utils.glm_utils import to_docling_document
 from docling.utils.profiling import ProfilingScope, TimeRecorder
 from docling.utils.utils import create_hash
 
@@ -43,9 +42,7 @@ class GlmModel:
     def __init__(self, options: GlmOptions):
         self.options = options
 
-        if self.options.model_names != "":
-            load_pretrained_nlp_models()
-        self.model = init_nlp_model(model_names=self.options.model_names)
+        self.model = nlp_model(loglevel="error", text_ordering=True)
 
     def _to_legacy_document(self, conv_res) -> DsDocument:
         title = ""
@@ -232,7 +229,7 @@ class GlmModel:
     def __call__(self, conv_res: ConversionResult) -> DoclingDocument:
         with TimeRecorder(conv_res, "glm", scope=ProfilingScope.DOCUMENT):
             ds_doc = self._to_legacy_document(conv_res)
-            ds_doc_dict = ds_doc.model_dump(by_alias=True)
+            ds_doc_dict = ds_doc.model_dump(by_alias=True, exclude_none=True)
 
             glm_doc = self.model.apply_on_doc(ds_doc_dict)
 

docling/models/rapid_ocr_model.py

@@ -118,24 +118,25 @@ class RapidOcrModel(BaseOcrModel):
                         del high_res_image
                         del im
 
-                        cells = [
-                            OcrCell(
-                                id=ix,
-                                text=line[1],
-                                confidence=line[2],
-                                bbox=BoundingBox.from_tuple(
-                                    coord=(
-                                        (line[0][0][0] / self.scale) + ocr_rect.l,
-                                        (line[0][0][1] / self.scale) + ocr_rect.t,
-                                        (line[0][2][0] / self.scale) + ocr_rect.l,
-                                        (line[0][2][1] / self.scale) + ocr_rect.t,
+                        if result is not None:
+                            cells = [
+                                OcrCell(
+                                    id=ix,
+                                    text=line[1],
+                                    confidence=line[2],
+                                    bbox=BoundingBox.from_tuple(
+                                        coord=(
+                                            (line[0][0][0] / self.scale) + ocr_rect.l,
+                                            (line[0][0][1] / self.scale) + ocr_rect.t,
+                                            (line[0][2][0] / self.scale) + ocr_rect.l,
+                                            (line[0][2][1] / self.scale) + ocr_rect.t,
+                                        ),
+                                        origin=CoordOrigin.TOPLEFT,
                                     ),
-                                    origin=CoordOrigin.TOPLEFT,
-                                ),
-                            )
-                            for ix, line in enumerate(result)
-                        ]
-                        all_ocr_cells.extend(cells)
+                                )
+                                for ix, line in enumerate(result)
+                            ]
+                            all_ocr_cells.extend(cells)
 
                     # Post-process the cells
                     page.cells = self.post_process_cells(all_ocr_cells, page.cells)

docling/pipeline/base_pipeline.py

@@ -126,6 +126,7 @@ class PaginatedPipeline(BasePipeline):  # TODO this is a bad name.
             # conv_res.status = ConversionStatus.FAILURE
             # return conv_res
 
+        total_elapsed_time = 0.0
         with TimeRecorder(conv_res, "doc_build", scope=ProfilingScope.DOCUMENT):
 
             for i in range(0, conv_res.input.page_count):
@@ -136,7 +137,7 @@ class PaginatedPipeline(BasePipeline):  # TODO this is a bad name.
                 for page_batch in chunkify(
                     conv_res.pages, settings.perf.page_batch_size
                 ):
-                    start_pb_time = time.time()
+                    start_batch_time = time.monotonic()
 
                     # 1. Initialise the page resources
                     init_pages = map(
@@ -149,8 +150,21 @@ class PaginatedPipeline(BasePipeline):  # TODO this is a bad name.
                     for p in pipeline_pages:  # Must exhaust!
                         pass
 
-                    end_pb_time = time.time() - start_pb_time
-                    _log.debug(f"Finished converting page batch time={end_pb_time:.3f}")
+                    end_batch_time = time.monotonic()
+                    total_elapsed_time += end_batch_time - start_batch_time
+                    if (
+                        self.pipeline_options.document_timeout is not None
+                        and total_elapsed_time > self.pipeline_options.document_timeout
+                    ):
+                        _log.warning(
+                            f"Document processing time ({total_elapsed_time:.3f} seconds) exceeded the specified timeout of {self.pipeline_options.document_timeout:.3f} seconds"
+                        )
+                        conv_res.status = ConversionStatus.PARTIAL_SUCCESS
+                        break
+
+                    _log.debug(
+                        f"Finished converting page batch time={end_batch_time:.3f}"
+                    )
 
             except Exception as e:
                 conv_res.status = ConversionStatus.FAILURE

docling/pipeline/standard_pdf_pipeline.py

@@ -97,7 +97,9 @@ class StandardPdfPipeline(PaginatedPipeline):
         local_dir: Optional[Path] = None, force: bool = False
     ) -> Path:
         from huggingface_hub import snapshot_download
+        from huggingface_hub.utils import disable_progress_bars
 
+        disable_progress_bars()
         download_path = snapshot_download(
             repo_id="ds4sd/docling-models",
             force_download=force,

docling/utils/glm_utils.py

@@ -0,0 +1,336 @@
+import re
+from pathlib import Path
+from typing import List
+
+import pandas as pd
+from docling_core.types.doc import (
+    BoundingBox,
+    CoordOrigin,
+    DocItemLabel,
+    DoclingDocument,
+    DocumentOrigin,
+    GroupLabel,
+    ProvenanceItem,
+    Size,
+    TableCell,
+    TableData,
+)
+
+
+def resolve_item(paths, obj):
+    """Find item in document from a reference path"""
+
+    if len(paths) == 0:
+        return obj
+
+    if paths[0] == "#":
+        return resolve_item(paths[1:], obj)
+
+    try:
+        key = int(paths[0])
+    except:
+        key = paths[0]
+
+    if len(paths) == 1:
+        if isinstance(key, str) and key in obj:
+            return obj[key]
+        elif isinstance(key, int) and key < len(obj):
+            return obj[key]
+        else:
+            return None
+
+    elif len(paths) > 1:
+        if isinstance(key, str) and key in obj:
+            return resolve_item(paths[1:], obj[key])
+        elif isinstance(key, int) and key < len(obj):
+            return resolve_item(paths[1:], obj[key])
+        else:
+            return None
+
+    else:
+        return None
+
+
+def _flatten_table_grid(grid: List[List[dict]]) -> List[dict]:
+    unique_objects = []
+    seen_spans = set()
+
+    for sublist in grid:
+        for obj in sublist:
+            # Convert the spans list to a tuple of tuples for hashing
+            spans_tuple = tuple(tuple(span) for span in obj["spans"])
+            if spans_tuple not in seen_spans:
+                seen_spans.add(spans_tuple)
+                unique_objects.append(obj)
+
+    return unique_objects
+
+
+def to_docling_document(doc_glm, update_name_label=False) -> DoclingDocument:
+    origin = DocumentOrigin(
+        mimetype="application/pdf",
+        filename=doc_glm["file-info"]["filename"],
+        binary_hash=doc_glm["file-info"]["document-hash"],
+    )
+    doc_name = Path(origin.filename).stem
+
+    doc: DoclingDocument = DoclingDocument(name=doc_name, origin=origin)
+
+    for page_dim in doc_glm["page-dimensions"]:
+        page_no = int(page_dim["page"])
+        size = Size(width=page_dim["width"], height=page_dim["height"])
+
+        doc.add_page(page_no=page_no, size=size)
+
+    if "properties" in doc_glm:
+        props = pd.DataFrame(
+            doc_glm["properties"]["data"], columns=doc_glm["properties"]["headers"]
+        )
+    else:
+        props = pd.DataFrame()
+
+    current_list = None
+
+    for ix, pelem in enumerate(doc_glm["page-elements"]):
+        ptype = pelem["type"]
+        span_i = pelem["span"][0]
+        span_j = pelem["span"][1]
+
+        if "iref" not in pelem:
+            # print(json.dumps(pelem, indent=2))
+            continue
+
+        iref = pelem["iref"]
+
+        if re.match("#/figures/(\\d+)/captions/(.+)", iref):
+            # print(f"skip {iref}")
+            continue
+
+        if re.match("#/tables/(\\d+)/captions/(.+)", iref):
+            # print(f"skip {iref}")
+            continue
+
+        path = iref.split("/")
+        obj = resolve_item(path, doc_glm)
+
+        if obj is None:
+            current_list = None
+            print(f"warning: undefined {path}")
+            continue
+
+        if ptype == "figure":
+            current_list = None
+            text = ""
+            caption_refs = []
+            for caption in obj["captions"]:
+                text += caption["text"]
+
+                for nprov in caption["prov"]:
+                    npaths = nprov["$ref"].split("/")
+                    nelem = resolve_item(npaths, doc_glm)
+
+                    if nelem is None:
+                        # print(f"warning: undefined caption {npaths}")
+                        continue
+
+                    span_i = nelem["span"][0]
+                    span_j = nelem["span"][1]
+
+                    cap_text = caption["text"][span_i:span_j]
+
+                    # doc_glm["page-elements"].remove(nelem)
+
+                    prov = ProvenanceItem(
+                        page_no=nelem["page"],
+                        charspan=tuple(nelem["span"]),
+                        bbox=BoundingBox.from_tuple(
+                            nelem["bbox"], origin=CoordOrigin.BOTTOMLEFT
+                        ),
+                    )
+
+                    caption_obj = doc.add_text(
+                        label=DocItemLabel.CAPTION, text=cap_text, prov=prov
+                    )
+                    caption_refs.append(caption_obj.get_ref())
+
+            prov = ProvenanceItem(
+                page_no=pelem["page"],
+                charspan=(0, len(text)),
+                bbox=BoundingBox.from_tuple(
+                    pelem["bbox"], origin=CoordOrigin.BOTTOMLEFT
+                ),
+            )
+
+            pic = doc.add_picture(prov=prov)
+            pic.captions.extend(caption_refs)
+            _add_child_elements(pic, doc, obj, pelem)
+
+        elif ptype == "table":
+            current_list = None
+            text = ""
+            caption_refs = []
+            for caption in obj["captions"]:
+                text += caption["text"]
+
+                for nprov in caption["prov"]:
+                    npaths = nprov["$ref"].split("/")
+                    nelem = resolve_item(npaths, doc_glm)
+
+                    if nelem is None:
+                        # print(f"warning: undefined caption {npaths}")
+                        continue
+
+                    span_i = nelem["span"][0]
+                    span_j = nelem["span"][1]
+
+                    cap_text = caption["text"][span_i:span_j]
+
+                    # doc_glm["page-elements"].remove(nelem)
+
+                    prov = ProvenanceItem(
+                        page_no=nelem["page"],
+                        charspan=tuple(nelem["span"]),
+                        bbox=BoundingBox.from_tuple(
+                            nelem["bbox"], origin=CoordOrigin.BOTTOMLEFT
+                        ),
+                    )
+
+                    caption_obj = doc.add_text(
+                        label=DocItemLabel.CAPTION, text=cap_text, prov=prov
+                    )
+                    caption_refs.append(caption_obj.get_ref())
+
+            table_cells_glm = _flatten_table_grid(obj["data"])
+
+            table_cells = []
+            for tbl_cell_glm in table_cells_glm:
+                if tbl_cell_glm["bbox"] is not None:
+                    bbox = BoundingBox.from_tuple(
+                        tbl_cell_glm["bbox"], origin=CoordOrigin.BOTTOMLEFT
+                    )
+                else:
+                    bbox = None
+
+                is_col_header = False
+                is_row_header = False
+                is_row_section = False
+
+                if tbl_cell_glm["type"] == "col_header":
+                    is_col_header = True
+                elif tbl_cell_glm["type"] == "row_header":
+                    is_row_header = True
+                elif tbl_cell_glm["type"] == "row_section":
+                    is_row_section = True
+
+                table_cells.append(
+                    TableCell(
+                        row_span=tbl_cell_glm["row-span"][1]
+                        - tbl_cell_glm["row-span"][0],
+                        col_span=tbl_cell_glm["col-span"][1]
+                        - tbl_cell_glm["col-span"][0],
+                        start_row_offset_idx=tbl_cell_glm["row-span"][0],
+                        end_row_offset_idx=tbl_cell_glm["row-span"][1],
+                        start_col_offset_idx=tbl_cell_glm["col-span"][0],
+                        end_col_offset_idx=tbl_cell_glm["col-span"][1],
+                        text=tbl_cell_glm["text"],
+                        bbox=bbox,
+                        column_header=is_col_header,
+                        row_header=is_row_header,
+                        row_section=is_row_section,
+                    )
+                )
+
+            tbl_data = TableData(
+                num_rows=obj.get("#-rows", 0),
+                num_cols=obj.get("#-cols", 0),
+                table_cells=table_cells,
+            )
+
+            prov = ProvenanceItem(
+                page_no=pelem["page"],
+                charspan=(0, 0),
+                bbox=BoundingBox.from_tuple(
+                    pelem["bbox"], origin=CoordOrigin.BOTTOMLEFT
+                ),
+            )
+
+            tbl = doc.add_table(data=tbl_data, prov=prov)
+            tbl.captions.extend(caption_refs)
+
+        elif ptype in ["form", "key_value_region"]:
+            label = DocItemLabel(ptype)
+            container_el = doc.add_group(label=GroupLabel.UNSPECIFIED, name=label)
+
+            _add_child_elements(container_el, doc, obj, pelem)
+
+        elif "text" in obj:
+            text = obj["text"][span_i:span_j]
+
+            type_label = pelem["type"]
+            name_label = pelem["name"]
+            if update_name_label and len(props) > 0 and type_label == "paragraph":
+                prop = props[
+                    (props["type"] == "semantic") & (props["subj_path"] == iref)
+                ]
+                if len(prop) == 1 and prop.iloc[0]["confidence"] > 0.85:
+                    name_label = prop.iloc[0]["label"]
+
+            prov = ProvenanceItem(
+                page_no=pelem["page"],
+                charspan=(0, len(text)),
+                bbox=BoundingBox.from_tuple(
+                    pelem["bbox"], origin=CoordOrigin.BOTTOMLEFT
+                ),
+            )
+            label = DocItemLabel(name_label)
+
+            if label == DocItemLabel.LIST_ITEM:
+                if current_list is None:
+                    current_list = doc.add_group(label=GroupLabel.LIST, name="list")
+
+                # TODO: Infer if this is a numbered or a bullet list item
+                doc.add_list_item(
+                    text=text, enumerated=False, prov=prov, parent=current_list
+                )
+            elif label == DocItemLabel.SECTION_HEADER:
+                current_list = None
+
+                doc.add_heading(text=text, prov=prov)
+            else:
+                current_list = None
+
+                doc.add_text(label=DocItemLabel(name_label), text=text, prov=prov)
+
+    return doc
+
+
+def _add_child_elements(container_el, doc, obj, pelem):
+    payload = obj.get("payload")
+    if payload is not None:
+        children = payload.get("children", [])
+
+        for child in children:
+            c_label = DocItemLabel(child["label"])
+            c_bbox = BoundingBox.model_validate(child["bbox"]).to_bottom_left_origin(
+                doc.pages[pelem["page"]].size.height
+            )
+            c_text = " ".join(
+                [
+                    cell["text"].replace("\x02", "-").strip()
+                    for cell in child["cells"]
+                    if len(cell["text"].strip()) > 0
+                ]
+            )
+
+            c_prov = ProvenanceItem(
+                page_no=pelem["page"], charspan=(0, len(c_text)), bbox=c_bbox
+            )
+            if c_label == DocItemLabel.LIST_ITEM:
+                # TODO: Infer if this is a numbered or a bullet list item
+                doc.add_list_item(parent=container_el, text=c_text, prov=c_prov)
+            elif c_label == DocItemLabel.SECTION_HEADER:
+                doc.add_heading(parent=container_el, text=c_text, prov=c_prov)
+            else:
+                doc.add_text(
+                    parent=container_el, label=c_label, text=c_text, prov=c_prov
+                )

{docling-2.9.0.dist-info → docling-2.11.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: docling
-Version: 2.9.0
+Version: 2.11.0
 Summary: SDK and CLI for parsing PDF, DOCX, HTML, and more, to a unified document representation for powering downstream workflows such as gen AI applications.
 Home-page: https://github.com/DS4SD/docling
 License: MIT
@@ -25,10 +25,10 @@ Provides-Extra: rapidocr
 Provides-Extra: tesserocr
 Requires-Dist: beautifulsoup4 (>=4.12.3,<5.0.0)
 Requires-Dist: certifi (>=2024.7.4)
-Requires-Dist: deepsearch-glm (>=0.26.1,<0.27.0)
-Requires-Dist: docling-core[chunking] (>=2.8.0,<3.0.0)
+Requires-Dist: deepsearch-glm (>=1.0.0,<2.0.0)
+Requires-Dist: docling-core[chunking] (>=2.9.0,<3.0.0)
 Requires-Dist: docling-ibm-models (>=2.0.6,<3.0.0)
-Requires-Dist: docling-parse (>=2.0.5,<3.0.0)
+Requires-Dist: docling-parse (>=3.0.0,<4.0.0)
 Requires-Dist: easyocr (>=1.7,<2.0)
 Requires-Dist: filetype (>=1.2.0,<2.0.0)
 Requires-Dist: huggingface_hub (>=0.23,<1)

{docling-2.9.0.dist-info → docling-2.11.0.dist-info}/RECORD

@@ -2,8 +2,8 @@ docling/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 docling/backend/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 docling/backend/abstract_backend.py,sha256=-or6kWVV7egQeyIuN-vI0Tr7Q1htalBZSlhgq_G2RdU,1678
 docling/backend/asciidoc_backend.py,sha256=kXZxOLk_LvLFVZwnJVVwjmvc3QWZ0iiG7VnwjgtC3hI,14051
-docling/backend/docling_parse_backend.py,sha256=csWy6ZGxDuZfNr0YTrUU40DXqelN_TJksWIYoXxZMjU,7633
-docling/backend/docling_parse_v2_backend.py,sha256=gUr9_fwHbkj238oYQPJ9AxpjFL2jGvhjBlBQPblmSAg,8589
+docling/backend/docling_parse_backend.py,sha256=_jY5f5-KGI3hi5pcZAY6e7tPLocSi5JUWrxraDVszqI,7631
+docling/backend/docling_parse_v2_backend.py,sha256=1TDUdMIp3fEjCWBNjusUHiCUmH1g6yZQ-b13scofP0Y,8637
 docling/backend/html_backend.py,sha256=qbu1W8xoTGnXMuZPRPLq68hDbCEj6ygnpxP5gYaodAQ,15593
 docling/backend/md_backend.py,sha256=tmuSCghjor9PqKIiVieCuZ4_t5JEjZMy3cq7u3yTgyU,14032
 docling/backend/msexcel_backend.py,sha256=23qUEScqr5GhY06xiqg-eBQ_JlAqO0FkPEmX6554sVA,12040
@@ -13,39 +13,40 @@ docling/backend/pdf_backend.py,sha256=unnw7QiRE1VXg6Pj-eYrtnFGrp5SSYiI324OlFxyv6
 docling/backend/pypdfium2_backend.py,sha256=B4bfv-dfzlWiKTfF8LN5fto_99YBu8A2c1_XIVwRUWI,8996
 docling/chunking/__init__.py,sha256=h83TDs0AuOV6oEPLAPrn9dpGKiU-2Vg6IRNo4cv6GDA,346
 docling/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-docling/cli/main.py,sha256=dWGiweoxaD4ptORsx1lzxc1zcDg2fgd4b-GG1cSdVTA,14628
+docling/cli/main.py,sha256=FFDUDADvK7QNW7xCs6dlsC7Bt_BMyrKdbZewKTEjm54,14624
 docling/datamodel/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-docling/datamodel/base_models.py,sha256=627IB8HZdXGmHNfsX4Qhf7kKSxx2btPjS7z8hitvhyE,5560
-docling/datamodel/document.py,sha256=Y0NEFphwz44VxIaRaDRhtmw6rifzSC7MqyaDBzaR0lM,20902
-docling/datamodel/pipeline_options.py,sha256=K65nEZ52aRfF8hWIzl0zVvRQj-3XVwoBbxTacGS6jEg,4960
+docling/datamodel/base_models.py,sha256=vwy59eDrkzCSaay24RlUvx4zEyuaUukOdOhw3622u2I,5616
+docling/datamodel/document.py,sha256=GNlTsgKgDqdqv2dfhpYmnqymxDQWWWC8HgE8uAta8V4,10265
+docling/datamodel/pipeline_options.py,sha256=1ouWNE5VhZolrWMb4RE6s_AxgNFr3_3PMtxB_YQ391A,5495
 docling/datamodel/settings.py,sha256=JK8lZPBjUx2kD2q-Qpg-o3vOElADMcyQbRUL0EHZ7us,1263
-docling/document_converter.py,sha256=bsXGQCUrbL2LmaqaaEmlkfSANl2XwBBx8HDLwFrqhFY,11570
+docling/document_converter.py,sha256=Iz5eerBWFPVJoXAMlXEivRQX2VLBiUkA07BL4NNbaEs,11583
 docling/exceptions.py,sha256=-FoP46rFJgz_jn5uDv2V052udEEg8gckk6uhoItchXc,85
 docling/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 docling/models/base_model.py,sha256=Yq_-FmUhqhE20vXYG3WiQXDRTIPjik1CyuEZ8iYTGAY,701
 docling/models/base_ocr_model.py,sha256=rGSpBF4dByITcsBaRIgvFKpiu0CrhmZS_PHIo686Dw0,6428
-docling/models/ds_glm_model.py,sha256=hBRCx6oFGhxBbKEJlRSWVndDwFtB5IpeLOowFAVqFM0,12033
+docling/models/ds_glm_model.py,sha256=YJkGxV46wh7G2Wr4vVzt9b8oewkUDPWpvI6AEaZDrs0,11872
 docling/models/easyocr_model.py,sha256=c2m4x9dZpSc-cMgeEdFBRVBlB78uMGlYD8Q_2gzRuMU,3734
 docling/models/layout_model.py,sha256=ZvbTSyxvXB5yLHNEti0Wv3trz0vwGuHySI5TCdApb0U,14011
 docling/models/ocr_mac_model.py,sha256=bLP14UUmZcSzjDe-HLj-mtksTuBmsCTg2C1wCxUpan0,4502
 docling/models/page_assemble_model.py,sha256=kSGNiRKhmzkpFH7xCiT3rulMsgJmUXFa6Th_eB-cLEk,7103
 docling/models/page_preprocessing_model.py,sha256=1gVrZjObKxAvXkkKvXnIFApPOggzgiTFPtt1CGbMbSs,2763
-docling/models/rapid_ocr_model.py,sha256=VQ0jaFmOzB9f-1JaqZ6d0o_El55Lr-nsFHfTNubMAuc,6005
+docling/models/rapid_ocr_model.py,sha256=ui152cerv9b9OeWyyyefs8qMLwYn0qsE2DFE_gHmaCM,6124
 docling/models/table_structure_model.py,sha256=-ANSQpiN2avt3B9sbi7dHcoULUJbMBalAR5xxlrM7To,8421
 docling/models/tesseract_ocr_cli_model.py,sha256=aKQBaty4cYu6zG_C5uy6Zm3eeRQo5fxIierbKixa2kc,6622
 docling/models/tesseract_ocr_model.py,sha256=RDf6iV1q-oXaGfZXv0bW6SqjHNKQvBUDlUsOkuz0neY,6095
 docling/pipeline/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-docling/pipeline/base_pipeline.py,sha256=IF1XWYgUGbdB4-teLkmM4Hvg_UNEfPrGuhExMRTUsk8,7168
+docling/pipeline/base_pipeline.py,sha256=hVvtk5E4DVZdl_SyNs_pYRUjN9C8PABhpVaeN5Z_fAY,7885
 docling/pipeline/simple_pipeline.py,sha256=mZqANqUtAOFAyqQEShErQnAUz6tJFOl6zVoazEDJ_wE,2254
-docling/pipeline/standard_pdf_pipeline.py,sha256=7sbkh9EwXlhSfJSgf-WyjB5jdJ1El7Pn4siSssTJpq8,8789
+docling/pipeline/standard_pdf_pipeline.py,sha256=B1q8xt3Dfecpi8s8DrcfPzdATh8TYgL43FDzzcS4vEA,8885
 docling/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 docling/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 docling/utils/export.py,sha256=KyGF1BVDHPFfHVXZc8vegsWlFfOgGPP2YckWpTadyI8,4694
+docling/utils/glm_utils.py,sha256=H1O_tDiRksMgw45rY9LhK6GjcZSOq5IyoGurGjoo-Ac,11211
 docling/utils/layout_utils.py,sha256=vlN0rc8i0ayRGn3WnaG-pdmqEL00KKGl2zez3Gj-hrk,32074
 docling/utils/profiling.py,sha256=YaMGoB9MMZpagF9mb5ndoHj8Lpb9aIdb7El-Pl7IcFs,1753
 docling/utils/utils.py,sha256=llhXSbIDNZ1MHOwBEfLHBAoJIAYI7QlPIonlI1jLUJ0,1208
-docling-2.9.0.dist-info/LICENSE,sha256=mBb7ErEcM8VS9OhiGHnQ2kk75HwPhr54W1Oiz3965MY,1088
-docling-2.9.0.dist-info/METADATA,sha256=AFpGTTO_IIyzV5AglCh8hhj3Z7UopU7mPkJ3n_UzrUs,7732
-docling-2.9.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-docling-2.9.0.dist-info/entry_points.txt,sha256=VOSzV77znM52dz5ysaDuJ0ijl1cnfrh1ZPg8od5OcTs,48
-docling-2.9.0.dist-info/RECORD,,
+docling-2.11.0.dist-info/LICENSE,sha256=mBb7ErEcM8VS9OhiGHnQ2kk75HwPhr54W1Oiz3965MY,1088
+docling-2.11.0.dist-info/METADATA,sha256=ajUVy5CuNDUp0x9tMCqO2px2M-ia-Vs7frIyb0_HxMo,7731
+docling-2.11.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+docling-2.11.0.dist-info/entry_points.txt,sha256=VOSzV77znM52dz5ysaDuJ0ijl1cnfrh1ZPg8od5OcTs,48
+docling-2.11.0.dist-info/RECORD,,