docling-core 2.8.0__tar.gz → 2.9.0__tar.gz

{docling_core-2.8.0 → docling_core-2.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: docling-core
-Version: 2.8.0
+Version: 2.9.0
 Summary: A python library to define and validate data types in Docling.
 Home-page: https://ds4sd.github.io/
 License: MIT

{docling_core-2.8.0 → docling_core-2.9.0}/docling_core/types/doc/document.py

@@ -380,6 +380,7 @@ class DocumentOrigin(BaseModel):
         "application/vnd.openxmlformats-officedocument.presentationml.template",
         "application/vnd.openxmlformats-officedocument.presentationml.slideshow",
         "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
         "text/asciidoc",
         "text/markdown",
     ]
@@ -445,7 +446,7 @@ class ImageRef(BaseModel):
     mimetype: str
     dpi: int
     size: Size
-    uri: Union[AnyUrl, Path]
+    uri: Union[AnyUrl, Path] = Field(union_mode="left_to_right")
     _pil: Optional[PILImage.Image] = None
 
     @property
@@ -1668,7 +1669,7 @@ class DoclingDocument(BaseModel):
         self,
         root: Optional[NodeItem] = None,
         with_groups: bool = False,
-        traverse_pictures: bool = True,
+        traverse_pictures: bool = False,
         page_no: Optional[int] = None,
         _level: int = 0,  # fixed parameter, carries through the node nesting level
     ) -> typing.Iterable[Tuple[NodeItem, int]]:  # tuple of node and level
@@ -1685,30 +1686,31 @@ class DoclingDocument(BaseModel):
         if not root:
             root = self.body
 
+        # Yield non-group items or group items when with_groups=True
         if not isinstance(root, GroupItem) or with_groups:
             if isinstance(root, DocItem):
-                if page_no is not None:
-                    for prov in root.prov:
-                        if prov.page_no == page_no:
-                            yield root, _level
-                else:
+                if page_no is None or any(
+                    prov.page_no == page_no for prov in root.prov
+                ):
                     yield root, _level
             else:
                 yield root, _level
 
+        # Handle picture traversal - only traverse children if requested
+        if isinstance(root, PictureItem) and not traverse_pictures:
+            return
+
         # Traverse children
         for child_ref in root.children:
             child = child_ref.resolve(self)
-
             if isinstance(child, NodeItem):
-                # If the child is a NodeItem, recursively traverse it
-                if not isinstance(child, PictureItem) or traverse_pictures:
-                    yield from self.iterate_items(
-                        child,
-                        _level=_level + 1,
-                        with_groups=with_groups,
-                        page_no=page_no,
-                    )
+                yield from self.iterate_items(
+                    child,
+                    with_groups=with_groups,
+                    traverse_pictures=traverse_pictures,
+                    page_no=page_no,
+                    _level=_level + 1,
+                )
 
     def _clear_picture_pil_cache(self):
         """Clear cache storage of all images."""
@@ -1864,7 +1866,7 @@ class DoclingDocument(BaseModel):
 
         """
         with open(filename, "r") as f:
-            return cls.model_validate(json.loads(f.read()))
+            return cls.model_validate_json(f.read())
 
     def save_as_yaml(
         self,
@@ -2115,10 +2117,30 @@ class DoclingDocument(BaseModel):
         # Bold, Italic, or Bold-Italic
         # Hence, any underscore that we print into Markdown is coming from document text
         # That means we need to escape it, to properly reflect content in the markdown
+        # However, we need to preserve underscores in image URLs
+        # to maintain their validity
+        # For example: ![image](path/to_image.png) should remain unchanged
         def escape_underscores(text):
-            # Replace "_" with "\_" only if it's not already escaped
-            escaped_text = re.sub(r"(?<!\\)_", r"\_", text)
-            return escaped_text
+            """Escape underscores but leave them intact in the URL.."""
+            # Firstly, identify all the URL patterns.
+            url_pattern = r"!\[.*?\]\((.*?)\)"
+            parts = []
+            last_end = 0
+
+            for match in re.finditer(url_pattern, text):
+                # Text to add before the URL (needs to be escaped)
+                before_url = text[last_end : match.start()]
+                parts.append(re.sub(r"(?<!\\)_", r"\_", before_url))
+
+                # Add the full URL part (do not escape)
+                parts.append(match.group(0))
+                last_end = match.end()
+
+            # Add the final part of the text (which needs to be escaped)
+            if last_end < len(text):
+                parts.append(re.sub(r"(?<!\\)_", r"\_", text[last_end:]))
+
+            return "".join(parts)
 
         mdtext = escape_underscores(mdtext)
 

{docling_core-2.8.0 → docling_core-2.9.0}/docling_core/types/legacy_doc/base.py

@@ -140,6 +140,7 @@ class BaseCell(AliasModel):
     obj_type: str = Field(
         alias="type", json_schema_extra=es_field(type="keyword", ignore_above=8191)
     )
+    payload: Optional[dict] = None
 
     def get_location_tokens(
         self,

docling_core-2.9.0/docling_core/utils/legacy.py

@@ -0,0 +1,346 @@
+#
+# Copyright IBM Corp. 2024 - 2024
+# SPDX-License-Identifier: MIT
+#
+
+"""Utilities for converting between legacy and new document format."""
+
+import hashlib
+import uuid
+from typing import Union
+
+from docling_core.types.doc import (
+    DocItem,
+    DocItemLabel,
+    DoclingDocument,
+    PictureItem,
+    SectionHeaderItem,
+    TableCell,
+    TableItem,
+    TextItem,
+)
+from docling_core.types.doc.document import ListItem
+from docling_core.types.legacy_doc.base import (
+    BaseCell,
+    BaseText,
+    Figure,
+    GlmTableCell,
+    PageDimensions,
+    PageReference,
+    Prov,
+    Ref,
+)
+from docling_core.types.legacy_doc.base import Table as DsSchemaTable
+from docling_core.types.legacy_doc.base import TableCell as DsTableCell
+from docling_core.types.legacy_doc.document import (
+    CCSDocumentDescription as DsDocumentDescription,
+)
+from docling_core.types.legacy_doc.document import CCSFileInfoObject as DsFileInfoObject
+from docling_core.types.legacy_doc.document import ExportedCCSDocument as DsDocument
+
+
+def _create_hash(string: str):
+    hasher = hashlib.sha256()
+    hasher.update(string.encode("utf-8"))
+
+    return hasher.hexdigest()
+
+
+def doc_item_label_to_legacy_type(label: DocItemLabel):
+    """Convert the DocItemLabel to the legacy type."""
+    _label_to_ds_type = {
+        DocItemLabel.TITLE: "title",
+        DocItemLabel.DOCUMENT_INDEX: "table-of-contents",
+        DocItemLabel.SECTION_HEADER: "subtitle-level-1",
+        DocItemLabel.CHECKBOX_SELECTED: "checkbox-selected",
+        DocItemLabel.CHECKBOX_UNSELECTED: "checkbox-unselected",
+        DocItemLabel.CAPTION: "caption",
+        DocItemLabel.PAGE_HEADER: "page-header",
+        DocItemLabel.PAGE_FOOTER: "page-footer",
+        DocItemLabel.FOOTNOTE: "footnote",
+        DocItemLabel.TABLE: "table",
+        DocItemLabel.FORMULA: "equation",
+        DocItemLabel.LIST_ITEM: "paragraph",
+        DocItemLabel.CODE: "paragraph",
+        DocItemLabel.PICTURE: "figure",
+        DocItemLabel.TEXT: "paragraph",
+        DocItemLabel.PARAGRAPH: "paragraph",
+    }
+    if label in _label_to_ds_type:
+        return _label_to_ds_type[label]
+    return label.value
+
+
+def doc_item_label_to_legacy_name(label: DocItemLabel):
+    """Convert the DocItemLabel to the legacy name."""
+    _reverse_label_name_mapping = {
+        DocItemLabel.CAPTION: "Caption",
+        DocItemLabel.FOOTNOTE: "Footnote",
+        DocItemLabel.FORMULA: "Formula",
+        DocItemLabel.LIST_ITEM: "List-item",
+        DocItemLabel.PAGE_FOOTER: "Page-footer",
+        DocItemLabel.PAGE_HEADER: "Page-header",
+        DocItemLabel.PICTURE: "Picture",
+        DocItemLabel.SECTION_HEADER: "Section-header",
+        DocItemLabel.TABLE: "Table",
+        DocItemLabel.TEXT: "Text",
+        DocItemLabel.TITLE: "Title",
+        DocItemLabel.DOCUMENT_INDEX: "Document Index",
+        DocItemLabel.CODE: "Code",
+        DocItemLabel.CHECKBOX_SELECTED: "Checkbox-Selected",
+        DocItemLabel.CHECKBOX_UNSELECTED: "Checkbox-Unselected",
+        DocItemLabel.FORM: "Form",
+        DocItemLabel.KEY_VALUE_REGION: "Key-Value Region",
+        DocItemLabel.PARAGRAPH: "paragraph",
+    }
+    if label in _reverse_label_name_mapping:
+        return _reverse_label_name_mapping[label]
+    return label.value
+
+
+def docling_document_to_legacy(doc: DoclingDocument, fallback_filaname: str = "file"):
+    """Convert a DoclingDocument to the legacy format."""
+    title = ""
+    desc: DsDocumentDescription = DsDocumentDescription(logs=[])
+
+    if doc.origin is not None:
+        document_hash = _create_hash(str(doc.origin.binary_hash))
+        filename = doc.origin.filename
+    else:
+        document_hash = _create_hash(str(uuid.uuid4()))
+        filename = fallback_filaname
+
+    page_hashes = [
+        PageReference(
+            hash=_create_hash(document_hash + ":" + str(p.page_no - 1)),
+            page=p.page_no,
+            model="default",
+        )
+        for p in doc.pages.values()
+    ]
+
+    file_info = DsFileInfoObject(
+        filename=filename,
+        document_hash=document_hash,
+        num_pages=len(doc.pages),
+        page_hashes=page_hashes,
+    )
+
+    main_text: list[Union[Ref, BaseText]] = []
+    tables: list[DsSchemaTable] = []
+    figures: list[Figure] = []
+    equations: list[BaseCell] = []
+    footnotes: list[BaseText] = []
+    page_headers: list[BaseText] = []
+    page_footers: list[BaseText] = []
+
+    # TODO: populate page_headers page_footers from doc.furniture
+
+    embedded_captions = set()
+    for ix, (item, level) in enumerate(doc.iterate_items(doc.body)):
+
+        if isinstance(item, (TableItem, PictureItem)) and len(item.captions) > 0:
+            caption = item.caption_text(doc)
+            if caption:
+                embedded_captions.add(caption)
+
+    for item, level in doc.iterate_items():
+        if isinstance(item, DocItem):
+            item_type = item.label
+
+            if isinstance(item, (TextItem, ListItem, SectionHeaderItem)):
+
+                if isinstance(item, ListItem) and item.marker:
+                    text = f"{item.marker} {item.text}"
+                else:
+                    text = item.text
+
+                # Can be empty.
+                prov = [
+                    Prov(
+                        bbox=p.bbox.as_tuple(),
+                        page=p.page_no,
+                        span=[0, len(item.text)],
+                    )
+                    for p in item.prov
+                ]
+                main_text.append(
+                    BaseText(
+                        text=text,
+                        obj_type=doc_item_label_to_legacy_type(item.label),
+                        name=doc_item_label_to_legacy_name(item.label),
+                        prov=prov,
+                    )
+                )
+
+                # skip captions of they are embedded in the actual
+                # floating object
+                if item_type == DocItemLabel.CAPTION and text in embedded_captions:
+                    continue
+
+            elif isinstance(item, TableItem) and item.data:
+                index = len(tables)
+                ref_str = f"#/tables/{index}"
+                main_text.append(
+                    Ref(
+                        name=doc_item_label_to_legacy_name(item.label),
+                        obj_type=doc_item_label_to_legacy_type(item.label),
+                        ref=ref_str,
+                    ),
+                )
+
+                # Initialise empty table data grid (only empty cells)
+                table_data = [
+                    [
+                        DsTableCell(
+                            text="",
+                            # bbox=[0,0,0,0],
+                            spans=[[i, j]],
+                            obj_type="body",
+                        )
+                        for j in range(item.data.num_cols)
+                    ]
+                    for i in range(item.data.num_rows)
+                ]
+
+                # Overwrite cells in table data for which there is actual cell content.
+                for cell in item.data.table_cells:
+                    for i in range(
+                        min(cell.start_row_offset_idx, item.data.num_rows),
+                        min(cell.end_row_offset_idx, item.data.num_rows),
+                    ):
+                        for j in range(
+                            min(cell.start_col_offset_idx, item.data.num_cols),
+                            min(cell.end_col_offset_idx, item.data.num_cols),
+                        ):
+                            celltype = "body"
+                            if cell.column_header:
+                                celltype = "col_header"
+                            elif cell.row_header:
+                                celltype = "row_header"
+                            elif cell.row_section:
+                                celltype = "row_section"
+
+                            def _make_spans(cell: TableCell, table_item: TableItem):
+                                for rspan in range(
+                                    min(
+                                        cell.start_row_offset_idx,
+                                        table_item.data.num_rows,
+                                    ),
+                                    min(
+                                        cell.end_row_offset_idx,
+                                        table_item.data.num_rows,
+                                    ),
+                                ):
+                                    for cspan in range(
+                                        min(
+                                            cell.start_col_offset_idx,
+                                            table_item.data.num_cols,
+                                        ),
+                                        min(
+                                            cell.end_col_offset_idx,
+                                            table_item.data.num_cols,
+                                        ),
+                                    ):
+                                        yield [rspan, cspan]
+
+                            spans = list(_make_spans(cell, item))
+                            table_data[i][j] = GlmTableCell(
+                                text=cell.text,
+                                bbox=(
+                                    cell.bbox.as_tuple()
+                                    if cell.bbox is not None
+                                    else None
+                                ),  # check if this is bottom-left
+                                spans=spans,
+                                obj_type=celltype,
+                                col=j,
+                                row=i,
+                                row_header=cell.row_header,
+                                row_section=cell.row_section,
+                                col_header=cell.column_header,
+                                row_span=[
+                                    cell.start_row_offset_idx,
+                                    cell.end_row_offset_idx,
+                                ],
+                                col_span=[
+                                    cell.start_col_offset_idx,
+                                    cell.end_col_offset_idx,
+                                ],
+                            )
+
+                # Compute the caption
+                caption = item.caption_text(doc)
+
+                tables.append(
+                    DsSchemaTable(
+                        text=caption,
+                        num_cols=item.data.num_cols,
+                        num_rows=item.data.num_rows,
+                        obj_type=doc_item_label_to_legacy_type(item.label),
+                        data=table_data,
+                        prov=[
+                            Prov(
+                                bbox=p.bbox.as_tuple(),
+                                page=p.page_no,
+                                span=[0, 0],
+                            )
+                            for p in item.prov
+                        ],
+                    )
+                )
+
+            elif isinstance(item, PictureItem):
+                index = len(figures)
+                ref_str = f"#/figures/{index}"
+                main_text.append(
+                    Ref(
+                        name=doc_item_label_to_legacy_name(item.label),
+                        obj_type=doc_item_label_to_legacy_type(item.label),
+                        ref=ref_str,
+                    ),
+                )
+
+                # Compute the caption
+                caption = item.caption_text(doc)
+
+                figures.append(
+                    Figure(
+                        prov=[
+                            Prov(
+                                bbox=p.bbox.as_tuple(),
+                                page=p.page_no,
+                                span=[0, len(caption)],
+                            )
+                            for p in item.prov
+                        ],
+                        obj_type=doc_item_label_to_legacy_type(item.label),
+                        text=caption,
+                        # data=[[]],
+                    )
+                )
+
+    page_dimensions = [
+        PageDimensions(page=p.page_no, height=p.size.height, width=p.size.width)
+        for p in doc.pages.values()
+    ]
+
+    legacy_doc: DsDocument = DsDocument(
+        name=title,
+        description=desc,
+        file_info=file_info,
+        main_text=main_text,
+        equations=equations,
+        footnotes=footnotes,
+        page_headers=page_headers,
+        page_footers=page_footers,
+        tables=tables,
+        figures=figures,
+        page_dimensions=page_dimensions,
+    )
+
+    return legacy_doc
+
+
+# def legacy_to_docling_document(legacy_doc: DsDocument) -> DoclingDocument:
+#     """Convert a legacy document to DoclingDocument."""

{docling_core-2.8.0 → docling_core-2.9.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "docling-core"
-version = "2.8.0"
+version = "2.9.0"
 description = "A python library to define and validate data types in Docling."
 license = "MIT"
 authors = [