docling-core 2.17.1__tar.gz → 2.18.0__tar.gz

{docling_core-2.17.1 → docling_core-2.18.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: docling-core
-Version: 2.17.1
+Version: 2.18.0
 Summary: A python library to define and validate data types in Docling.
 Home-page: https://ds4sd.github.io/
 License: MIT
@@ -66,7 +66,7 @@ pip install docling-core
 
 To develop for Docling Core, you need Python 3.9 / 3.10 / 3.11 / 3.12 / 3.13 and Poetry. You can then install from your local clone's root dir:
 ```bash
-poetry install
+poetry install --all-extras
 ```
 
 To run the pytest suite, execute:

{docling_core-2.17.1 → docling_core-2.18.0}/README.md

@@ -23,7 +23,7 @@ pip install docling-core
 
 To develop for Docling Core, you need Python 3.9 / 3.10 / 3.11 / 3.12 / 3.13 and Poetry. You can then install from your local clone's root dir:
 ```bash
-poetry install
+poetry install --all-extras
 ```
 
 To run the pytest suite, execute:

{docling_core-2.17.1 → docling_core-2.18.0}/docling_core/types/doc/document.py

@@ -13,6 +13,7 @@ import sys
 import textwrap
 import typing
 import warnings
+from enum import Enum
 from io import BytesIO
 from pathlib import Path
 from typing import Any, Dict, Final, List, Literal, Optional, Tuple, Union
@@ -44,13 +45,17 @@ from docling_core.types.doc import BoundingBox, Size
 from docling_core.types.doc.base import ImageRefMode
 from docling_core.types.doc.labels import CodeLanguageLabel, DocItemLabel, GroupLabel
 from docling_core.types.doc.tokens import DocumentToken, TableToken
-from docling_core.types.doc.utils import relative_path
+from docling_core.types.doc.utils import (
+    get_html_tag_with_text_direction,
+    get_text_direction,
+    relative_path,
+)
 
 _logger = logging.getLogger(__name__)
 
 Uint64 = typing.Annotated[int, Field(ge=0, le=(2**64 - 1))]
 LevelNumber = typing.Annotated[int, Field(ge=1, le=100)]
-CURRENT_VERSION: Final = "1.0.0"
+CURRENT_VERSION: Final = "1.1.0"
 
 DEFAULT_EXPORT_LABELS = {
     DocItemLabel.TITLE,
@@ -66,6 +71,8 @@ DEFAULT_EXPORT_LABELS = {
     DocItemLabel.LIST_ITEM,
     DocItemLabel.CODE,
     DocItemLabel.REFERENCE,
+    DocItemLabel.PAGE_HEADER,
+    DocItemLabel.PAGE_FOOTER,
 }
 
 
@@ -509,6 +516,16 @@ class ProvenanceItem(BaseModel):
     charspan: Tuple[int, int]
 
 
+class ContentLayer(str, Enum):
+    """ContentLayer."""
+
+    BODY = "body"
+    FURNITURE = "furniture"
+
+
+DEFAULT_CONTENT_LAYERS = {ContentLayer.BODY}
+
+
 class NodeItem(BaseModel):
     """NodeItem."""
 
@@ -516,6 +533,8 @@ class NodeItem(BaseModel):
     parent: Optional[RefItem] = None
     children: List[RefItem] = []
 
+    content_layer: ContentLayer = ContentLayer.BODY
+
     model_config = ConfigDict(extra="forbid")
 
     def get_ref(self):
@@ -866,7 +885,9 @@ class PictureItem(FloatingItem):
 
         caption_text = ""
         if len(text) > 0:
-            caption_text = f"<figcaption>{text}</figcaption>"
+            caption_text = get_html_tag_with_text_direction(
+                html_tag="figcaption", text=text
+            )
 
         default_response = f"<figure>{caption_text}</figure>"
 
@@ -1090,15 +1111,28 @@ class TableItem(FloatingItem):
                 if colspan > 1:
                     opening_tag += f' colspan="{colspan}"'
 
+                text_dir = get_text_direction(content)
+                if text_dir == "rtl":
+                    opening_tag += f' dir="{dir}"'
+
                 body += f"<{opening_tag}>{content}</{celltag}>"
             body += "</tr>"
 
+        # dir = get_text_direction(text)
+
         if len(text) > 0 and len(body) > 0:
-            body = f"<table><caption>{text}</caption><tbody>{body}</tbody></table>"
+            caption_text = get_html_tag_with_text_direction(
+                html_tag="caption", text=text
+            )
+            body = f"<table>{caption_text}<tbody>{body}</tbody></table>"
+
         elif len(text) == 0 and len(body) > 0:
             body = f"<table><tbody>{body}</tbody></table>"
         elif len(text) > 0 and len(body) == 0:
-            body = f"<table><caption>{text}</caption></table>"
+            caption_text = get_html_tag_with_text_direction(
+                html_tag="caption", text=text
+            )
+            body = f"<table>{caption_text}</table>"
         else:
             body = "<table></table>"
 
@@ -1423,8 +1457,8 @@ class DoclingDocument(BaseModel):
         # generated from synthetic data.
     )
 
-    furniture: GroupItem = GroupItem(
-        name="_root_", self_ref="#/furniture"
+    furniture: Annotated[GroupItem, Field(deprecated=True)] = GroupItem(
+        name="_root_", self_ref="#/furniture", content_layer=ContentLayer.FURNITURE
     )  # List[RefItem] = []
     body: GroupItem = GroupItem(name="_root_", self_ref="#/body")  # List[RefItem] = []
 
@@ -1436,11 +1470,28 @@ class DoclingDocument(BaseModel):
 
     pages: Dict[int, PageItem] = {}  # empty as default
 
+    @model_validator(mode="before")
+    @classmethod
+    def transform_to_content_layer(cls, data: dict) -> dict:
+        """transform_to_content_layer."""
+        # Since version 1.1.0, all NodeItems carry content_layer property.
+        # We must assign previous page_header and page_footer instances to furniture.
+        # Note: model_validators which check on the version must use "before".
+        if "version" in data and data["version"] == "1.0.0":
+            for item in data.get("texts", []):
+                if "label" in item and item["label"] in [
+                    DocItemLabel.PAGE_HEADER.value,
+                    DocItemLabel.PAGE_FOOTER.value,
+                ]:
+                    item["content_layer"] = "furniture"
+        return data
+
     def add_group(
         self,
         label: Optional[GroupLabel] = None,
         name: Optional[str] = None,
         parent: Optional[NodeItem] = None,
+        content_layer: Optional[ContentLayer] = None,
     ) -> GroupItem:
         """add_group.
 
@@ -1460,6 +1511,8 @@ class DoclingDocument(BaseModel):
             group.name = name
         if label is not None:
             group.label = label
+        if content_layer:
+            group.content_layer = content_layer
 
         self.groups.append(group)
         parent.children.append(RefItem(cref=cref))
@@ -1474,6 +1527,7 @@ class DoclingDocument(BaseModel):
         orig: Optional[str] = None,
         prov: Optional[ProvenanceItem] = None,
         parent: Optional[NodeItem] = None,
+        content_layer: Optional[ContentLayer] = None,
     ):
         """add_list_item.
 
@@ -1504,6 +1558,8 @@ class DoclingDocument(BaseModel):
         )
         if prov:
             list_item.prov.append(prov)
+        if content_layer:
+            list_item.content_layer = content_layer
 
         self.texts.append(list_item)
         parent.children.append(RefItem(cref=cref))
@@ -1517,6 +1573,7 @@ class DoclingDocument(BaseModel):
         orig: Optional[str] = None,
         prov: Optional[ProvenanceItem] = None,
         parent: Optional[NodeItem] = None,
+        content_layer: Optional[ContentLayer] = None,
     ):
         """add_text.
 
@@ -1530,16 +1587,40 @@ class DoclingDocument(BaseModel):
         # Catch a few cases that are in principle allowed
         # but that will create confusion down the road
         if label in [DocItemLabel.TITLE]:
-            return self.add_title(text=text, orig=orig, prov=prov, parent=parent)
+            return self.add_title(
+                text=text,
+                orig=orig,
+                prov=prov,
+                parent=parent,
+                content_layer=content_layer,
+            )
 
         elif label in [DocItemLabel.LIST_ITEM]:
-            return self.add_list_item(text=text, orig=orig, prov=prov, parent=parent)
+            return self.add_list_item(
+                text=text,
+                orig=orig,
+                prov=prov,
+                parent=parent,
+                content_layer=content_layer,
+            )
 
         elif label in [DocItemLabel.SECTION_HEADER]:
-            return self.add_heading(text=text, orig=orig, prov=prov, parent=parent)
+            return self.add_heading(
+                text=text,
+                orig=orig,
+                prov=prov,
+                parent=parent,
+                content_layer=content_layer,
+            )
 
         elif label in [DocItemLabel.CODE]:
-            return self.add_code(text=text, orig=orig, prov=prov, parent=parent)
+            return self.add_code(
+                text=text,
+                orig=orig,
+                prov=prov,
+                parent=parent,
+                content_layer=content_layer,
+            )
 
         else:
 
@@ -1561,6 +1642,9 @@ class DoclingDocument(BaseModel):
             if prov:
                 text_item.prov.append(prov)
 
+            if content_layer:
+                text_item.content_layer = content_layer
+
             self.texts.append(text_item)
             parent.children.append(RefItem(cref=cref))
 
@@ -1573,6 +1657,7 @@ class DoclingDocument(BaseModel):
         prov: Optional[ProvenanceItem] = None,
         parent: Optional[NodeItem] = None,
         label: DocItemLabel = DocItemLabel.TABLE,
+        content_layer: Optional[ContentLayer] = None,
     ):
         """add_table.
 
@@ -1594,6 +1679,9 @@ class DoclingDocument(BaseModel):
         )
         if prov:
             tbl_item.prov.append(prov)
+        if content_layer:
+            tbl_item.content_layer = content_layer
+
         if caption:
             tbl_item.captions.append(caption.get_ref())
 
@@ -1609,6 +1697,7 @@ class DoclingDocument(BaseModel):
         caption: Optional[Union[TextItem, RefItem]] = None,
         prov: Optional[ProvenanceItem] = None,
         parent: Optional[NodeItem] = None,
+        content_layer: Optional[ContentLayer] = None,
     ):
         """add_picture.
 
@@ -1633,6 +1722,8 @@ class DoclingDocument(BaseModel):
         )
         if prov:
             fig_item.prov.append(prov)
+        if content_layer:
+            fig_item.content_layer = content_layer
         if caption:
             fig_item.captions.append(caption.get_ref())
 
@@ -1647,6 +1738,7 @@ class DoclingDocument(BaseModel):
         orig: Optional[str] = None,
         prov: Optional[ProvenanceItem] = None,
         parent: Optional[NodeItem] = None,
+        content_layer: Optional[ContentLayer] = None,
     ):
         """add_title.
 
@@ -1672,6 +1764,8 @@ class DoclingDocument(BaseModel):
         )
         if prov:
             text_item.prov.append(prov)
+        if content_layer:
+            text_item.content_layer = content_layer
 
         self.texts.append(text_item)
         parent.children.append(RefItem(cref=cref))
@@ -1685,6 +1779,7 @@ class DoclingDocument(BaseModel):
         orig: Optional[str] = None,
         prov: Optional[ProvenanceItem] = None,
         parent: Optional[NodeItem] = None,
+        content_layer: Optional[ContentLayer] = None,
     ):
         """add_code.
 
@@ -1710,6 +1805,8 @@ class DoclingDocument(BaseModel):
         )
         if code_language:
             code_item.code_language = code_language
+        if content_layer:
+            code_item.content_layer = content_layer
         if prov:
             code_item.prov.append(prov)
 
@@ -1725,6 +1822,7 @@ class DoclingDocument(BaseModel):
         level: LevelNumber = 1,
         prov: Optional[ProvenanceItem] = None,
         parent: Optional[NodeItem] = None,
+        content_layer: Optional[ContentLayer] = None,
     ):
         """add_heading.
 
@@ -1752,6 +1850,8 @@ class DoclingDocument(BaseModel):
         )
         if prov:
             section_header_item.prov.append(prov)
+        if content_layer:
+            section_header_item.content_layer = content_layer
 
         self.texts.append(section_header_item)
         parent.children.append(RefItem(cref=cref))
@@ -1779,6 +1879,7 @@ class DoclingDocument(BaseModel):
         with_groups: bool = False,
         traverse_pictures: bool = False,
         page_no: Optional[int] = None,
+        included_content_layers: set[ContentLayer] = DEFAULT_CONTENT_LAYERS,
         _level: int = 0,  # fixed parameter, carries through the node nesting level
     ) -> typing.Iterable[Tuple[NodeItem, int]]:  # tuple of node and level
         """iterate_elements.
@@ -1795,14 +1896,22 @@ class DoclingDocument(BaseModel):
             root = self.body
 
         # Yield non-group items or group items when with_groups=True
-        if not isinstance(root, GroupItem) or with_groups:
-            if isinstance(root, DocItem):
-                if page_no is None or any(
-                    prov.page_no == page_no for prov in root.prov
-                ):
-                    yield root, _level
-            else:
-                yield root, _level
+
+        # Combine conditions to have a single yield point
+        should_yield = (
+            (not isinstance(root, GroupItem) or with_groups)
+            and (
+                not isinstance(root, DocItem)
+                or (
+                    page_no is None
+                    or any(prov.page_no == page_no for prov in root.prov)
+                )
+            )
+            and root.content_layer in included_content_layers
+        )
+
+        if should_yield:
+            yield root, _level
 
         # Handle picture traversal - only traverse children if requested
         if isinstance(root, PictureItem) and not traverse_pictures:
@@ -2470,17 +2579,17 @@ class DoclingDocument(BaseModel):
                 continue
 
             elif isinstance(item, TextItem) and item.label in [DocItemLabel.TITLE]:
+                text_inner = _prepare_tag_content(item.text)
+                text = get_html_tag_with_text_direction(html_tag="h1", text=text_inner)
 
-                text = f"<h1>{_prepare_tag_content(item.text)}</h1>"
                 html_texts.append(text)
 
             elif isinstance(item, SectionHeaderItem):
 
                 section_level: int = min(item.level + 1, 6)
 
-                text = (
-                    f"<h{(section_level)}>"
-                    f"{_prepare_tag_content(item.text)}</h{(section_level)}>"
+                text = get_html_tag_with_text_direction(
+                    html_tag=f"h{section_level}", text=_prepare_tag_content(item.text)
                 )
                 html_texts.append(text)
 
@@ -2544,13 +2653,15 @@ class DoclingDocument(BaseModel):
                     )
 
             elif isinstance(item, ListItem):
-
-                text = f"<li>{_prepare_tag_content(item.text)}</li>"
+                text = get_html_tag_with_text_direction(
+                    html_tag="li", text=_prepare_tag_content(item.text)
+                )
                 html_texts.append(text)
 
             elif isinstance(item, TextItem) and item.label in [DocItemLabel.LIST_ITEM]:
-
-                text = f"<li>{_prepare_tag_content(item.text)}</li>"
+                text = get_html_tag_with_text_direction(
+                    html_tag="li", text=_prepare_tag_content(item.text)
+                )
                 html_texts.append(text)
 
             elif isinstance(item, CodeItem):
@@ -2562,8 +2673,11 @@ class DoclingDocument(BaseModel):
 
             elif isinstance(item, TextItem):
 
-                text = f"<p>{_prepare_tag_content(item.text)}</p>"
+                text = get_html_tag_with_text_direction(
+                    html_tag="p", text=_prepare_tag_content(item.text)
+                )
                 html_texts.append(text)
+
             elif isinstance(item, TableItem):
 
                 text = item.export_to_html(doc=self, add_caption=True)

{docling_core-2.17.1 → docling_core-2.18.0}/docling_core/types/doc/utils.py

@@ -5,6 +5,7 @@
 
 """Utils for document types."""
 
+import unicodedata
 from pathlib import Path
 
 
@@ -46,3 +47,29 @@ def relative_path(src: Path, target: Path) -> Path:
 
     # Combine and return the result
     return Path(*up_segments, *down_segments)
+
+
+def get_html_tag_with_text_direction(html_tag: str, text: str) -> str:
+    """Form the HTML element with tag, text, and optional dir attribute."""
+    text_dir = get_text_direction(text)
+
+    if text_dir == "ltr":
+        return f"<{html_tag}>{text}</{html_tag}>"
+    else:
+        return f'<{html_tag} dir="{text_dir}">{text}</{html_tag}>'
+
+
+def get_text_direction(text: str) -> str:
+    """Determine the text direction of a given string as LTR or RTL script."""
+    if not text:
+        return "ltr"  # Default for empty input
+
+    rtl_scripts = {"R", "AL"}
+    rtl_chars = sum(unicodedata.bidirectional(c) in rtl_scripts for c in text)
+
+    return (
+        "rtl"
+        if unicodedata.bidirectional(text[0]) in rtl_scripts
+        or rtl_chars > len(text) / 2
+        else "ltr"
+    )

{docling_core-2.17.1 → docling_core-2.18.0}/docling_core/utils/legacy.py

@@ -25,7 +25,7 @@ from docling_core.types.doc import (
     TableItem,
     TextItem,
 )
-from docling_core.types.doc.document import GroupItem, ListItem, TableData
+from docling_core.types.doc.document import ContentLayer, GroupItem, ListItem, TableData
 from docling_core.types.doc.labels import GroupLabel
 from docling_core.types.legacy_doc.base import (
     BaseCell,
@@ -400,7 +400,7 @@ def legacy_to_docling_document(legacy_doc: DsDocument) -> DoclingDocument:  # no
             doc.add_text(
                 label=DocItemLabel.PAGE_HEADER,
                 text=text_item.text,
-                parent=doc.furniture,
+                content_layer=ContentLayer.FURNITURE,
             )
 
     # page footers
@@ -412,7 +412,7 @@ def legacy_to_docling_document(legacy_doc: DsDocument) -> DoclingDocument:  # no
             doc.add_text(
                 label=DocItemLabel.PAGE_FOOTER,
                 text=text_item.text,
-                parent=doc.furniture,
+                content_layer=ContentLayer.FURNITURE,
             )
 
     # footnotes

{docling_core-2.17.1 → docling_core-2.18.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "docling-core"
-version = "2.17.1"
+version = "2.18.0"
 description = "A python library to define and validate data types in Docling."
 license = "MIT"
 authors = [