docling-core 2.8.0__tar.gz → 2.10.0__tar.gz

{docling_core-2.8.0 → docling_core-2.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: docling-core
-Version: 2.8.0
+Version: 2.10.0
 Summary: A python library to define and validate data types in Docling.
 Home-page: https://ds4sd.github.io/
 License: MIT
@@ -35,6 +35,7 @@ Requires-Dist: pyyaml (>=5.1,<7.0.0)
 Requires-Dist: semchunk (>=2.2.0,<3.0.0) ; extra == "chunking"
 Requires-Dist: tabulate (>=0.9.0,<0.10.0)
 Requires-Dist: transformers (>=4.34.0,<5.0.0) ; extra == "chunking"
+Requires-Dist: typer (>=0.12.5,<0.13.0)
 Requires-Dist: typing-extensions (>=4.12.2,<5.0.0)
 Project-URL: Repository, https://github.com/DS4SD/docling-core
 Description-Content-Type: text/markdown

docling_core-2.10.0/docling_core/cli/__init__.py

@@ -0,0 +1 @@
+"""CLI package."""

docling_core-2.10.0/docling_core/cli/view.py

@@ -0,0 +1,68 @@
+#
+# Copyright IBM Corp. 2024 - 2024
+# SPDX-License-Identifier: MIT
+#
+
+"""CLI for docling viewer."""
+import importlib
+import tempfile
+import webbrowser
+from pathlib import Path
+from typing import Annotated, Optional
+
+import typer
+
+from docling_core.types.doc import DoclingDocument
+from docling_core.types.doc.base import ImageRefMode
+from docling_core.utils.file import resolve_source_to_path
+
+app = typer.Typer(
+    name="Docling",
+    no_args_is_help=True,
+    add_completion=False,
+    pretty_exceptions_enable=False,
+)
+
+
+def version_callback(value: bool):
+    """Callback for version inspection."""
+    if value:
+        docling_core_version = importlib.metadata.version("docling-core")
+        print(f"Docling Core version: {docling_core_version}")
+        raise typer.Exit()
+
+
+@app.command(no_args_is_help=True)
+def view(
+    source: Annotated[
+        str,
+        typer.Argument(
+            ...,
+            metavar="source",
+            help="Docling JSON file to view.",
+        ),
+    ],
+    version: Annotated[
+        Optional[bool],
+        typer.Option(
+            "--version",
+            callback=version_callback,
+            is_eager=True,
+            help="Show version information.",
+        ),
+    ] = None,
+):
+    """Display a Docling JSON file on the default browser."""
+    path = resolve_source_to_path(source=source)
+    doc = DoclingDocument.load_from_json(filename=path)
+    target_path = Path(tempfile.mkdtemp()) / "out.html"
+    html_output = doc.export_to_html(image_mode=ImageRefMode.EMBEDDED)
+    with open(target_path, "w") as f:
+        f.write(html_output)
+    webbrowser.open(url=f"file://{target_path.absolute().resolve()}")
+
+
+click_app = typer.main.get_command(app)
+
+if __name__ == "__main__":
+    app()

{docling_core-2.8.0 → docling_core-2.10.0}/docling_core/transforms/chunker/hybrid_chunker.py

@@ -44,7 +44,9 @@ class HybridChunker(BaseChunker):
 
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
-    tokenizer: Union[PreTrainedTokenizerBase, str]
+    tokenizer: Union[PreTrainedTokenizerBase, str] = (
+        "sentence-transformers/all-MiniLM-L6-v2"
+    )
     max_tokens: int = None  # type: ignore[assignment]
     merge_peers: bool = True
 
@@ -96,6 +98,7 @@ class HybridChunker(BaseChunker):
             doc_items=doc_chunk.meta.doc_items[window_start : window_end + 1],
             headings=doc_chunk.meta.headings,
             captions=doc_chunk.meta.captions,
+            origin=doc_chunk.meta.origin,
         )
         new_chunk = DocChunk(text=window_text, meta=meta)
         return new_chunk
@@ -242,6 +245,7 @@ class HybridChunker(BaseChunker):
                         doc_items=window_items,
                         headings=current_headings_and_captions[0],
                         captions=current_headings_and_captions[1],
+                        origin=chunk.meta.origin,
                     )
                     new_chunk = DocChunk(
                         text=window_text,

{docling_core-2.8.0 → docling_core-2.10.0}/docling_core/types/doc/document.py

@@ -49,7 +49,6 @@ DEFAULT_EXPORT_LABELS = {
     DocItemLabel.DOCUMENT_INDEX,
     DocItemLabel.SECTION_HEADER,
     DocItemLabel.PARAGRAPH,
-    DocItemLabel.CAPTION,
     DocItemLabel.TABLE,
     DocItemLabel.PICTURE,
     DocItemLabel.FORMULA,
@@ -58,6 +57,7 @@ DEFAULT_EXPORT_LABELS = {
     DocItemLabel.TEXT,
     DocItemLabel.LIST_ITEM,
     DocItemLabel.CODE,
+    DocItemLabel.REFERENCE,
 }
 
 
@@ -380,6 +380,7 @@ class DocumentOrigin(BaseModel):
         "application/vnd.openxmlformats-officedocument.presentationml.template",
         "application/vnd.openxmlformats-officedocument.presentationml.slideshow",
         "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
         "text/asciidoc",
         "text/markdown",
     ]
@@ -445,7 +446,7 @@ class ImageRef(BaseModel):
     mimetype: str
     dpi: int
     size: Size
-    uri: Union[AnyUrl, Path]
+    uri: Union[AnyUrl, Path] = Field(union_mode="left_to_right")
     _pil: Optional[PILImage.Image] = None
 
     @property
@@ -592,6 +593,21 @@ class DocItem(
 class TextItem(DocItem):
     """TextItem."""
 
+    label: typing.Literal[
+        DocItemLabel.CAPTION,
+        DocItemLabel.CHECKBOX_SELECTED,
+        DocItemLabel.CHECKBOX_UNSELECTED,
+        DocItemLabel.CODE,
+        DocItemLabel.FOOTNOTE,
+        DocItemLabel.FORMULA,
+        DocItemLabel.PAGE_FOOTER,
+        DocItemLabel.PAGE_HEADER,
+        DocItemLabel.PARAGRAPH,
+        DocItemLabel.REFERENCE,
+        DocItemLabel.TEXT,
+        DocItemLabel.TITLE,
+    ]
+
     orig: str  # untreated representation
     text: str  # sanitized representation
 
@@ -643,8 +659,10 @@ class TextItem(DocItem):
 class SectionHeaderItem(TextItem):
     """SectionItem."""
 
-    label: typing.Literal[DocItemLabel.SECTION_HEADER] = DocItemLabel.SECTION_HEADER
-    level: LevelNumber
+    label: typing.Literal[DocItemLabel.SECTION_HEADER] = (
+        DocItemLabel.SECTION_HEADER  # type: ignore[assignment]
+    )
+    level: LevelNumber = 1
 
     def export_to_document_tokens(
         self,
@@ -694,9 +712,11 @@ class SectionHeaderItem(TextItem):
 class ListItem(TextItem):
     """SectionItem."""
 
-    label: typing.Literal[DocItemLabel.LIST_ITEM] = DocItemLabel.LIST_ITEM
+    label: typing.Literal[DocItemLabel.LIST_ITEM] = (
+        DocItemLabel.LIST_ITEM  # type: ignore[assignment]
+    )
     enumerated: bool = False
-    marker: str  # The bullet or number symbol that prefixes this list item
+    marker: str = "-"  # The bullet or number symbol that prefixes this list item
 
 
 class FloatingItem(DocItem):
@@ -922,7 +942,10 @@ class TableItem(FloatingItem):
     """TableItem."""
 
     data: TableData
-    label: typing.Literal[DocItemLabel.TABLE] = DocItemLabel.TABLE
+    label: typing.Literal[
+        DocItemLabel.DOCUMENT_INDEX,
+        DocItemLabel.TABLE,
+    ] = DocItemLabel.TABLE
 
     def export_to_dataframe(self) -> pd.DataFrame:
         """Export the table as a Pandas DataFrame."""
@@ -1271,9 +1294,19 @@ class TableItem(FloatingItem):
 class KeyValueItem(DocItem):
     """KeyValueItem."""
 
+    label: typing.Literal[DocItemLabel.KEY_VALUE_REGION] = DocItemLabel.KEY_VALUE_REGION
 
-ContentItem = Union[
-    TextItem, SectionHeaderItem, ListItem, PictureItem, TableItem, KeyValueItem
+
+ContentItem = Annotated[
+    Union[
+        TextItem,
+        SectionHeaderItem,
+        ListItem,
+        PictureItem,
+        TableItem,
+        KeyValueItem,
+    ],
+    Field(discriminator="label"),
 ]
 
 
@@ -1375,13 +1408,13 @@ class DoclingDocument(BaseModel):
         self,
         label: Optional[GroupLabel] = None,
         name: Optional[str] = None,
-        parent: Optional[GroupItem] = None,
+        parent: Optional[NodeItem] = None,
     ) -> GroupItem:
         """add_group.
 
         :param label: Optional[GroupLabel]:  (Default value = None)
         :param name: Optional[str]:  (Default value = None)
-        :param parent: Optional[GroupItem]:  (Default value = None)
+        :param parent: Optional[NodeItem]:  (Default value = None)
 
         """
         if not parent:
@@ -1408,7 +1441,7 @@ class DoclingDocument(BaseModel):
         marker: Optional[str] = None,
         orig: Optional[str] = None,
         prov: Optional[ProvenanceItem] = None,
-        parent: Optional[GroupItem] = None,
+        parent: Optional[NodeItem] = None,
     ):
         """add_list_item.
 
@@ -1416,7 +1449,7 @@ class DoclingDocument(BaseModel):
         :param text: str:
         :param orig: Optional[str]:  (Default value = None)
         :param prov: Optional[ProvenanceItem]:  (Default value = None)
-        :param parent: Optional[GroupItem]:  (Default value = None)
+        :param parent: Optional[NodeItem]:  (Default value = None)
 
         """
         if not parent:
@@ -1451,7 +1484,7 @@ class DoclingDocument(BaseModel):
         text: str,
         orig: Optional[str] = None,
         prov: Optional[ProvenanceItem] = None,
-        parent: Optional[GroupItem] = None,
+        parent: Optional[NodeItem] = None,
     ):
         """add_text.
 
@@ -1459,7 +1492,7 @@ class DoclingDocument(BaseModel):
         :param text: str:
         :param orig: Optional[str]:  (Default value = None)
         :param prov: Optional[ProvenanceItem]:  (Default value = None)
-        :param parent: Optional[GroupItem]:  (Default value = None)
+        :param parent: Optional[NodeItem]:  (Default value = None)
 
         """
         # Catch a few cases that are in principle allowed
@@ -1503,15 +1536,16 @@ class DoclingDocument(BaseModel):
         data: TableData,
         caption: Optional[Union[TextItem, RefItem]] = None,  # This is not cool yet.
         prov: Optional[ProvenanceItem] = None,
-        parent: Optional[GroupItem] = None,
+        parent: Optional[NodeItem] = None,
+        label: DocItemLabel = DocItemLabel.TABLE,
     ):
         """add_table.
 
-        :param data: BaseTableData:
-        :param caption: Optional[Union[TextItem:
-        :param RefItem]]:  (Default value = None)
-        :param # This is not cool yet.prov: Optional[ProvenanceItem]
-        :param parent: Optional[GroupItem]:  (Default value = None)
+        :param data: TableData:
+        :param caption: Optional[Union[TextItem, RefItem]]:  (Default value = None)
+        :param prov: Optional[ProvenanceItem]:  (Default value = None)
+        :param parent: Optional[NodeItem]:  (Default value = None)
+        :param label: DocItemLabel:  (Default value = DocItemLabel.TABLE)
 
         """
         if not parent:
@@ -1521,7 +1555,7 @@ class DoclingDocument(BaseModel):
         cref = f"#/tables/{table_index}"
 
         tbl_item = TableItem(
-            label=DocItemLabel.TABLE, data=data, self_ref=cref, parent=parent.get_ref()
+            label=label, data=data, self_ref=cref, parent=parent.get_ref()
         )
         if prov:
             tbl_item.prov.append(prov)
@@ -1539,7 +1573,7 @@ class DoclingDocument(BaseModel):
         image: Optional[ImageRef] = None,
         caption: Optional[Union[TextItem, RefItem]] = None,
         prov: Optional[ProvenanceItem] = None,
-        parent: Optional[GroupItem] = None,
+        parent: Optional[NodeItem] = None,
     ):
         """add_picture.
 
@@ -1547,7 +1581,7 @@ class DoclingDocument(BaseModel):
         :param caption: Optional[Union[TextItem:
         :param RefItem]]:  (Default value = None)
         :param prov: Optional[ProvenanceItem]:  (Default value = None)
-        :param parent: Optional[GroupItem]:  (Default value = None)
+        :param parent: Optional[NodeItem]:  (Default value = None)
         """
         if not parent:
             parent = self.body
@@ -1577,14 +1611,14 @@ class DoclingDocument(BaseModel):
         text: str,
         orig: Optional[str] = None,
         prov: Optional[ProvenanceItem] = None,
-        parent: Optional[GroupItem] = None,
+        parent: Optional[NodeItem] = None,
     ):
         """add_title.
 
         :param text: str:
         :param orig: Optional[str]:  (Default value = None)
         :param prov: Optional[ProvenanceItem]:  (Default value = None)
-        :param parent: Optional[GroupItem]:  (Default value = None)
+        :param parent: Optional[NodeItem]:  (Default value = None)
         """
         if not parent:
             parent = self.body
@@ -1615,7 +1649,7 @@ class DoclingDocument(BaseModel):
         orig: Optional[str] = None,
         level: LevelNumber = 1,
         prov: Optional[ProvenanceItem] = None,
-        parent: Optional[GroupItem] = None,
+        parent: Optional[NodeItem] = None,
     ):
         """add_heading.
 
@@ -1624,7 +1658,7 @@ class DoclingDocument(BaseModel):
         :param orig: Optional[str]:  (Default value = None)
         :param level: LevelNumber:  (Default value = 1)
         :param prov: Optional[ProvenanceItem]:  (Default value = None)
-        :param parent: Optional[GroupItem]:  (Default value = None)
+        :param parent: Optional[NodeItem]:  (Default value = None)
         """
         if not parent:
             parent = self.body
@@ -1668,7 +1702,7 @@ class DoclingDocument(BaseModel):
         self,
         root: Optional[NodeItem] = None,
         with_groups: bool = False,
-        traverse_pictures: bool = True,
+        traverse_pictures: bool = False,
         page_no: Optional[int] = None,
         _level: int = 0,  # fixed parameter, carries through the node nesting level
     ) -> typing.Iterable[Tuple[NodeItem, int]]:  # tuple of node and level
@@ -1685,30 +1719,31 @@ class DoclingDocument(BaseModel):
         if not root:
             root = self.body
 
+        # Yield non-group items or group items when with_groups=True
         if not isinstance(root, GroupItem) or with_groups:
             if isinstance(root, DocItem):
-                if page_no is not None:
-                    for prov in root.prov:
-                        if prov.page_no == page_no:
-                            yield root, _level
-                else:
+                if page_no is None or any(
+                    prov.page_no == page_no for prov in root.prov
+                ):
                     yield root, _level
             else:
                 yield root, _level
 
+        # Handle picture traversal - only traverse children if requested
+        if isinstance(root, PictureItem) and not traverse_pictures:
+            return
+
         # Traverse children
         for child_ref in root.children:
             child = child_ref.resolve(self)
-
             if isinstance(child, NodeItem):
-                # If the child is a NodeItem, recursively traverse it
-                if not isinstance(child, PictureItem) or traverse_pictures:
-                    yield from self.iterate_items(
-                        child,
-                        _level=_level + 1,
-                        with_groups=with_groups,
-                        page_no=page_no,
-                    )
+                yield from self.iterate_items(
+                    child,
+                    with_groups=with_groups,
+                    traverse_pictures=traverse_pictures,
+                    page_no=page_no,
+                    _level=_level + 1,
+                )
 
     def _clear_picture_pil_cache(self):
         """Clear cache storage of all images."""
@@ -1864,7 +1899,7 @@ class DoclingDocument(BaseModel):
 
         """
         with open(filename, "r") as f:
-            return cls.model_validate(json.loads(f.read()))
+            return cls.model_validate_json(f.read())
 
     def save_as_yaml(
         self,
@@ -2053,10 +2088,6 @@ class DoclingDocument(BaseModel):
                 text = f"```\n{item.text}\n```\n"
                 mdtexts.append(text)
 
-            elif isinstance(item, TextItem) and item.label in [DocItemLabel.CAPTION]:
-                # captions are printed in picture and table ... skipping for now
-                continue
-
             elif isinstance(item, ListItem) and item.label in [DocItemLabel.LIST_ITEM]:
                 in_list = True
                 # Calculate indent based on list_nesting_level
@@ -2115,10 +2146,30 @@ class DoclingDocument(BaseModel):
         # Bold, Italic, or Bold-Italic
         # Hence, any underscore that we print into Markdown is coming from document text
         # That means we need to escape it, to properly reflect content in the markdown
+        # However, we need to preserve underscores in image URLs
+        # to maintain their validity
+        # For example: ![image](path/to_image.png) should remain unchanged
         def escape_underscores(text):
-            # Replace "_" with "\_" only if it's not already escaped
-            escaped_text = re.sub(r"(?<!\\)_", r"\_", text)
-            return escaped_text
+            """Escape underscores but leave them intact in the URL.."""
+            # Firstly, identify all the URL patterns.
+            url_pattern = r"!\[.*?\]\((.*?)\)"
+            parts = []
+            last_end = 0
+
+            for match in re.finditer(url_pattern, text):
+                # Text to add before the URL (needs to be escaped)
+                before_url = text[last_end : match.start()]
+                parts.append(re.sub(r"(?<!\\)_", r"\_", before_url))
+
+                # Add the full URL part (do not escape)
+                parts.append(match.group(0))
+                last_end = match.end()
+
+            # Add the final part of the text (which needs to be escaped)
+            if last_end < len(text):
+                parts.append(re.sub(r"(?<!\\)_", r"\_", text[last_end:]))
+
+            return "".join(parts)
 
         mdtext = escape_underscores(mdtext)
 
@@ -2328,10 +2379,6 @@ class DoclingDocument(BaseModel):
                 text = f"<pre>{item.text}</pre>"
                 html_texts.append(text)
 
-            elif isinstance(item, TextItem) and item.label in [DocItemLabel.CAPTION]:
-                # captions are printed in picture and table ... skipping for now
-                continue
-
             elif isinstance(item, ListItem):
 
                 text = f"<li>{item.text}</li>"
@@ -2533,10 +2580,6 @@ class DoclingDocument(BaseModel):
                 result += f"<unordered_list>{delim}"
                 in_ordered_list.append(False)
 
-            elif isinstance(item, TextItem) and item.label in [DocItemLabel.CAPTION]:
-                # captions are printed in picture and table ... skipping for now
-                continue
-
             elif isinstance(item, SectionHeaderItem):
 
                 result += item.export_to_document_tokens(
@@ -2642,10 +2685,6 @@ class DoclingDocument(BaseModel):
                     indent * level + f"item-{i} at level {level}: {item.label}: {text}"
                 )
 
-            elif isinstance(item, TextItem) and item.label in [DocItemLabel.CAPTION]:
-                # captions are printed in picture and table ... skipping for now
-                continue
-
             elif isinstance(item, ListItem) and item.label in [DocItemLabel.LIST_ITEM]:
                 text = get_text(text=item.text, max_text_len=max_text_len)
 

{docling_core-2.8.0 → docling_core-2.10.0}/docling_core/types/legacy_doc/base.py

@@ -140,6 +140,7 @@ class BaseCell(AliasModel):
     obj_type: str = Field(
         alias="type", json_schema_extra=es_field(type="keyword", ignore_above=8191)
     )
+    payload: Optional[dict] = None
 
     def get_location_tokens(
         self,