docling-core 2.38.2__tar.gz → 2.40.0__tar.gz

{docling_core-2.38.2 → docling_core-2.40.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docling-core
-Version: 2.38.2
+Version: 2.40.0
 Summary: A python library to define and validate data types in Docling.
 Author-email: Cesar Berrospi Ramis <ceb@zurich.ibm.com>, Panos Vagenas <pva@zurich.ibm.com>, Michele Dolfi <dol@zurich.ibm.com>, Christoph Auer <cau@zurich.ibm.com>, Peter Staar <taa@zurich.ibm.com>
 Maintainer-email: Panos Vagenas <pva@zurich.ibm.com>, Michele Dolfi <dol@zurich.ibm.com>, Christoph Auer <cau@zurich.ibm.com>, Peter Staar <taa@zurich.ibm.com>, Cesar Berrospi Ramis <ceb@zurich.ibm.com>

{docling_core-2.38.2 → docling_core-2.40.0}/docling_core/transforms/chunker/hierarchical_chunker.py

@@ -35,11 +35,10 @@ from docling_core.types.doc.document import (
     DocumentOrigin,
     InlineGroup,
     LevelNumber,
-    OrderedList,
+    ListGroup,
     SectionHeaderItem,
     TableItem,
     TitleItem,
-    UnorderedList,
 )
 
 _VERSION: Final = "1.0.0"
@@ -240,7 +239,7 @@ class HierarchicalChunker(BaseChunker):
                     heading_by_level.pop(k, None)
                 continue
             elif (
-                isinstance(item, (OrderedList, UnorderedList, InlineGroup, DocItem))
+                isinstance(item, (ListGroup, InlineGroup, DocItem))
                 and item.self_ref not in visited
             ):
                 ser_res = my_doc_ser.serialize(item=item, visited=visited)

{docling_core-2.38.2 → docling_core-2.40.0}/docling_core/transforms/serializer/base.py

@@ -17,12 +17,11 @@ from docling_core.types.doc.document import (
     FormItem,
     InlineGroup,
     KeyValueItem,
+    ListGroup,
     NodeItem,
-    OrderedList,
     PictureItem,
     TableItem,
     TextItem,
-    UnorderedList,
 )
 
 
@@ -128,7 +127,7 @@ class BaseListSerializer(ABC):
     def serialize(
         self,
         *,
-        item: Union[UnorderedList, OrderedList],
+        item: ListGroup,
         doc_serializer: "BaseDocSerializer",
         doc: DoclingDocument,
         **kwargs: Any,

{docling_core-2.38.2 → docling_core-2.40.0}/docling_core/transforms/serializer/common.py

@@ -39,8 +39,8 @@ from docling_core.types.doc.document import (
     FormItem,
     InlineGroup,
     KeyValueItem,
+    ListGroup,
     NodeItem,
-    OrderedList,
     PictureClassificationData,
     PictureDataType,
     PictureItem,
@@ -49,7 +49,6 @@ from docling_core.types.doc.document import (
     TableAnnotationType,
     TableItem,
     TextItem,
-    UnorderedList,
 )
 from docling_core.types.doc.labels import DocItemLabel
 
@@ -89,7 +88,7 @@ def _iterate_items(
     ):
         if add_page_breaks:
             if (
-                isinstance(item, (UnorderedList, OrderedList, InlineGroup))
+                isinstance(item, (ListGroup, InlineGroup))
                 and item.self_ref not in my_visited
             ):
                 # if group starts with new page, yield page break before group node
@@ -316,7 +315,7 @@ class DocSerializer(BaseModel, BaseDocSerializer):
         ########
         # groups
         ########
-        if isinstance(item, (UnorderedList, OrderedList)):
+        if isinstance(item, ListGroup):
             part = self.list_serializer.serialize(
                 item=item,
                 doc_serializer=self,

{docling_core-2.38.2 → docling_core-2.40.0}/docling_core/transforms/serializer/doctags.py

@@ -1,7 +1,7 @@
 """Define classes for Doctags serialization."""
 
 from enum import Enum
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional
 
 from pydantic import BaseModel
 from typing_extensions import override
@@ -34,9 +34,9 @@ from docling_core.types.doc.document import (
     FormItem,
     InlineGroup,
     KeyValueItem,
+    ListGroup,
     ListItem,
     NodeItem,
-    OrderedList,
     PictureClassificationData,
     PictureItem,
     PictureMoleculeData,
@@ -44,7 +44,6 @@ from docling_core.types.doc.document import (
     ProvenanceItem,
     TableItem,
     TextItem,
-    UnorderedList,
 )
 from docling_core.types.doc.labels import DocItemLabel, PictureClassificationLabel
 from docling_core.types.doc.tokens import DocumentToken
@@ -376,7 +375,7 @@ class DocTagsListSerializer(BaseModel, BaseListSerializer):
     def serialize(
         self,
         *,
-        item: Union[UnorderedList, OrderedList],
+        item: ListGroup,
         doc_serializer: "BaseDocSerializer",
         doc: DoclingDocument,
         list_level: int = 0,
@@ -406,7 +405,7 @@ class DocTagsListSerializer(BaseModel, BaseListSerializer):
             text_res = f"{text_res}{delim}"
             wrap_tag = (
                 DocumentToken.ORDERED_LIST.value
-                if isinstance(item, OrderedList)
+                if item.first_item_is_enumerated(doc)
                 else DocumentToken.UNORDERED_LIST.value
             )
             text_res = _wrap(text=text_res, wrap_tag=wrap_tag)

{docling_core-2.38.2 → docling_core-2.40.0}/docling_core/transforms/serializer/html.py

@@ -58,9 +58,9 @@ from docling_core.types.doc.document import (
     ImageRef,
     InlineGroup,
     KeyValueItem,
+    ListGroup,
     ListItem,
     NodeItem,
-    OrderedList,
     PictureClassificationData,
     PictureItem,
     PictureMoleculeData,
@@ -70,7 +70,6 @@ from docling_core.types.doc.document import (
     TableItem,
     TextItem,
     TitleItem,
-    UnorderedList,
 )
 from docling_core.types.doc.labels import DocItemLabel
 from docling_core.types.doc.utils import (
@@ -117,6 +116,8 @@ class HTMLParams(CommonParams):
 
     include_annotations: bool = True
 
+    show_original_list_item_marker: bool = True
+
 
 class HTMLTextSerializer(BaseModel, BaseTextSerializer):
     """HTML-specific text item serializer."""
@@ -162,7 +163,19 @@ class HTMLTextSerializer(BaseModel, BaseTextSerializer):
         elif isinstance(item, ListItem):
             # List items are handled by list serializer
             text_inner = self._prepare_content(item.text)
-            text = get_html_tag_with_text_direction(html_tag="li", text=text_inner)
+            text = (
+                get_html_tag_with_text_direction(
+                    html_tag="li",
+                    text=text_inner,
+                    attrs=(
+                        {"style": f"list-style-type: '{item.marker} ';"}
+                        if params.show_original_list_item_marker and item.marker
+                        else {}
+                    ),
+                )
+                if text_inner
+                else ""
+            )
 
         elif is_inline_scope:
             text = self._prepare_content(item.text)
@@ -680,7 +693,7 @@ class HTMLListSerializer(BaseModel, BaseListSerializer):
     def serialize(
         self,
         *,
-        item: Union[UnorderedList, OrderedList],
+        item: ListGroup,
         doc_serializer: "BaseDocSerializer",
         doc: DoclingDocument,
         list_level: int = 0,
@@ -690,7 +703,7 @@ class HTMLListSerializer(BaseModel, BaseListSerializer):
     ) -> SerializationResult:
         """Serializes a list to HTML."""
         my_visited: set[str] = visited if visited is not None else set()
-
+        params = HTMLParams(**kwargs)
         # Get all child parts
         parts = doc_serializer.get_parts(
             item=item,
@@ -706,17 +719,51 @@ class HTMLListSerializer(BaseModel, BaseListSerializer):
                 (
                     p.text
                     if (
-                        (p.text.startswith("<li>") and p.text.endswith("</li>"))
-                        or (p.text.startswith("<ol>") and p.text.endswith("</ol>"))
-                        or (p.text.startswith("<ul>") and p.text.endswith("</ul>"))
+                        (
+                            p.text.startswith(("<li>", "<li "))
+                            and p.text.endswith("</li>")
+                        )
+                        or (
+                            p.text.startswith(("<ol>", "<ol "))
+                            and p.text.endswith("</ol>")
+                        )
+                        or (
+                            p.text.startswith(("<ul>", "<ul "))
+                            and p.text.endswith("</ul>")
+                        )
+                    )
+                    else (
+                        get_html_tag_with_text_direction(
+                            html_tag="li",
+                            text=p.text,
+                            attrs=(
+                                {
+                                    "style": f"list-style-type: '{grandparent_item.marker} ';"
+                                }
+                                if params.show_original_list_item_marker
+                                and grandparent_item.marker
+                                else {}
+                            ),
+                        )
+                        if p.spans
+                        and p.spans[0].item.parent
+                        and isinstance(
+                            (parent_item := p.spans[0].item.parent.resolve(doc)),
+                            InlineGroup,
+                        )
+                        and parent_item.parent
+                        and isinstance(
+                            (grandparent_item := parent_item.parent.resolve(doc)),
+                            ListItem,
+                        )
+                        else f"<li>{p.text}</li>"
                     )
-                    else f"<li>{p.text}</li>"
                 )
                 for p in parts
             ]
         )
         if text_res:
-            tag = "ol" if isinstance(item, OrderedList) else "ul"
+            tag = "ol" if item.first_item_is_enumerated(doc) else "ul"
             text_res = f"<{tag}>\n{text_res}\n</{tag}>"
 
         return create_ser_result(text=text_res, span_source=parts)

{docling_core-2.38.2 → docling_core-2.40.0}/docling_core/transforms/serializer/markdown.py

@@ -7,6 +7,7 @@
 import html
 import re
 import textwrap
+from enum import Enum
 from pathlib import Path
 from typing import Any, Optional, Union
 
@@ -31,7 +32,6 @@ from docling_core.transforms.serializer.common import (
     CommonParams,
     DocSerializer,
     _get_annotation_text,
-    _PageBreakSerResult,
     create_ser_result,
 )
 from docling_core.types.doc.base import ImageRefMode
@@ -48,8 +48,9 @@ from docling_core.types.doc.document import (
     ImageRef,
     InlineGroup,
     KeyValueItem,
+    ListGroup,
+    ListItem,
     NodeItem,
-    OrderedList,
     PictureClassificationData,
     PictureItem,
     PictureMoleculeData,
@@ -58,7 +59,6 @@ from docling_core.types.doc.document import (
     TableItem,
     TextItem,
     TitleItem,
-    UnorderedList,
 )
 
 
@@ -79,6 +79,14 @@ def _get_annotation_ser_result(
     )
 
 
+class OrigListItemMarkerMode(str, Enum):
+    """Display mode for original list item marker."""
+
+    NEVER = "never"
+    ALWAYS = "always"
+    AUTO = "auto"
+
+
 class MarkdownParams(CommonParams):
     """Markdown-specific serialization parameters."""
 
@@ -93,6 +101,8 @@ class MarkdownParams(CommonParams):
     escape_html: bool = True
     include_annotations: bool = True
     mark_annotations: bool = False
+    orig_list_item_marker_mode: OrigListItemMarkerMode = OrigListItemMarkerMode.AUTO
+    ensure_valid_list_item_marker: bool = True
 
 
 class MarkdownTextSerializer(BaseModel, BaseTextSerializer):
@@ -117,7 +127,7 @@ class MarkdownTextSerializer(BaseModel, BaseTextSerializer):
         escape_html = True
         escape_underscores = True
         processing_pending = True
-        if isinstance(item, (TitleItem, SectionHeaderItem)):
+        if isinstance(item, (ListItem, TitleItem, SectionHeaderItem)):
             # case where processing/formatting should be applied first (in inner scope)
             processing_pending = False
             if (
@@ -127,7 +137,7 @@ class MarkdownTextSerializer(BaseModel, BaseTextSerializer):
                     (child_group := item.children[0].resolve(doc)), InlineGroup
                 )
             ):
-                # case of heading with inline
+                # case of inline within heading / list item
                 ser_res = doc_serializer.serialize(item=child_group)
                 text = ser_res.text
                 for span in ser_res.spans:
@@ -140,8 +150,55 @@ class MarkdownTextSerializer(BaseModel, BaseTextSerializer):
                     formatting=item.formatting,
                     hyperlink=item.hyperlink,
                 )
-            num_hashes = 1 if isinstance(item, TitleItem) else item.level + 1
-            text_part = f"{num_hashes * '#'} {text}"
+
+            if isinstance(item, ListItem):
+                pieces: list[str] = []
+                case_auto = (
+                    params.orig_list_item_marker_mode == OrigListItemMarkerMode.AUTO
+                    and bool(re.search(r"[a-zA-Z0-9]", item.marker))
+                )
+                case_already_valid = (
+                    params.ensure_valid_list_item_marker
+                    and params.orig_list_item_marker_mode
+                    != OrigListItemMarkerMode.NEVER
+                    and (
+                        item.marker in ["-", "*", "+"]
+                        or re.fullmatch(r"\d+\.", item.marker)
+                    )
+                )
+
+                # wrap with outer marker (if applicable)
+                if params.ensure_valid_list_item_marker and not case_already_valid:
+                    assert item.parent and isinstance(
+                        (list_group := item.parent.resolve(doc)), ListGroup
+                    )
+                    if list_group.first_item_is_enumerated(doc) and (
+                        params.orig_list_item_marker_mode != OrigListItemMarkerMode.AUTO
+                        or not item.marker
+                    ):
+                        pos = -1
+                        for i, child in enumerate(list_group.children):
+                            if child.resolve(doc) == item:
+                                pos = i
+                                break
+                        md_marker = f"{pos + 1}."
+                    else:
+                        md_marker = "-"
+                    pieces.append(md_marker)
+
+                # include original marker (if applicable)
+                if item.marker and (
+                    params.orig_list_item_marker_mode == OrigListItemMarkerMode.ALWAYS
+                    or case_auto
+                    or case_already_valid
+                ):
+                    pieces.append(item.marker)
+
+                pieces.append(text)
+                text_part = " ".join(pieces)
+            else:
+                num_hashes = 1 if isinstance(item, TitleItem) else item.level + 1
+                text_part = f"{num_hashes * '#'} {text}"
         elif isinstance(item, CodeItem):
             text_part = f"`{text}`" if is_inline_scope else f"```\n{text}\n```"
             escape_html = False
@@ -452,7 +509,7 @@ class MarkdownListSerializer(BaseModel, BaseListSerializer):
     def serialize(
         self,
         *,
-        item: Union[UnorderedList, OrderedList],
+        item: ListGroup,
         doc_serializer: "BaseDocSerializer",
         doc: DoclingDocument,
         list_level: int = 0,
@@ -473,27 +530,24 @@ class MarkdownListSerializer(BaseModel, BaseListSerializer):
         sep = "\n"
         my_parts: list[SerializationResult] = []
         for p in parts:
-            if p.text and p.text[0] == " " and my_parts:
-                my_parts[-1].text = sep.join([my_parts[-1].text, p.text])  # update last
+            if (
+                my_parts
+                and p.text
+                and p.spans
+                and p.spans[0].item.parent
+                and isinstance(p.spans[0].item.parent.resolve(doc), InlineGroup)
+            ):
+                my_parts[-1].text = f"{my_parts[-1].text}{p.text}"  # append to last
                 my_parts[-1].spans.extend(p.spans)
             else:
                 my_parts.append(p)
 
         indent_str = list_level * params.indent * " "
-        is_ol = isinstance(item, OrderedList)
         text_res = sep.join(
             [
                 # avoid additional marker on already evaled sublists
-                (
-                    c.text
-                    if c.text and c.text[0] == " "
-                    else (
-                        f"{indent_str}"
-                        f"{'' if isinstance(c, _PageBreakSerResult) else (f'{i + 1}. ' if is_ol else '- ')}"  # noqa: E501
-                        f"{c.text}"
-                    )
-                )
-                for i, c in enumerate(my_parts)
+                (c.text if c.text and c.text[0] == " " else f"{indent_str}{c.text}")
+                for c in my_parts
             ]
         )
         return create_ser_result(text=text_res, span_source=my_parts)

{docling_core-2.38.2 → docling_core-2.40.0}/docling_core/types/doc/__init__.py

@@ -32,6 +32,7 @@ from .document import (
     ImageRef,
     InlineGroup,
     KeyValueItem,
+    ListGroup,
     ListItem,
     MiscAnnotation,
     NodeItem,