docling-core 2.25.0__tar.gz → 2.26.1__tar.gz

{docling_core-2.25.0 → docling_core-2.26.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: docling-core
-Version: 2.25.0
+Version: 2.26.1
 Summary: A python library to define and validate data types in Docling.
 Home-page: https://github.com/docling-project
 License: MIT

{docling_core-2.25.0 → docling_core-2.26.1}/docling_core/experimental/serializer/base.py

@@ -11,6 +11,7 @@ from typing import Optional, Union
 from pydantic import AnyUrl, BaseModel
 
 from docling_core.types.doc.document import (
+    DocItem,
     DoclingDocument,
     FloatingItem,
     FormItem,
@@ -25,10 +26,19 @@ from docling_core.types.doc.document import (
 )
 
 
+class Span(BaseModel):
+    """Class encapsulating fine-granular document span information."""
+
+    item: DocItem
+    # prov_idx: Optional[PositiveInt] = None  # None to be interpreted as whole DocItem
+
+
 class SerializationResult(BaseModel):
     """SerializationResult."""
 
-    text: str
+    text: str = ""
+    spans: list[Span] = []
+    # group: Optional[GroupItem] = None  # set when result reflects specific group item
 
 
 class BaseTextSerializer(ABC):
@@ -163,7 +173,9 @@ class BaseDocSerializer(ABC):
     """Base class for document serializers."""
 
     @abstractmethod
-    def serialize(self, **kwargs) -> SerializationResult:
+    def serialize(
+        self, *, item: Optional[NodeItem] = None, **kwargs
+    ) -> SerializationResult:
         """Run the serialization."""
         ...
 
@@ -222,6 +234,20 @@ class BaseDocSerializer(ABC):
         ...
 
     @abstractmethod
-    def get_excluded_refs(self, **kwargs) -> list[str]:
+    def get_excluded_refs(self, **kwargs) -> set[str]:
         """Get references to excluded items."""
         ...
+
+    @abstractmethod
+    def requires_page_break(self) -> bool:
+        """Whether to add page breaks."""
+        ...
+
+
+class BaseSerializerProvider(ABC):
+    """Base class for document serializer providers."""
+
+    @abstractmethod
+    def get_serializer(self, doc: DoclingDocument) -> BaseDocSerializer:
+        """Get a the associated serializer."""
+        ...

{docling_core-2.25.0 → docling_core-2.26.1}/docling_core/experimental/serializer/common.py

@@ -4,12 +4,12 @@
 #
 
 """Define base classes for serialization."""
+import re
 import sys
 from abc import abstractmethod
-from copy import deepcopy
 from functools import cached_property
 from pathlib import Path
-from typing import Any, Optional, Union
+from typing import Any, Iterable, Optional, Tuple, Union
 
 from pydantic import AnyUrl, BaseModel, NonNegativeInt, computed_field
 from typing_extensions import Self, override
@@ -25,6 +25,7 @@ from docling_core.experimental.serializer.base import (
     BaseTableSerializer,
     BaseTextSerializer,
     SerializationResult,
+    Span,
 )
 from docling_core.types.doc.document import (
     DOCUMENT_TOKENS_EXPORT_LABELS,
@@ -49,6 +50,81 @@ _DEFAULT_LABELS = DOCUMENT_TOKENS_EXPORT_LABELS
 _DEFAULT_LAYERS = {cl for cl in ContentLayer}
 
 
+class _PageBreakNode(NodeItem):
+    """Page break node."""
+
+    prev_page: int
+    next_page: int
+
+
+class _PageBreakSerResult(SerializationResult):
+    """Page break serialization result."""
+
+    node: _PageBreakNode
+
+
+def _iterate_items(
+    doc: DoclingDocument,
+    layers: Optional[set[ContentLayer]],
+    node: Optional[NodeItem] = None,
+    traverse_pictures: bool = False,
+    add_page_breaks: bool = False,
+):
+    prev_page_nr: Optional[int] = None
+    page_break_i = 0
+    for item, _ in doc.iterate_items(
+        root=node,
+        with_groups=True,
+        included_content_layers=layers,
+        traverse_pictures=traverse_pictures,
+    ):
+        if isinstance(item, DocItem):
+            if item.prov:
+                page_no = item.prov[0].page_no
+                if add_page_breaks and (prev_page_nr is None or page_no > prev_page_nr):
+                    if prev_page_nr is not None:  # close previous range
+                        yield _PageBreakNode(
+                            self_ref=f"#/pb/{page_break_i}",
+                            prev_page=prev_page_nr,
+                            next_page=page_no,
+                        )
+                        page_break_i += 1
+                    prev_page_nr = page_no
+        yield item
+
+
+def create_ser_result(
+    *,
+    text: str = "",
+    span_source: Union[DocItem, list[SerializationResult]] = [],
+) -> SerializationResult:
+    """Function for creating `SerializationResult` instances.
+
+    Args:
+        text: the text the use. Defaults to "".
+        span_source: the item or list of results to use as span source. Defaults to [].
+
+    Returns:
+        The created `SerializationResult`.
+    """
+    spans: list[Span]
+    if isinstance(span_source, DocItem):
+        spans = [Span(item=span_source)]
+    else:
+        results: list[SerializationResult] = span_source
+        spans = []
+        span_ids: set[str] = set()
+        for ser_res in results:
+            for span in ser_res.spans:
+                if (span_id := span.item.self_ref) not in span_ids:
+                    span_ids.add(span_id)
+                    spans.append(span)
+    return SerializationResult(
+        text=text,
+        spans=spans,
+    )
+
+
 class CommonParams(BaseModel):
     """Common serialization parameters."""
 
@@ -95,7 +171,7 @@ class DocSerializer(BaseModel, BaseDocSerializer):
 
     params: CommonParams = CommonParams()
 
-    _excluded_refs_cache: dict[str, list[str]] = {}
+    _excluded_refs_cache: dict[str, set[str]] = {}
 
     @computed_field  # type: ignore[misc]
     @cached_property
@@ -113,19 +189,19 @@ class DocSerializer(BaseModel, BaseDocSerializer):
         return refs
 
     @override
-    def get_excluded_refs(self, **kwargs) -> list[str]:
+    def get_excluded_refs(self, **kwargs) -> set[str]:
         """References to excluded items."""
         params = self.params.merge_with_patch(patch=kwargs)
         params_json = params.model_dump_json()
         refs = self._excluded_refs_cache.get(params_json)
         if refs is None:
-            refs = [
+            refs = {
                 item.self_ref
-                for ix, (item, _) in enumerate(
-                    self.doc.iterate_items(
-                        with_groups=True,
+                for ix, item in enumerate(
+                    _iterate_items(
+                        doc=self.doc,
                         traverse_pictures=True,
-                        included_content_layers=params.layers,
+                        layers=params.layers,
                     )
                 )
                 if (
@@ -145,56 +221,21 @@ class DocSerializer(BaseModel, BaseDocSerializer):
                         )
                     )
                 )
-            ]
+            }
             self._excluded_refs_cache[params_json] = refs
         return refs
 
     @abstractmethod
-    def serialize_page(self, parts: list[SerializationResult]) -> SerializationResult:
-        """Serialize a page out of its parts."""
-        ...
-
-    @abstractmethod
-    def serialize_doc(self, pages: list[SerializationResult]) -> SerializationResult:
+    def serialize_doc(
+        self, *, parts: list[SerializationResult], **kwargs
+    ) -> SerializationResult:
         """Serialize a document out of its pages."""
         ...
 
     def _serialize_body(self) -> SerializationResult:
         """Serialize the document body."""
-        # find page ranges if available; otherwise regard whole doc as a single page
-        last_page: Optional[int] = None
-        starts: list[int] = []
-        for ix, (item, _) in enumerate(
-            self.doc.iterate_items(
-                with_groups=True,
-                traverse_pictures=True,
-                included_content_layers=self.params.layers,
-            )
-        ):
-            if isinstance(item, DocItem):
-                if item.prov:
-                    if last_page is None or item.prov[0].page_no > last_page:
-                        starts.append(ix)
-                        last_page = item.prov[0].page_no
-        page_ranges = [
-            (
-                (starts[i] if i > 0 else 0),
-                (starts[i + 1] if i < len(starts) - 1 else sys.maxsize),
-            )
-            for i, _ in enumerate(starts)
-        ] or [
-            (0, sys.maxsize)
-        ]  # use whole range if no pages detected
-
-        page_results: list[SerializationResult] = []
-        for page_range in page_ranges:
-            params_to_pass = deepcopy(self.params)
-            params_to_pass.start_idx = page_range[0]
-            params_to_pass.stop_idx = page_range[1]
-            subparts = self.get_parts(**params_to_pass.model_dump())
-            page_res = self.serialize_page(subparts)
-            page_results.append(page_res)
-        res = self.serialize_doc(page_results)
+        subparts = self.get_parts()
+        res = self.serialize_doc(parts=subparts)
         return res
 
     @override
@@ -209,7 +250,8 @@ class DocSerializer(BaseModel, BaseDocSerializer):
     ) -> SerializationResult:
         """Serialize a given node."""
         my_visited: set[str] = visited if visited is not None else set()
-        empty_res = SerializationResult(text="")
+        my_kwargs = self.params.merge_with_patch(patch=kwargs).model_dump()
+        empty_res = create_ser_result()
         if item is None or item == self.doc.body:
             if self.doc.body.self_ref not in my_visited:
                 my_visited.add(self.doc.body.self_ref)
@@ -217,6 +259,8 @@ class DocSerializer(BaseModel, BaseDocSerializer):
             else:
                 return empty_res
 
+        my_visited.add(item.self_ref)
+
         ########
         # groups
         ########
@@ -228,7 +272,7 @@ class DocSerializer(BaseModel, BaseDocSerializer):
                 list_level=list_level,
                 is_inline_scope=is_inline_scope,
                 visited=my_visited,
-                **kwargs,
+                **my_kwargs,
             )
         elif isinstance(item, InlineGroup):
             part = self.inline_serializer.serialize(
@@ -237,7 +281,7 @@ class DocSerializer(BaseModel, BaseDocSerializer):
                 doc=self.doc,
                 list_level=list_level,
                 visited=my_visited,
-                **kwargs,
+                **my_kwargs,
             )
         ###########
         # doc items
@@ -253,7 +297,7 @@ class DocSerializer(BaseModel, BaseDocSerializer):
                         doc_serializer=self,
                         doc=self.doc,
                         is_inline_scope=is_inline_scope,
-                        **kwargs,
+                        **my_kwargs,
                     )
                     if item.self_ref not in self.get_excluded_refs(**kwargs)
                     else empty_res
@@ -263,7 +307,7 @@ class DocSerializer(BaseModel, BaseDocSerializer):
                 item=item,
                 doc_serializer=self,
                 doc=self.doc,
-                **kwargs,
+                **my_kwargs,
             )
         elif isinstance(item, PictureItem):
             part = self.picture_serializer.serialize(
@@ -271,28 +315,33 @@ class DocSerializer(BaseModel, BaseDocSerializer):
                 doc_serializer=self,
                 doc=self.doc,
                 visited=my_visited,
-                **kwargs,
+                **my_kwargs,
             )
         elif isinstance(item, KeyValueItem):
             part = self.key_value_serializer.serialize(
                 item=item,
                 doc_serializer=self,
                 doc=self.doc,
-                **kwargs,
+                **my_kwargs,
             )
         elif isinstance(item, FormItem):
             part = self.form_serializer.serialize(
                 item=item,
                 doc_serializer=self,
                 doc=self.doc,
-                **kwargs,
+                **my_kwargs,
+            )
+        elif isinstance(item, _PageBreakNode):
+            part = _PageBreakSerResult(
+                text=self._create_page_break(node=item),
+                node=item,
             )
         else:
             part = self.fallback_serializer.serialize(
                 item=item,
                 doc_serializer=self,
                 doc=self.doc,
-                **kwargs,
+                **my_kwargs,
             )
         return part
 
@@ -312,18 +361,19 @@ class DocSerializer(BaseModel, BaseDocSerializer):
         parts: list[SerializationResult] = []
         my_visited: set[str] = visited if visited is not None else set()
         params = self.params.merge_with_patch(patch=kwargs)
-        for item, _ in self.doc.iterate_items(
-            root=item,
-            with_groups=True,
-            traverse_pictures=traverse_pictures,
-            included_content_layers=params.layers,
+
+        for node in _iterate_items(
+            node=item,
+            doc=self.doc,
+            layers=params.layers,
+            add_page_breaks=self.requires_page_break(),
         ):
-            if item.self_ref in my_visited:
+            if node.self_ref in my_visited:
                 continue
             else:
-                my_visited.add(item.self_ref)
+                my_visited.add(node.self_ref)
             part = self.serialize(
-                item=item,
+                item=node,
                 list_level=list_level,
                 is_inline_scope=is_inline_scope,
                 visited=my_visited,
@@ -393,15 +443,51 @@ class DocSerializer(BaseModel, BaseDocSerializer):
     ) -> SerializationResult:
         """Serialize the item's captions."""
         params = self.params.merge_with_patch(patch=kwargs)
+        results: list[SerializationResult] = []
         if DocItemLabel.CAPTION in params.labels:
-            text_parts: list[str] = [
-                it.text
+            results = [
+                create_ser_result(text=it.text, span_source=it)
                 for cap in item.captions
                 if isinstance(it := cap.resolve(self.doc), TextItem)
                 and it.self_ref not in self.get_excluded_refs(**kwargs)
             ]
-            text_res = params.caption_delim.join(text_parts)
+            text_res = params.caption_delim.join([r.text for r in results])
             text_res = self.post_process(text=text_res)
         else:
             text_res = ""
-        return SerializationResult(text=text_res)
+        return create_ser_result(text=text_res, span_source=results)
+
+    def _get_applicable_pages(self) -> Optional[list[int]]:
+        pages = {
+            item.prov[0].page_no: ...
+            for ix, (item, _) in enumerate(
+                self.doc.iterate_items(
+                    with_groups=True,
+                    included_content_layers=self.params.layers,
+                    traverse_pictures=True,
+                )
+            )
+            if (
+                isinstance(item, DocItem)
+                and item.prov
+                and (
+                    self.params.pages is None
+                    or item.prov[0].page_no in self.params.pages
+                )
+                and ix >= self.params.start_idx
+                and ix < self.params.stop_idx
+            )
+        }
+        return [p for p in pages] or None
+
+    def _create_page_break(self, node: _PageBreakNode) -> str:
+        return f"#_#_DOCLING_DOC_PAGE_BREAK_{node.prev_page}_{node.next_page}_#_#"
+
+    def _get_page_breaks(self, text: str) -> Iterable[Tuple[str, int, int]]:
+        pattern = r"#_#_DOCLING_DOC_PAGE_BREAK_(\d+)_(\d+)_#_#"
+        matches = re.finditer(pattern, text)
+        for match in matches:
+            full_match = match.group(0)
+            prev_page_nr = int(match.group(1))
+            next_page_nr = int(match.group(2))
+            yield (full_match, prev_page_nr, next_page_nr)