docling-core 2.26.0__py3-none-any.whl → 2.26.2__py3-none-any.whl

docling_core/experimental/serializer/base.py

@@ -234,10 +234,15 @@ class BaseDocSerializer(ABC):
         ...
 
     @abstractmethod
-    def get_excluded_refs(self, **kwargs) -> list[str]:
+    def get_excluded_refs(self, **kwargs) -> set[str]:
         """Get references to excluded items."""
         ...
 
+    @abstractmethod
+    def requires_page_break(self) -> bool:
+        """Whether to add page breaks."""
+        ...
+
 
 class BaseSerializerProvider(ABC):
     """Base class for document serializer providers."""

docling_core/experimental/serializer/common.py

@@ -4,12 +4,12 @@
 #
 
 """Define base classes for serialization."""
+import re
 import sys
 from abc import abstractmethod
-from copy import deepcopy
 from functools import cached_property
 from pathlib import Path
-from typing import Any, Optional, Union
+from typing import Any, Iterable, Optional, Tuple, Union
 
 from pydantic import AnyUrl, BaseModel, NonNegativeInt, computed_field
 from typing_extensions import Self, override
@@ -50,6 +50,49 @@ _DEFAULT_LABELS = DOCUMENT_TOKENS_EXPORT_LABELS
 _DEFAULT_LAYERS = {cl for cl in ContentLayer}
 
 
+class _PageBreakNode(NodeItem):
+    """Page break node."""
+
+    prev_page: int
+    next_page: int
+
+
+class _PageBreakSerResult(SerializationResult):
+    """Page break serialization result."""
+
+    node: _PageBreakNode
+
+
+def _iterate_items(
+    doc: DoclingDocument,
+    layers: Optional[set[ContentLayer]],
+    node: Optional[NodeItem] = None,
+    traverse_pictures: bool = False,
+    add_page_breaks: bool = False,
+):
+    prev_page_nr: Optional[int] = None
+    page_break_i = 0
+    for item, _ in doc.iterate_items(
+        root=node,
+        with_groups=True,
+        included_content_layers=layers,
+        traverse_pictures=traverse_pictures,
+    ):
+        if isinstance(item, DocItem):
+            if item.prov:
+                page_no = item.prov[0].page_no
+                if add_page_breaks and (prev_page_nr is None or page_no > prev_page_nr):
+                    if prev_page_nr is not None:  # close previous range
+                        yield _PageBreakNode(
+                            self_ref=f"#/pb/{page_break_i}",
+                            prev_page=prev_page_nr,
+                            next_page=page_no,
+                        )
+                        page_break_i += 1
+                    prev_page_nr = page_no
+        yield item
+
+
 def create_ser_result(
     *,
     text: str = "",
@@ -128,7 +171,7 @@ class DocSerializer(BaseModel, BaseDocSerializer):
 
     params: CommonParams = CommonParams()
 
-    _excluded_refs_cache: dict[str, list[str]] = {}
+    _excluded_refs_cache: dict[str, set[str]] = {}
 
     @computed_field  # type: ignore[misc]
     @cached_property
@@ -146,19 +189,19 @@ class DocSerializer(BaseModel, BaseDocSerializer):
         return refs
 
     @override
-    def get_excluded_refs(self, **kwargs) -> list[str]:
+    def get_excluded_refs(self, **kwargs) -> set[str]:
         """References to excluded items."""
         params = self.params.merge_with_patch(patch=kwargs)
         params_json = params.model_dump_json()
         refs = self._excluded_refs_cache.get(params_json)
         if refs is None:
-            refs = [
+            refs = {
                 item.self_ref
-                for ix, (item, _) in enumerate(
-                    self.doc.iterate_items(
-                        with_groups=True,
+                for ix, item in enumerate(
+                    _iterate_items(
+                        doc=self.doc,
                         traverse_pictures=True,
-                        included_content_layers=params.layers,
+                        layers=params.layers,
                     )
                 )
                 if (
@@ -178,64 +221,21 @@ class DocSerializer(BaseModel, BaseDocSerializer):
                         )
                     )
                 )
-            ]
+            }
             self._excluded_refs_cache[params_json] = refs
         return refs
 
-    @abstractmethod
-    def serialize_page(
-        self, *, parts: list[SerializationResult], **kwargs
-    ) -> SerializationResult:
-        """Serialize a page out of its parts."""
-        ...
-
     @abstractmethod
     def serialize_doc(
-        self, *, pages: dict[Optional[int], SerializationResult], **kwargs
+        self, *, parts: list[SerializationResult], **kwargs
     ) -> SerializationResult:
         """Serialize a document out of its pages."""
         ...
 
     def _serialize_body(self) -> SerializationResult:
         """Serialize the document body."""
-        # find page ranges if available; otherwise regard whole doc as a single page
-        prev_start: int = 0
-        prev_page_nr: Optional[int] = None
-        range_by_page_nr: dict[Optional[int], tuple[int, int]] = {}
-
-        for ix, (item, _) in enumerate(
-            self.doc.iterate_items(
-                with_groups=True,
-                traverse_pictures=True,
-                included_content_layers=self.params.layers,
-            )
-        ):
-            if isinstance(item, DocItem):
-                if item.prov:
-                    page_no = item.prov[0].page_no
-                    if prev_page_nr is None or page_no > prev_page_nr:
-                        if prev_page_nr is not None:  # close previous range
-                            range_by_page_nr[prev_page_nr] = (prev_start, ix)
-
-                        prev_start = ix
-                        # could alternatively always start 1st page from 0:
-                        # prev_start = ix if prev_page_nr is not None else 0
-
-                        prev_page_nr = page_no
-
-        # close last (and single if no pages) range
-        range_by_page_nr[prev_page_nr] = (prev_start, sys.maxsize)
-
-        page_results: dict[Optional[int], SerializationResult] = {}
-        for page_nr in range_by_page_nr:
-            page_range = range_by_page_nr[page_nr]
-            params_to_pass = deepcopy(self.params)
-            params_to_pass.start_idx = page_range[0]
-            params_to_pass.stop_idx = page_range[1]
-            subparts = self.get_parts(**params_to_pass.model_dump())
-            page_res = self.serialize_page(parts=subparts)
-            page_results[page_nr] = page_res
-        res = self.serialize_doc(pages=page_results)
+        subparts = self.get_parts()
+        res = self.serialize_doc(parts=subparts)
         return res
 
     @override
@@ -331,6 +331,11 @@ class DocSerializer(BaseModel, BaseDocSerializer):
                 doc=self.doc,
                 **my_kwargs,
             )
+        elif isinstance(item, _PageBreakNode):
+            part = _PageBreakSerResult(
+                text=self._create_page_break(node=item),
+                node=item,
+            )
         else:
             part = self.fallback_serializer.serialize(
                 item=item,
@@ -356,18 +361,19 @@ class DocSerializer(BaseModel, BaseDocSerializer):
         parts: list[SerializationResult] = []
         my_visited: set[str] = visited if visited is not None else set()
         params = self.params.merge_with_patch(patch=kwargs)
-        for item, _ in self.doc.iterate_items(
-            root=item,
-            with_groups=True,
-            traverse_pictures=traverse_pictures,
-            included_content_layers=params.layers,
+
+        for node in _iterate_items(
+            node=item,
+            doc=self.doc,
+            layers=params.layers,
+            add_page_breaks=self.requires_page_break(),
         ):
-            if item.self_ref in my_visited:
+            if node.self_ref in my_visited:
                 continue
             else:
-                my_visited.add(item.self_ref)
+                my_visited.add(node.self_ref)
             part = self.serialize(
-                item=item,
+                item=node,
                 list_level=list_level,
                 is_inline_scope=is_inline_scope,
                 visited=my_visited,
@@ -450,3 +456,38 @@ class DocSerializer(BaseModel, BaseDocSerializer):
         else:
             text_res = ""
         return create_ser_result(text=text_res, span_source=results)
+
+    def _get_applicable_pages(self) -> Optional[list[int]]:
+        pages = {
+            item.prov[0].page_no: ...
+            for ix, (item, _) in enumerate(
+                self.doc.iterate_items(
+                    with_groups=True,
+                    included_content_layers=self.params.layers,
+                    traverse_pictures=True,
+                )
+            )
+            if (
+                isinstance(item, DocItem)
+                and item.prov
+                and (
+                    self.params.pages is None
+                    or item.prov[0].page_no in self.params.pages
+                )
+                and ix >= self.params.start_idx
+                and ix < self.params.stop_idx
+            )
+        }
+        return [p for p in pages] or None
+
+    def _create_page_break(self, node: _PageBreakNode) -> str:
+        return f"#_#_DOCLING_DOC_PAGE_BREAK_{node.prev_page}_{node.next_page}_#_#"
+
+    def _get_page_breaks(self, text: str) -> Iterable[Tuple[str, int, int]]:
+        pattern = r"#_#_DOCLING_DOC_PAGE_BREAK_(\d+)_(\d+)_#_#"
+        matches = re.finditer(pattern, text)
+        for match in matches:
+            full_match = match.group(0)
+            prev_page_nr = int(match.group(1))
+            next_page_nr = int(match.group(2))
+            yield (full_match, prev_page_nr, next_page_nr)

docling_core/experimental/serializer/doctags.py

@@ -476,28 +476,21 @@ class DocTagsDocSerializer(DocSerializer):
     params: DocTagsParams = DocTagsParams()
 
     @override
-    def serialize_page(
+    def serialize_doc(
         self, *, parts: list[SerializationResult], **kwargs
     ) -> SerializationResult:
-        """Serialize a page out of its parts."""
+        """Serialize a document out of its pages."""
         delim = _get_delim(params=self.params)
         text_res = delim.join([p.text for p in parts if p.text])
-        return create_ser_result(text=text_res, span_source=parts)
 
-    @override
-    def serialize_doc(
-        self, *, pages: dict[Optional[int], SerializationResult], **kwargs
-    ) -> SerializationResult:
-        """Serialize a document out of its pages."""
-        delim = _get_delim(params=self.params)
         if self.params.add_page_break:
-            page_sep = f"{delim}<{DocumentToken.PAGE_BREAK.value}>{delim}"
-            content = page_sep.join([text for k in pages if (text := pages[k].text)])
-        else:
-            content = self.serialize_page(parts=list(pages.values())).text
+            page_sep = f"<{DocumentToken.PAGE_BREAK.value}>"
+            for full_match, _, _ in self._get_page_breaks(text=text_res):
+                text_res = text_res.replace(full_match, page_sep)
+
         wrap_tag = DocumentToken.DOCUMENT.value
-        text_res = f"<{wrap_tag}>{content}{delim}</{wrap_tag}>"
-        return create_ser_result(text=text_res, span_source=list(pages.values()))
+        text_res = f"<{wrap_tag}>{text_res}{delim}</{wrap_tag}>"
+        return create_ser_result(text=text_res, span_source=parts)
 
     @override
     def serialize_captions(
@@ -526,3 +519,8 @@ class DocTagsDocSerializer(DocSerializer):
         if text_res:
             text_res = _wrap(text=text_res, wrap_tag=DocumentToken.CAPTION.value)
         return create_ser_result(text=text_res, span_source=results)
+
+    @override
+    def requires_page_break(self):
+        """Whether to add page breaks."""
+        return self.params.add_page_break

docling_core/experimental/serializer/html.py

@@ -192,7 +192,7 @@ class HTMLTextSerializer(BaseModel, BaseTextSerializer):
         is_inline_scope: bool,
     ) -> str:
         code_text = self._prepare_content(
-            item.text, do_escape_html=True, do_replace_newline=False
+            item.text, do_escape_html=False, do_replace_newline=False
         )
         if is_inline_scope:
             text = f"<code>{code_text}</code>"
@@ -765,37 +765,42 @@ class HTMLDocSerializer(DocSerializer):
         """Apply HTML-specific hyperlink serialization."""
         return f'<a href="{str(hyperlink)}">{text}</a>'
 
-    @override
-    def serialize_page(
-        self, parts: list[SerializationResult], **kwargs
-    ) -> SerializationResult:
-        """Serialize a page out of its parts."""
-        # Join all parts with newlines
-        body_content = "\n".join([p.text for p in parts if p.text])
-        return create_ser_result(
-            text=f"<div class='page'>\n{body_content}\n</div>",
-            span_source=parts,
-        )
-
     @override
     def serialize_doc(
-        self, pages: dict[Optional[int], SerializationResult], **kwargs
+        self, parts: list[SerializationResult], **kwargs
     ) -> SerializationResult:
         """Serialize a document out of its pages."""
         # Create HTML structure
         html_parts = [
             "<!DOCTYPE html>",
+            "<html>",
             self._generate_head(),
             "<body>",
         ]
 
         if self.params.output_style == HTMLOutputStyle.SPLIT_PAGE:
+            html_content = "\n".join([p.text for p in parts if p.text])
+            next_page: Optional[int] = None
+            prev_full_match_end = 0
+            pages = {}
+            for full_match, prev_page, next_page in self._get_page_breaks(html_content):
+                this_match_start = html_content.find(full_match)
+                pages[prev_page] = html_content[prev_full_match_end:this_match_start]
+                prev_full_match_end = this_match_start + len(full_match)
+
+            # capture last page
+            if next_page is not None:
+                pages[next_page] = html_content[prev_full_match_end:]
+
             html_parts.append("<table>")
             html_parts.append("<tbody>")
 
+            applicable_pages = self._get_applicable_pages()
             for page_no, page in pages.items():
 
                 if isinstance(page_no, int):
+                    if applicable_pages is not None and page_no not in applicable_pages:
+                        continue
                     page_img = self.doc.pages[page_no].image
 
                     html_parts.append("<tr>")
@@ -831,7 +836,7 @@ class HTMLDocSerializer(DocSerializer):
                     html_parts.append("</td>")
 
                     html_parts.append("<td>")
-                    html_parts.append(page.text)
+                    html_parts.append(f"<div class='page'>\n{page}\n</div>")
                     html_parts.append("</td>")
 
                     html_parts.append("</tr>")
@@ -845,9 +850,9 @@ class HTMLDocSerializer(DocSerializer):
 
         elif self.params.output_style == HTMLOutputStyle.SINGLE_COLUMN:
             # Add all pages
-            for page_no, page in pages.items():
-                if page.text:
-                    html_parts.append(page.text)
+            html_content = "\n".join([p.text for p in parts if p.text])
+            html_content = f"<div class='page'>\n{html_content}\n</div>"
+            html_parts.append(html_content)
         else:
             raise ValueError(f"unknown output-style: {self.params.output_style}")
 
@@ -857,7 +862,7 @@ class HTMLDocSerializer(DocSerializer):
         # Join with newlines
         html_content = "\n".join(html_parts)
 
-        return create_ser_result(text=html_content, span_source=list(pages.values()))
+        return create_ser_result(text=html_content, span_source=parts)
 
     @override
     def serialize_captions(
@@ -929,3 +934,8 @@ class HTMLDocSerializer(DocSerializer):
     def _get_default_css(self) -> str:
         """Return default CSS styles for the HTML document."""
         return "<style></style>"
+
+    @override
+    def requires_page_break(self):
+        """Whether to add page breaks."""
+        return self.params.output_style == HTMLOutputStyle.SPLIT_PAGE

docling_core/experimental/serializer/markdown.py

@@ -29,6 +29,7 @@ from docling_core.experimental.serializer.base import (
 from docling_core.experimental.serializer.common import (
     CommonParams,
     DocSerializer,
+    _PageBreakSerResult,
     create_ser_result,
 )
 from docling_core.types.doc.base import ImageRefMode
@@ -375,7 +376,11 @@ class MarkdownListSerializer(BaseModel, BaseListSerializer):
                 (
                     c.text
                     if c.text and c.text[0] == " "
-                    else f"{indent_str}{f'{i + 1}.' if is_ol else '-'} {c.text}"
+                    else (
+                        f"{indent_str}"
+                        f"{'' if isinstance(c, _PageBreakSerResult) else (f'{i + 1}. ' if is_ol else '- ')}"  # noqa: E501
+                        f"{c.text}"
+                    )
                 )
                 for i, c in enumerate(my_parts)
             ]
@@ -404,6 +409,7 @@ class MarkdownInlineSerializer(BaseInlineSerializer):
             list_level=list_level,
             is_inline_scope=True,
             visited=my_visited,
+            **kwargs,
         )
         text_res = " ".join([p.text for p in parts if p.text])
         return create_ser_result(text=text_res, span_source=parts)
@@ -516,21 +522,19 @@ class MarkdownDocSerializer(DocSerializer):
         return res
 
     @override
-    def serialize_page(
+    def serialize_doc(
         self, *, parts: list[SerializationResult], **kwargs
     ) -> SerializationResult:
-        """Serialize a page out of its parts."""
+        """Serialize a document out of its parts."""
         text_res = "\n\n".join([p.text for p in parts if p.text])
+        if self.params.page_break_placeholder:
+            page_sep = self.params.page_break_placeholder or ""
+            for full_match, _, _ in self._get_page_breaks(text=text_res):
+                text_res = text_res.replace(full_match, page_sep)
+
         return create_ser_result(text=text_res, span_source=parts)
 
     @override
-    def serialize_doc(
-        self, *, pages: dict[Optional[int], SerializationResult], **kwargs
-    ) -> SerializationResult:
-        """Serialize a document out of its pages."""
-        if self.params.page_break_placeholder is not None:
-            sep = f"\n\n{self.params.page_break_placeholder}\n\n"
-            text_res = sep.join([text for k in pages if (text := pages[k].text)])
-            return create_ser_result(text=text_res, span_source=list(pages.values()))
-        else:
-            return self.serialize_page(parts=list(pages.values()))
+    def requires_page_break(self):
+        """Whether to add page breaks."""
+        return self.params.page_break_placeholder is not None

{docling_core-2.26.0.dist-info → docling_core-2.26.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: docling-core
-Version: 2.26.0
+Version: 2.26.2
 Summary: A python library to define and validate data types in Docling.
 Home-page: https://github.com/docling-project
 License: MIT

{docling_core-2.26.0.dist-info → docling_core-2.26.2.dist-info}/RECORD

@@ -3,12 +3,12 @@ docling_core/cli/__init__.py,sha256=C63yWifzpA0IV7YWDatpAdrhoV8zjqxAKv0xMf09VdM,
 docling_core/cli/view.py,sha256=gwxSBYhGqwznMR8pdXaEuAh2bjFD5X_g11xFYSgFgtM,1764
 docling_core/experimental/__init__.py,sha256=XnAVSUHbA6OFhNSpoYqSD3u83-xVaUaki1DIKFw69Ew,99
 docling_core/experimental/serializer/__init__.py,sha256=CECQlMoCDUxkg4RAUdC3itA3I3qFhKhe2HcYghN6_xw,105
-docling_core/experimental/serializer/base.py,sha256=WwTBNGlaSfy2KBwHFmZ9c5o4Hj27bGmjnnNWD0TEXqo,5784
-docling_core/experimental/serializer/common.py,sha256=KoiPCzfSBrHSrXwAOAOL-ISPBRoJinzGS2ojK5JVH8c,15266
-docling_core/experimental/serializer/doctags.py,sha256=r_b9dUsfQqucK8TtmE_e5n5NAMcCGfBsEfjpnP-3evM,18028
-docling_core/experimental/serializer/html.py,sha256=7PQHa1T1fctQocvwrOosjS3K44Tjh7FpcCBWZoZa3r8,31260
+docling_core/experimental/serializer/base.py,sha256=1sD1v5rWC4MT_Y6BWpMDjUAwuEqC0TR9YjQJZlhPt50,5901
+docling_core/experimental/serializer/common.py,sha256=z80B2BzUdDfp_HgZ1KA64vK-oV07jcgxLs1XyCgp7sI,16152
+docling_core/experimental/serializer/doctags.py,sha256=e97FJHh77x--g2t1O2YprBzF8lkihn_xOr59EjnR7ag,17794
+docling_core/experimental/serializer/html.py,sha256=C-xf_PH28R6DoISFDoZwBK67rsfFmTemgZ_VW2i0haY,31896
 docling_core/experimental/serializer/html_styles.py,sha256=-jBwS4EU7yfKoz0GSoxhwx90OmIKieO6TwPw57IuxcA,4692
-docling_core/experimental/serializer/markdown.py,sha256=wcF75hZFBQxIJKrd_1-DRuVN3g1ofQGHTrb11pUQdUw,17680
+docling_core/experimental/serializer/markdown.py,sha256=5bvONhaA1EdAD0c3WlWfr2x2KmRaSZd8muG-91XVHgc,17733
 docling_core/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 docling_core/resources/schemas/doc/ANN.json,sha256=04U5j-PU9m5w7IagJ_rHcAx7qUtLkUuaWZO9GuYHnTA,4202
 docling_core/resources/schemas/doc/DOC.json,sha256=9tVKpCqDGGq3074Nn5qlUCdTN-5k1Q0ri_scJblwnLE,6686
@@ -65,8 +65,8 @@ docling_core/utils/generate_jsonschema.py,sha256=uNX1O5XnjyB5nA66XqZXTt3YbGuR2ty
 docling_core/utils/legacy.py,sha256=SqNQAxl97aHfoJEsC9vZcMJg5FNkmqKPFi-wdSrnfI0,24442
 docling_core/utils/validate.py,sha256=aQ11UbFyl8iD_N7yTTZmm_VVeXz8KcCyn3GLXgkfYRM,2049
 docling_core/utils/validators.py,sha256=azcrndLzhNkTWnbFSu9shJ5D3j_znnLrIFA5R8hzmGU,2798
-docling_core-2.26.0.dist-info/LICENSE,sha256=2M9-6EoQ1sxFztTOkXGAtwUDJvnWaAHdB9BYWVwGkIw,1087
-docling_core-2.26.0.dist-info/METADATA,sha256=_GeilTNKmhnT5woL2myAwHd6KjP29rlBb9C8Ed8_WJ8,5843
-docling_core-2.26.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-docling_core-2.26.0.dist-info/entry_points.txt,sha256=oClcdb2L2RKx4jdqUykY16Kum_f0_whwWhGzIodyidc,216
-docling_core-2.26.0.dist-info/RECORD,,
+docling_core-2.26.2.dist-info/LICENSE,sha256=2M9-6EoQ1sxFztTOkXGAtwUDJvnWaAHdB9BYWVwGkIw,1087
+docling_core-2.26.2.dist-info/METADATA,sha256=motwJC3ixvU3oYstkDMsu5YmnHryDzuFfn3udzpDGrY,5843
+docling_core-2.26.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+docling_core-2.26.2.dist-info/entry_points.txt,sha256=oClcdb2L2RKx4jdqUykY16Kum_f0_whwWhGzIodyidc,216
+docling_core-2.26.2.dist-info/RECORD,,