docling-core 2.42.0__py3-none-any.whl → 2.43.0__py3-none-any.whl

docling_core/transforms/chunker/__init__.py

@@ -11,3 +11,4 @@ from docling_core.transforms.chunker.hierarchical_chunker import (
     DocMeta,
     HierarchicalChunker,
 )
+from docling_core.transforms.chunker.page_chunker import PageChunker

docling_core/transforms/chunker/page_chunker.py

@@ -0,0 +1,59 @@
+"""Page-based chunker implementation: each chunk corresponds to a single page."""
+
+from __future__ import annotations
+
+from typing import Any, Iterator
+
+from pydantic import ConfigDict
+from typing_extensions import override
+
+from docling_core.transforms.chunker import BaseChunker, DocChunk, DocMeta
+from docling_core.transforms.chunker.hierarchical_chunker import (
+    ChunkingSerializerProvider,
+)
+from docling_core.types import DoclingDocument as DLDocument
+
+
+class PageChunker(BaseChunker):
+    r"""Chunker implementation that yields one chunk per page."""
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    serializer_provider: ChunkingSerializerProvider = ChunkingSerializerProvider()
+
+    @override
+    def chunk(
+        self,
+        dl_doc: DLDocument,
+        **kwargs: Any,
+    ) -> Iterator[DocChunk]:
+        """Chunk the provided document by page."""
+        my_doc_ser = self.serializer_provider.get_serializer(doc=dl_doc)
+        if dl_doc.pages:
+            # chunk by page
+            for page_no in sorted(dl_doc.pages.keys()):
+                ser_res = my_doc_ser.serialize(pages={page_no})
+                if not ser_res.text:
+                    continue
+                yield DocChunk(
+                    text=ser_res.text,
+                    meta=DocMeta(
+                        doc_items=ser_res.get_unique_doc_items(),
+                        headings=None,
+                        captions=None,
+                        origin=dl_doc.origin,
+                    ),
+                )
+        else:
+            # if no pages, treat whole document as single chunk
+            ser_res = my_doc_ser.serialize()
+            if ser_res.text:
+                yield DocChunk(
+                    text=ser_res.text,
+                    meta=DocMeta(
+                        doc_items=ser_res.get_unique_doc_items(),
+                        headings=None,
+                        captions=None,
+                        origin=dl_doc.origin,
+                    ),
+                )

docling_core/transforms/serializer/base.py

@@ -39,6 +39,16 @@ class SerializationResult(BaseModel):
     spans: list[Span] = []
     # group: Optional[GroupItem] = None  # set when result reflects specific group item
 
+    def get_unique_doc_items(self) -> list[DocItem]:
+        """Get the doc items corresponding to this result."""
+        seen_doc_item_refs: set[str] = set()
+        doc_items: list[DocItem] = []
+        for span in self.spans:
+            if span.item.self_ref not in seen_doc_item_refs:
+                seen_doc_item_refs.add(span.item.self_ref)
+                doc_items.append(span.item)
+        return doc_items
+
 
 class BaseTextSerializer(ABC):
     """Base class for text item serializers."""

docling_core/transforms/serializer/common.py

@@ -285,7 +285,7 @@ class DocSerializer(BaseModel, BaseDocSerializer):
 
     def _serialize_body(self, **kwargs) -> SerializationResult:
         """Serialize the document body."""
-        subparts = self.get_parts()
+        subparts = self.get_parts(**kwargs)
         res = self.serialize_doc(parts=subparts, **kwargs)
         return res
 

docling_core/types/doc/document.py

@@ -4098,7 +4098,10 @@ class DoclingDocument(BaseModel):
         return result
 
     def _with_pictures_refs(
-        self, image_dir: Path, reference_path: Optional[Path] = None
+        self,
+        image_dir: Path,
+        page_no: Optional[int],
+        reference_path: Optional[Path] = None,
     ) -> "DoclingDocument":
         """Document with images as refs.
 
@@ -4111,7 +4114,7 @@ class DoclingDocument(BaseModel):
         image_dir.mkdir(parents=True, exist_ok=True)
 
         if image_dir.is_dir():
-            for item, level in result.iterate_items(with_groups=False):
+            for item, level in result.iterate_items(page_no=page_no, with_groups=False):
                 if isinstance(item, PictureItem):
 
                     if (
@@ -4211,7 +4214,7 @@ class DoclingDocument(BaseModel):
             os.makedirs(artifacts_dir, exist_ok=True)
 
         new_doc = self._make_copy_with_refmode(
-            artifacts_dir, image_mode, reference_path=reference_path
+            artifacts_dir, image_mode, page_no=None, reference_path=reference_path
         )
 
         out = new_doc.export_to_dict(
@@ -4254,7 +4257,7 @@ class DoclingDocument(BaseModel):
             os.makedirs(artifacts_dir, exist_ok=True)
 
         new_doc = self._make_copy_with_refmode(
-            artifacts_dir, image_mode, reference_path=reference_path
+            artifacts_dir, image_mode, page_no=None, reference_path=reference_path
         )
 
         out = new_doc.export_to_dict(
@@ -4327,7 +4330,7 @@ class DoclingDocument(BaseModel):
             os.makedirs(artifacts_dir, exist_ok=True)
 
         new_doc = self._make_copy_with_refmode(
-            artifacts_dir, image_mode, reference_path=reference_path
+            artifacts_dir, image_mode, page_no, reference_path=reference_path
         )
 
         md_out = new_doc.export_to_markdown(
@@ -4503,7 +4506,7 @@ class DoclingDocument(BaseModel):
             os.makedirs(artifacts_dir, exist_ok=True)
 
         new_doc = self._make_copy_with_refmode(
-            artifacts_dir, image_mode, reference_path=reference_path
+            artifacts_dir, image_mode, page_no, reference_path=reference_path
         )
 
         html_out = new_doc.export_to_html(
@@ -4542,6 +4545,7 @@ class DoclingDocument(BaseModel):
         self,
         artifacts_dir: Path,
         image_mode: ImageRefMode,
+        page_no: Optional[int],
         reference_path: Optional[Path] = None,
     ):
         new_doc = None
@@ -4549,7 +4553,7 @@ class DoclingDocument(BaseModel):
             new_doc = self
         elif image_mode == ImageRefMode.REFERENCED:
             new_doc = self._with_pictures_refs(
-                image_dir=artifacts_dir, reference_path=reference_path
+                image_dir=artifacts_dir, page_no=page_no, reference_path=reference_path
             )
         elif image_mode == ImageRefMode.EMBEDDED:
             new_doc = self._with_embedded_pictures()

{docling_core-2.42.0.dist-info → docling_core-2.43.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docling-core
-Version: 2.42.0
+Version: 2.43.0
 Summary: A python library to define and validate data types in Docling.
 Author-email: Cesar Berrospi Ramis <ceb@zurich.ibm.com>, Panos Vagenas <pva@zurich.ibm.com>, Michele Dolfi <dol@zurich.ibm.com>, Christoph Auer <cau@zurich.ibm.com>, Peter Staar <taa@zurich.ibm.com>
 Maintainer-email: Panos Vagenas <pva@zurich.ibm.com>, Michele Dolfi <dol@zurich.ibm.com>, Christoph Auer <cau@zurich.ibm.com>, Peter Staar <taa@zurich.ibm.com>, Cesar Berrospi Ramis <ceb@zurich.ibm.com>

{docling_core-2.42.0.dist-info → docling_core-2.43.0.dist-info}/RECORD

@@ -17,17 +17,18 @@ docling_core/search/mapping.py,sha256=6rqG7LgYSeWmooKNEcRa5gFDLp1ZdzPqDGlwTA5gpO
 docling_core/search/meta.py,sha256=wSurrsqdP1N3gQKx027fVdzVmc33a7Y6rPl-FClQvtA,3318
 docling_core/search/package.py,sha256=Lz2ml2eDy5t0ZimnGTq-DXHAn-f18w0bn4H5xrhs75A,1841
 docling_core/transforms/__init__.py,sha256=P81y_oqkiTN4Ld5crh1gQ6BbHqqR6C6nBt9ACDd57ds,106
-docling_core/transforms/chunker/__init__.py,sha256=YdizSKXLmmK9eyYBsarHWr8Mx_AoA0PT7c0absibZMk,306
+docling_core/transforms/chunker/__init__.py,sha256=Qg5RhC-2QqdXKEfjzNGJaVi0NqBCL3xAhKWJGOlrE3M,375
 docling_core/transforms/chunker/base.py,sha256=kJaRrGQynglG9wpy0IaAYTf4MKheWH5BAPzx4LE9yIg,2824
 docling_core/transforms/chunker/hierarchical_chunker.py,sha256=uDf-qGiIT_4JUEg9NOdzvDqAPOTqycKJ-jEpDkV3jJU,8243
 docling_core/transforms/chunker/hybrid_chunker.py,sha256=xjkz8hy3tXXzkJzf7QMFOEq_v8V7Jcs9tCY0Mxjge74,12548
+docling_core/transforms/chunker/page_chunker.py,sha256=gLUlqA_klK-rkuPVYuJKi3ZuTIGdd2HD7ces72AiZ2U,2018
 docling_core/transforms/chunker/tokenizer/__init__.py,sha256=-bhXOTpoI7SYk7vn47z8Ek-RZFjJk4TfZawxsFuNHnE,34
 docling_core/transforms/chunker/tokenizer/base.py,sha256=2gOBQPYJYC0iWXOgMG3DiNP7xEBtii7DYcib0iECq5o,575
 docling_core/transforms/chunker/tokenizer/huggingface.py,sha256=aZ_RNQIzcNkAHGHZw3SBCoqJHM2Ihb65eiM29O9BR6o,2506
 docling_core/transforms/chunker/tokenizer/openai.py,sha256=zt2kwcC-r8MafeEG0CESab8E4RIC9aaFXxxnxOGyTMA,918
 docling_core/transforms/serializer/__init__.py,sha256=CECQlMoCDUxkg4RAUdC3itA3I3qFhKhe2HcYghN6_xw,105
-docling_core/transforms/serializer/base.py,sha256=s3Anl_3-QJM1t29Bz-iOgLhAcfG3BZuwZqdYTi5Xfr0,6846
-docling_core/transforms/serializer/common.py,sha256=Dkw9axJqU2qlZuEFRDa6Av11PIL2ejOOOCAahtoK9sA,19106
+docling_core/transforms/serializer/base.py,sha256=TI8Epj7gyxdTet9j-Rs4o5U09gfACfAIVoirlschviM,7266
+docling_core/transforms/serializer/common.py,sha256=0TNEGoA_rJ-qkVYp-X8SMUr3jTrbf6TRzPzwufYh5JM,19114
 docling_core/transforms/serializer/doctags.py,sha256=TD0yAm1qSVy-GsE6svpUAI-Yqjcf2rrTZ3ac9YU3gbE,19858
 docling_core/transforms/serializer/html.py,sha256=oxnUhszRPBINiK1tq2dwf5QjTCrIV_q15vsrPVqBeME,38988
 docling_core/transforms/serializer/html_styles.py,sha256=-jBwS4EU7yfKoz0GSoxhwx90OmIKieO6TwPw57IuxcA,4692
@@ -41,7 +42,7 @@ docling_core/types/__init__.py,sha256=MVRSgsk5focwGyAplh_TRR3dEecIXpd98g_u3zZ5HX
 docling_core/types/base.py,sha256=PusJskRVL19y-hq0BgXr5e8--QEqSqLnFNJ8UbOqW88,8318
 docling_core/types/doc/__init__.py,sha256=8hOhm5W9mArf3zwgfoMxDs1pHizhLFSAZlLu1tPBBRk,1641
 docling_core/types/doc/base.py,sha256=i98y4IF250adR-8BSS374K90fwfwG-vBfWh14tLC5Cs,15906
-docling_core/types/doc/document.py,sha256=ShxqS9A9N1KhVS73C7eoTBo0WgCI-TRbjwW8frhFEns,199154
+docling_core/types/doc/document.py,sha256=SUqIJ-huO3ELLRdCMUYjkkXHeGGkeY2oaOWFQ1nq5lg,199315
 docling_core/types/doc/labels.py,sha256=-W1-LW6z0J9F9ExJqR0Wd1WeqWTaY3Unm-j1UkQGlC4,7330
 docling_core/types/doc/page.py,sha256=35h1xdtCM3-AaN8Dim9jDseZIiw-3GxpB-ofF-H2rQQ,41878
 docling_core/types/doc/tokens.py,sha256=z22l9J81_sg9CYMvOuLmPuLsNT7h_s7wao2UT89DvI8,9278
@@ -74,9 +75,9 @@ docling_core/utils/generate_jsonschema.py,sha256=uNX1O5XnjyB5nA66XqZXTt3YbGuR2ty
 docling_core/utils/legacy.py,sha256=5lghO48OEcV9V51tRnH3YSKgLtdqhr-Q5C_OcJZ8TOs,24392
 docling_core/utils/validate.py,sha256=aQ11UbFyl8iD_N7yTTZmm_VVeXz8KcCyn3GLXgkfYRM,2049
 docling_core/utils/validators.py,sha256=azcrndLzhNkTWnbFSu9shJ5D3j_znnLrIFA5R8hzmGU,2798
-docling_core-2.42.0.dist-info/licenses/LICENSE,sha256=2M9-6EoQ1sxFztTOkXGAtwUDJvnWaAHdB9BYWVwGkIw,1087
-docling_core-2.42.0.dist-info/METADATA,sha256=R7dpA43x-3YVrXztgu3_Pt87SWRrjaeBW0jXPndawdw,6453
-docling_core-2.42.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-docling_core-2.42.0.dist-info/entry_points.txt,sha256=ER4zROQWkFMHIrY-oqY5E4HeCcCIg8dLkNztYGxdb7c,59
-docling_core-2.42.0.dist-info/top_level.txt,sha256=O-tcXpGiurlud-1ZxMq1b-OmrfAVA4sajcgWU32RtfA,13
-docling_core-2.42.0.dist-info/RECORD,,
+docling_core-2.43.0.dist-info/licenses/LICENSE,sha256=2M9-6EoQ1sxFztTOkXGAtwUDJvnWaAHdB9BYWVwGkIw,1087
+docling_core-2.43.0.dist-info/METADATA,sha256=4aQkJub9YsHu2OrYq4Y2vIlBYHcOCC-ruxCN6EnWbW4,6453
+docling_core-2.43.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+docling_core-2.43.0.dist-info/entry_points.txt,sha256=ER4zROQWkFMHIrY-oqY5E4HeCcCIg8dLkNztYGxdb7c,59
+docling_core-2.43.0.dist-info/top_level.txt,sha256=O-tcXpGiurlud-1ZxMq1b-OmrfAVA4sajcgWU32RtfA,13
+docling_core-2.43.0.dist-info/RECORD,,