docling-core 2.42.0__tar.gz → 2.43.1__tar.gz

{docling_core-2.42.0 → docling_core-2.43.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docling-core
-Version: 2.42.0
+Version: 2.43.1
 Summary: A python library to define and validate data types in Docling.
 Author-email: Cesar Berrospi Ramis <ceb@zurich.ibm.com>, Panos Vagenas <pva@zurich.ibm.com>, Michele Dolfi <dol@zurich.ibm.com>, Christoph Auer <cau@zurich.ibm.com>, Peter Staar <taa@zurich.ibm.com>
 Maintainer-email: Panos Vagenas <pva@zurich.ibm.com>, Michele Dolfi <dol@zurich.ibm.com>, Christoph Auer <cau@zurich.ibm.com>, Peter Staar <taa@zurich.ibm.com>, Cesar Berrospi Ramis <ceb@zurich.ibm.com>

{docling_core-2.42.0 → docling_core-2.43.1}/docling_core/transforms/chunker/__init__.py

@@ -11,3 +11,4 @@ from docling_core.transforms.chunker.hierarchical_chunker import (
     DocMeta,
     HierarchicalChunker,
 )
+from docling_core.transforms.chunker.page_chunker import PageChunker

docling_core-2.43.1/docling_core/transforms/chunker/page_chunker.py

@@ -0,0 +1,59 @@
+"""Page-based chunker implementation: each chunk corresponds to a single page."""
+
+from __future__ import annotations
+
+from typing import Any, Iterator
+
+from pydantic import ConfigDict
+from typing_extensions import override
+
+from docling_core.transforms.chunker import BaseChunker, DocChunk, DocMeta
+from docling_core.transforms.chunker.hierarchical_chunker import (
+    ChunkingSerializerProvider,
+)
+from docling_core.types import DoclingDocument as DLDocument
+
+
+class PageChunker(BaseChunker):
+    r"""Chunker implementation that yields one chunk per page."""
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    serializer_provider: ChunkingSerializerProvider = ChunkingSerializerProvider()
+
+    @override
+    def chunk(
+        self,
+        dl_doc: DLDocument,
+        **kwargs: Any,
+    ) -> Iterator[DocChunk]:
+        """Chunk the provided document by page."""
+        my_doc_ser = self.serializer_provider.get_serializer(doc=dl_doc)
+        if dl_doc.pages:
+            # chunk by page
+            for page_no in sorted(dl_doc.pages.keys()):
+                ser_res = my_doc_ser.serialize(pages={page_no})
+                if not ser_res.text:
+                    continue
+                yield DocChunk(
+                    text=ser_res.text,
+                    meta=DocMeta(
+                        doc_items=ser_res.get_unique_doc_items(),
+                        headings=None,
+                        captions=None,
+                        origin=dl_doc.origin,
+                    ),
+                )
+        else:
+            # if no pages, treat whole document as single chunk
+            ser_res = my_doc_ser.serialize()
+            if ser_res.text:
+                yield DocChunk(
+                    text=ser_res.text,
+                    meta=DocMeta(
+                        doc_items=ser_res.get_unique_doc_items(),
+                        headings=None,
+                        captions=None,
+                        origin=dl_doc.origin,
+                    ),
+                )

{docling_core-2.42.0 → docling_core-2.43.1}/docling_core/transforms/serializer/base.py

@@ -39,6 +39,16 @@ class SerializationResult(BaseModel):
     spans: list[Span] = []
     # group: Optional[GroupItem] = None  # set when result reflects specific group item
 
+    def get_unique_doc_items(self) -> list[DocItem]:
+        """Get the doc items corresponding to this result."""
+        seen_doc_item_refs: set[str] = set()
+        doc_items: list[DocItem] = []
+        for span in self.spans:
+            if span.item.self_ref not in seen_doc_item_refs:
+                seen_doc_item_refs.add(span.item.self_ref)
+                doc_items.append(span.item)
+        return doc_items
+
 
 class BaseTextSerializer(ABC):
     """Base class for text item serializers."""

{docling_core-2.42.0 → docling_core-2.43.1}/docling_core/transforms/serializer/common.py

@@ -285,7 +285,7 @@ class DocSerializer(BaseModel, BaseDocSerializer):
 
     def _serialize_body(self, **kwargs) -> SerializationResult:
         """Serialize the document body."""
-        subparts = self.get_parts()
+        subparts = self.get_parts(**kwargs)
         res = self.serialize_doc(parts=subparts, **kwargs)
         return res
 

{docling_core-2.42.0 → docling_core-2.43.1}/docling_core/transforms/serializer/html.py

@@ -713,6 +713,23 @@ class HTMLListSerializer(BaseModel, BaseListSerializer):
             **kwargs,
         )
 
+        # Append nested list to parent list item:
+        i = 0
+        while i < len(parts):
+            prt = parts[i]
+            if prt.text.startswith(("<ul>", "<ol>")):
+                for j in range(i - 1, -1, -1):
+                    if parts[j].text.startswith(("<li>", "<li ")) and parts[
+                        j
+                    ].text.endswith("</li>"):
+                        before, _, _ = parts[j].text.rpartition("</li>")
+                        parts[j].text = f"{before}\n{prt.text}\n</li>"
+                        break
+                if j > -1:
+                    parts.pop(i)
+            else:
+                i += 1
+
         # Add all child parts
         text_res = "\n".join(
             [

{docling_core-2.42.0 → docling_core-2.43.1}/docling_core/transforms/visualizer/layout_visualizer.py

@@ -148,7 +148,9 @@ class LayoutVisualizer(BaseVisualizer):
         prev_image = None
         prev_page_nr = None
         for idx, (elem, _) in enumerate(
-            doc.iterate_items(included_content_layers=included_content_layers)
+            doc.iterate_items(
+                included_content_layers=included_content_layers, traverse_pictures=True
+            )
         ):
             if not isinstance(elem, DocItem):
                 continue

{docling_core-2.42.0 → docling_core-2.43.1}/docling_core/types/doc/document.py

@@ -4098,7 +4098,10 @@ class DoclingDocument(BaseModel):
         return result
 
     def _with_pictures_refs(
-        self, image_dir: Path, reference_path: Optional[Path] = None
+        self,
+        image_dir: Path,
+        page_no: Optional[int],
+        reference_path: Optional[Path] = None,
     ) -> "DoclingDocument":
         """Document with images as refs.
 
@@ -4111,7 +4114,7 @@ class DoclingDocument(BaseModel):
         image_dir.mkdir(parents=True, exist_ok=True)
 
         if image_dir.is_dir():
-            for item, level in result.iterate_items(with_groups=False):
+            for item, level in result.iterate_items(page_no=page_no, with_groups=False):
                 if isinstance(item, PictureItem):
 
                     if (
@@ -4211,7 +4214,7 @@ class DoclingDocument(BaseModel):
             os.makedirs(artifacts_dir, exist_ok=True)
 
         new_doc = self._make_copy_with_refmode(
-            artifacts_dir, image_mode, reference_path=reference_path
+            artifacts_dir, image_mode, page_no=None, reference_path=reference_path
         )
 
         out = new_doc.export_to_dict(
@@ -4254,7 +4257,7 @@ class DoclingDocument(BaseModel):
             os.makedirs(artifacts_dir, exist_ok=True)
 
         new_doc = self._make_copy_with_refmode(
-            artifacts_dir, image_mode, reference_path=reference_path
+            artifacts_dir, image_mode, page_no=None, reference_path=reference_path
         )
 
         out = new_doc.export_to_dict(
@@ -4327,7 +4330,7 @@ class DoclingDocument(BaseModel):
             os.makedirs(artifacts_dir, exist_ok=True)
 
         new_doc = self._make_copy_with_refmode(
-            artifacts_dir, image_mode, reference_path=reference_path
+            artifacts_dir, image_mode, page_no, reference_path=reference_path
         )
 
         md_out = new_doc.export_to_markdown(
@@ -4503,7 +4506,7 @@ class DoclingDocument(BaseModel):
             os.makedirs(artifacts_dir, exist_ok=True)
 
         new_doc = self._make_copy_with_refmode(
-            artifacts_dir, image_mode, reference_path=reference_path
+            artifacts_dir, image_mode, page_no, reference_path=reference_path
         )
 
         html_out = new_doc.export_to_html(
@@ -4542,6 +4545,7 @@ class DoclingDocument(BaseModel):
         self,
         artifacts_dir: Path,
         image_mode: ImageRefMode,
+        page_no: Optional[int],
         reference_path: Optional[Path] = None,
     ):
         new_doc = None
@@ -4549,7 +4553,7 @@ class DoclingDocument(BaseModel):
             new_doc = self
         elif image_mode == ImageRefMode.REFERENCED:
             new_doc = self._with_pictures_refs(
-                image_dir=artifacts_dir, reference_path=reference_path
+                image_dir=artifacts_dir, page_no=page_no, reference_path=reference_path
             )
         elif image_mode == ImageRefMode.EMBEDDED:
             new_doc = self._with_embedded_pictures()

{docling_core-2.42.0 → docling_core-2.43.1}/docling_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docling-core
-Version: 2.42.0
+Version: 2.43.1
 Summary: A python library to define and validate data types in Docling.
 Author-email: Cesar Berrospi Ramis <ceb@zurich.ibm.com>, Panos Vagenas <pva@zurich.ibm.com>, Michele Dolfi <dol@zurich.ibm.com>, Christoph Auer <cau@zurich.ibm.com>, Peter Staar <taa@zurich.ibm.com>
 Maintainer-email: Panos Vagenas <pva@zurich.ibm.com>, Michele Dolfi <dol@zurich.ibm.com>, Christoph Auer <cau@zurich.ibm.com>, Peter Staar <taa@zurich.ibm.com>, Cesar Berrospi Ramis <ceb@zurich.ibm.com>

{docling_core-2.42.0 → docling_core-2.43.1}/docling_core.egg-info/SOURCES.txt

@@ -30,6 +30,7 @@ docling_core/transforms/chunker/__init__.py
 docling_core/transforms/chunker/base.py
 docling_core/transforms/chunker/hierarchical_chunker.py
 docling_core/transforms/chunker/hybrid_chunker.py
+docling_core/transforms/chunker/page_chunker.py
 docling_core/transforms/chunker/tokenizer/__init__.py
 docling_core/transforms/chunker/tokenizer/base.py
 docling_core/transforms/chunker/tokenizer/huggingface.py
@@ -98,6 +99,7 @@ test/test_json_schema_to_search_mapper.py
 test/test_nlp_qa.py
 test/test_otsl_table_export.py
 test/test_page.py
+test/test_page_chunker.py
 test/test_rec_schema.py
 test/test_search_meta.py
 test/test_serialization.py

{docling_core-2.42.0 → docling_core-2.43.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "docling-core"
-version = "2.42.0"  # DO NOT EDIT, updated automatically
+version = "2.43.1"  # DO NOT EDIT, updated automatically
 description = "A python library to define and validate data types in Docling."
 license = "MIT"
 license-files = ["LICENSE"]

{docling_core-2.42.0 → docling_core-2.43.1}/test/test_base.py

@@ -36,8 +36,8 @@ def test_identifier():
     )
 
     # schema_json(): no need to set by_alias since it is True by the default
-    tf = open("test/data/json_schemas/base_identifier.json", encoding="utf-8")
-    gold_json = json.load(tf)
+    with open("test/data/json_schemas/base_identifier.json", encoding="utf-8") as tf:
+        gold_json = json.load(tf)
 
     assert Identifier.model_json_schema() == gold_json
 

{docling_core-2.42.0 → docling_core-2.43.1}/test/test_docling_doc.py

@@ -1349,12 +1349,51 @@ def test_save_pictures():
 
     doc: DoclingDocument = _construct_doc()
 
-    new_doc = doc._with_pictures_refs(image_dir=Path("./test/data/constructed_images/"))
+    new_doc = doc._with_pictures_refs(
+        image_dir=Path("./test/data/constructed_images/"), page_no=None
+    )
 
     img_paths = new_doc._list_images_on_disk()
     assert len(img_paths) == 1, "len(img_paths)!=1"
 
 
+def test_save_pictures_with_page():
+    # Given
+    doc = DoclingDocument(name="Dummy")
+
+    doc.add_page(page_no=1, size=Size(width=2000, height=4000), image=None)
+    doc.add_page(
+        page_no=2,
+        size=Size(width=2000, height=4000),
+    )
+    image = PILImage.new(mode="RGB", size=(200, 400), color=(0, 0, 0))
+    doc.add_picture(
+        image=ImageRef.from_pil(image=image, dpi=72),
+        prov=ProvenanceItem(
+            page_no=2,
+            bbox=BoundingBox(
+                b=0, l=0, r=200, t=400, coord_origin=CoordOrigin.BOTTOMLEFT
+            ),
+            charspan=(1, 2),
+        ),
+    )
+
+    # When
+    with_ref = doc._with_pictures_refs(
+        image_dir=Path("./test/data/constructed_images/"), page_no=1
+    )
+    # Then
+    n_images = len(with_ref._list_images_on_disk())
+    assert n_images == 0
+    # When
+    with_ref = with_ref._with_pictures_refs(
+        image_dir=Path("./test/data/constructed_images/"), page_no=2
+    )
+    n_images = len(with_ref._list_images_on_disk())
+    # Then
+    assert n_images == 1
+
+
 def _normalise_string_wrt_filepaths(instr: str, paths: List[Path]):
 
     for p in paths:
@@ -1406,7 +1445,8 @@ def test_save_to_disk():
     image_dir = Path("./test/data/doc/constructed_images/")
 
     doc_with_references = doc._with_pictures_refs(
-        image_dir=image_dir  # Path("./test/data/constructed_images/")
+        image_dir=image_dir,  # Path("./test/data/constructed_images/")
+        page_no=None,
     )
 
     # paths will be different on different machines, so needs to be kept!
@@ -1779,9 +1819,10 @@ def test_document_manipulation():
 
     # Test the handling of list items in insert_* methods, both with and without parent groups
 
-    li_sibling = doc.insert_list_item(
-        sibling=node, text="Inserted List Item, Incorrect Parent", after=False
-    )
+    with pytest.warns(DeprecationWarning, match="ListItem parent must be a ListGroup"):
+        li_sibling = doc.insert_list_item(
+            sibling=node, text="Inserted List Item, Incorrect Parent", after=False
+        )
     doc.insert_list_item(
         sibling=li_sibling, text="Inserted List Item, Correct Parent", after=True
     )
@@ -1791,12 +1832,13 @@ def test_document_manipulation():
         text="Inserted Text with LIST_ITEM Label, Correct Parent",
         after=False,
     )
-    doc.insert_text(
-        sibling=node,
-        label=DocItemLabel.LIST_ITEM,
-        text="Inserted Text with LIST_ITEM Label, Incorrect Parent",
-        after=True,
-    )
+    with pytest.warns(DeprecationWarning, match="ListItem parent must be a ListGroup"):
+        doc.insert_text(
+            sibling=node,
+            label=DocItemLabel.LIST_ITEM,
+            text="Inserted Text with LIST_ITEM Label, Incorrect Parent",
+            after=True,
+        )
 
     filename = Path(
         "test/data/doc/constructed_doc.inserted_list_items_with_insert_*.json"

{docling_core-2.42.0 → docling_core-2.43.1}/test/test_doctags_load.py

@@ -60,7 +60,8 @@ def test_doctags_load_from_files():
 
 def test_doctags_load_from_memory():
 
-    doctags = Path("test/data/doc/page_with_pic.dt").open("r").read()
+    with Path("test/data/doc/page_with_pic.dt").open() as file:
+        doctags = file.read()
     image = PILImage.open(Path("test/data/doc/page_with_pic.png"))
 
     doctags_doc = DocTagsDocument.from_doctags_and_image_pairs([doctags], [image])
@@ -75,7 +76,8 @@ def test_doctags_load_from_memory():
 
 
 def test_doctags_load_without_image():
-    doctags = Path("test/data/doc/page_with_pic.dt").open("r").read()
+    with Path("test/data/doc/page_with_pic.dt").open() as file:
+        doctags = file.read()
     doctags_doc = DocTagsDocument.from_doctags_and_image_pairs([doctags], None)
     doc = DoclingDocument.load_from_doctags(doctags_doc)
     exp = "test/data/doc/page_without_pic.dt.json"
@@ -86,7 +88,8 @@ def test_doctags_load_without_image():
 
 
 def test_doctags_load_for_kv_region():
-    doctags = Path("test/data/doc/doc_with_kv.dt").open("r").read()
+    with Path("test/data/doc/doc_with_kv.dt").open() as file:
+        doctags = file.read()
     image = PILImage.open(Path("test/data/doc/doc_with_kv.png"))
     doctags_doc = DocTagsDocument.from_doctags_and_image_pairs([doctags], [image])
     doc = DoclingDocument.load_from_doctags(doctags_doc)
@@ -98,7 +101,8 @@ def test_doctags_load_for_kv_region():
 
 
 def test_multipage_doctags_load():
-    doctags = Path("test/data/doc/2206.01062.yaml.dt").open("r").read()
+    with Path("test/data/doc/2206.01062.yaml.dt").open() as file:
+        doctags = file.read()
     doctags_doc = DocTagsDocument.from_multipage_doctags_and_images(doctags, None)
     doc = DoclingDocument.load_from_doctags(doctags_doc)
     exp = "test/data/doc/2206.01062.yaml.dt.json"

docling_core-2.43.1/test/test_page_chunker.py

@@ -0,0 +1,36 @@
+import json
+from pathlib import Path
+
+from docling_core.transforms.chunker.hierarchical_chunker import DocChunk
+from docling_core.transforms.chunker.page_chunker import PageChunker
+from docling_core.types.doc.document import DoclingDocument
+
+from .test_data_gen_flag import GEN_TEST_DATA
+
+
+def _process(act_data, exp_path_str):
+    if GEN_TEST_DATA:
+        with open(exp_path_str, mode="w", encoding="utf-8") as f:
+            json.dump(act_data, fp=f, indent=4)
+            f.write("\n")
+    else:
+        with open(exp_path_str, encoding="utf-8") as f:
+            exp_data = json.load(fp=f)
+        assert exp_data == act_data
+
+
+def test_page_chunks():
+    src = Path("./test/data/doc/cross_page_lists.json")
+    doc = DoclingDocument.load_from_json(src)
+
+    chunker = PageChunker()
+
+    chunk_iter = chunker.chunk(dl_doc=doc)
+    chunks = list(chunk_iter)
+    act_data = dict(
+        root=[DocChunk.model_validate(n).export_json_dict() for n in chunks]
+    )
+    _process(
+        act_data=act_data,
+        exp_path_str=src.parent / f"{src.stem}_chunks.json",
+    )