docling-core 2.7.0__py3-none-any.whl → 2.8.0__py3-none-any.whl

docling_core/transforms/chunker/__init__.py

@@ -7,6 +7,7 @@
 
 from docling_core.transforms.chunker.base import BaseChunk, BaseChunker, BaseMeta
 from docling_core.transforms.chunker.hierarchical_chunker import (
+    DocChunk,
     DocMeta,
     HierarchicalChunker,
 )

docling_core/transforms/chunker/base.py

@@ -4,6 +4,7 @@
 #
 
 """Define base classes for chunking."""
+import json
 from abc import ABC, abstractmethod
 from typing import Any, ClassVar, Iterator
 
@@ -11,6 +12,8 @@ from pydantic import BaseModel
 
 from docling_core.types.doc import DoclingDocument as DLDocument
 
+DFLT_DELIM = "\n"
+
 
 class BaseMeta(BaseModel):
     """Chunk metadata base class."""
@@ -45,6 +48,8 @@ class BaseChunk(BaseModel):
 class BaseChunker(BaseModel, ABC):
     """Chunker base class."""
 
+    delim: str = DFLT_DELIM
+
     @abstractmethod
     def chunk(self, dl_doc: DLDocument, **kwargs) -> Iterator[BaseChunk]:
         """Chunk the provided document.
@@ -59,3 +64,32 @@ class BaseChunker(BaseModel, ABC):
             Iterator[BaseChunk]: iterator over extracted chunks
         """
         raise NotImplementedError()
+
+    def serialize(self, chunk: BaseChunk) -> str:
+        """Serialize the given chunk. This base implementation is embedding-targeted.
+
+        Args:
+            chunk: chunk to serialize
+
+        Returns:
+            str: the serialized form of the chunk
+        """
+        meta = chunk.meta.export_json_dict()
+
+        items = []
+        for k in meta:
+            if k not in chunk.meta.excluded_embed:
+                if isinstance(meta[k], list):
+                    items.append(
+                        self.delim.join(
+                            [
+                                d if isinstance(d, str) else json.dumps(d)
+                                for d in meta[k]
+                            ]
+                        )
+                    )
+                else:
+                    items.append(json.dumps(meta[k]))
+        items.append(chunk.text)
+
+        return self.delim.join(items)

docling_core/transforms/chunker/hierarchical_chunker.py

@@ -104,7 +104,7 @@ class DocMeta(BaseMeta):
 
 
 class DocChunk(BaseChunk):
-    """Data model for Hierarchical Chunker chunks."""
+    """Data model for document chunks."""
 
     meta: DocMeta
 
@@ -119,7 +119,6 @@ class HierarchicalChunker(BaseChunker):
     """
 
     merge_list_items: bool = True
-    delim: str = "\n"
 
     @classmethod
     def _triplet_serialize(cls, table_df: DataFrame) -> str:

docling_core/transforms/chunker/hybrid_chunker.py

@@ -0,0 +1,272 @@
+#
+# Copyright IBM Corp. 2024 - 2024
+# SPDX-License-Identifier: MIT
+#
+
+"""Hybrid chunker implementation leveraging both doc structure & token awareness."""
+
+import warnings
+from typing import Iterable, Iterator, Optional, Union
+
+from pydantic import BaseModel, ConfigDict, PositiveInt, TypeAdapter, model_validator
+from typing_extensions import Self
+
+try:
+    import semchunk
+    from transformers import AutoTokenizer, PreTrainedTokenizerBase
+except ImportError:
+    raise RuntimeError(
+        "Module requires 'chunking' extra; to install, run: "
+        "`pip install 'docling-core[chunking]'`"
+    )
+
+from docling_core.transforms.chunker import (
+    BaseChunk,
+    BaseChunker,
+    DocChunk,
+    DocMeta,
+    HierarchicalChunker,
+)
+from docling_core.types import DoclingDocument
+from docling_core.types.doc.document import TextItem
+
+
+class HybridChunker(BaseChunker):
+    r"""Chunker doing tokenization-aware refinements on top of document layout chunking.
+
+    Args:
+        tokenizer: The tokenizer to use; either instantiated object or name or path of
+            respective pretrained model
+        max_tokens: The maximum number of tokens per chunk. If not set, limit is
+            resolved from the tokenizer
+        merge_peers: Whether to merge undersized chunks sharing same relevant metadata
+    """
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    tokenizer: Union[PreTrainedTokenizerBase, str]
+    max_tokens: int = None  # type: ignore[assignment]
+    merge_peers: bool = True
+
+    _inner_chunker: HierarchicalChunker = HierarchicalChunker()
+
+    @model_validator(mode="after")
+    def _patch_tokenizer_and_max_tokens(self) -> Self:
+        self._tokenizer = (
+            self.tokenizer
+            if isinstance(self.tokenizer, PreTrainedTokenizerBase)
+            else AutoTokenizer.from_pretrained(self.tokenizer)
+        )
+        if self.max_tokens is None:
+            self.max_tokens = TypeAdapter(PositiveInt).validate_python(
+                self._tokenizer.model_max_length
+            )
+        return self
+
+    def _count_tokens(self, text: Optional[Union[str, list[str]]]):
+        if text is None:
+            return 0
+        elif isinstance(text, list):
+            total = 0
+            for t in text:
+                total += self._count_tokens(t)
+            return total
+        return len(self._tokenizer.tokenize(text, max_length=None))
+
+    class _ChunkLengthInfo(BaseModel):
+        total_len: int
+        text_len: int
+        other_len: int
+
+    def _doc_chunk_length(self, doc_chunk: DocChunk):
+        text_length = self._count_tokens(doc_chunk.text)
+        headings_length = self._count_tokens(doc_chunk.meta.headings)
+        captions_length = self._count_tokens(doc_chunk.meta.captions)
+        total = text_length + headings_length + captions_length
+        return self._ChunkLengthInfo(
+            total_len=total,
+            text_len=text_length,
+            other_len=total - text_length,
+        )
+
+    def _make_chunk_from_doc_items(
+        self, doc_chunk: DocChunk, window_text: str, window_start: int, window_end: int
+    ):
+        meta = DocMeta(
+            doc_items=doc_chunk.meta.doc_items[window_start : window_end + 1],
+            headings=doc_chunk.meta.headings,
+            captions=doc_chunk.meta.captions,
+        )
+        new_chunk = DocChunk(text=window_text, meta=meta)
+        return new_chunk
+
+    def _merge_text(self, t1, t2):
+        if t1 == "":
+            return t2
+        elif t2 == "":
+            return t1
+        else:
+            return f"{t1}{self.delim}{t2}"
+
+    def _split_by_doc_items(self, doc_chunk: DocChunk) -> list[DocChunk]:
+        if doc_chunk.meta.doc_items is None or len(doc_chunk.meta.doc_items) <= 1:
+            return [doc_chunk]
+        length = self._doc_chunk_length(doc_chunk)
+        if length.total_len <= self.max_tokens:
+            return [doc_chunk]
+        else:
+            chunks = []
+            window_start = 0
+            window_end = 0
+            window_text = ""
+            window_text_length = 0
+            other_length = length.other_len
+            num_items = len(doc_chunk.meta.doc_items)
+            while window_end < num_items:
+                doc_item = doc_chunk.meta.doc_items[window_end]
+                if isinstance(doc_item, TextItem):
+                    text = doc_item.text
+                else:
+                    raise RuntimeError("Non-TextItem split not implemented yet")
+                text_length = self._count_tokens(text)
+                if (
+                    text_length + window_text_length + other_length < self.max_tokens
+                    and window_end < num_items - 1
+                ):
+                    # Still room left to add more to this chunk AND still at least one
+                    # item left
+                    window_end += 1
+                    window_text_length += text_length
+                    window_text = self._merge_text(window_text, text)
+                elif text_length + window_text_length + other_length < self.max_tokens:
+                    # All the items in the window fit into the chunk and there are no
+                    # other items left
+                    window_text = self._merge_text(window_text, text)
+                    new_chunk = self._make_chunk_from_doc_items(
+                        doc_chunk, window_text, window_start, window_end
+                    )
+                    chunks.append(new_chunk)
+                    window_end = num_items
+                elif window_start == window_end:
+                    # Only one item in the window and it doesn't fit into the chunk. So
+                    # we'll just make it a chunk for now and it will get split in the
+                    # plain text splitter.
+                    window_text = self._merge_text(window_text, text)
+                    new_chunk = self._make_chunk_from_doc_items(
+                        doc_chunk, window_text, window_start, window_end
+                    )
+                    chunks.append(new_chunk)
+                    window_start = window_end + 1
+                    window_end = window_start
+                    window_text = ""
+                    window_text_length = 0
+                else:
+                    # Multiple items in the window but they don't fit into the chunk.
+                    # However, the existing items must have fit or we wouldn't have
+                    # gotten here. So we put everything but the last item into the chunk
+                    # and then start a new window INCLUDING the current window end.
+                    new_chunk = self._make_chunk_from_doc_items(
+                        doc_chunk, window_text, window_start, window_end - 1
+                    )
+                    chunks.append(new_chunk)
+                    window_start = window_end
+                    window_text = ""
+                    window_text_length = 0
+            return chunks
+
+    def _split_using_plain_text(
+        self,
+        doc_chunk: DocChunk,
+    ) -> list[DocChunk]:
+        lengths = self._doc_chunk_length(doc_chunk)
+        if lengths.total_len <= self.max_tokens:
+            return [DocChunk(**doc_chunk.export_json_dict())]
+        else:
+            # How much room is there for text after subtracting out the headers and
+            # captions:
+            available_length = self.max_tokens - lengths.other_len
+            sem_chunker = semchunk.chunkerify(
+                self._tokenizer, chunk_size=available_length
+            )
+            if available_length <= 0:
+                warnings.warn(
+                    f"Headers and captions for this chunk are longer than the total amount of size for the chunk, chunk will be ignored: {doc_chunk.text=}"  # noqa
+                )
+                return []
+            text = doc_chunk.text
+            segments = sem_chunker.chunk(text)
+            chunks = [DocChunk(text=s, meta=doc_chunk.meta) for s in segments]
+            return chunks
+
+    def _merge_chunks_with_matching_metadata(self, chunks: list[DocChunk]):
+        output_chunks = []
+        window_start = 0
+        window_end = 0
+        num_chunks = len(chunks)
+        while window_end < num_chunks:
+            chunk = chunks[window_end]
+            lengths = self._doc_chunk_length(chunk)
+            headings_and_captions = (chunk.meta.headings, chunk.meta.captions)
+            ready_to_append = False
+            if window_start == window_end:
+                # starting a new block of chunks to potentially merge
+                current_headings_and_captions = headings_and_captions
+                window_text = chunk.text
+                window_other_length = lengths.other_len
+                window_text_length = lengths.text_len
+                window_items = chunk.meta.doc_items
+                window_end += 1
+                first_chunk_of_window = chunk
+            elif (
+                headings_and_captions == current_headings_and_captions
+                and window_text_length + window_other_length + lengths.text_len
+                <= self.max_tokens
+            ):
+                # there is room to include the new chunk so add it to the window and
+                # continue
+                window_text = self._merge_text(window_text, chunk.text)
+                window_text_length += lengths.text_len
+                window_items = window_items + chunk.meta.doc_items
+                window_end += 1
+            else:
+                ready_to_append = True
+
+            if ready_to_append or window_end == num_chunks:
+                # no more room OR the start of new metadata.  Either way, end the block
+                # and use the current window_end as the start of a new block
+                if window_start + 1 == window_end:
+                    # just one chunk so use it as is
+                    output_chunks.append(first_chunk_of_window)
+                else:
+                    new_meta = DocMeta(
+                        doc_items=window_items,
+                        headings=current_headings_and_captions[0],
+                        captions=current_headings_and_captions[1],
+                    )
+                    new_chunk = DocChunk(
+                        text=window_text,
+                        meta=new_meta,
+                    )
+                    output_chunks.append(new_chunk)
+                # no need to reset window_text, etc. because that will be reset in the
+                # next iteration in the if window_start == window_end block
+                window_start = window_end
+
+        return output_chunks
+
+    def chunk(self, dl_doc: DoclingDocument, **kwargs) -> Iterator[BaseChunk]:
+        r"""Chunk the provided document.
+
+        Args:
+            dl_doc (DLDocument): document to chunk
+
+        Yields:
+            Iterator[Chunk]: iterator over extracted chunks
+        """
+        res: Iterable[DocChunk]
+        res = self._inner_chunker.chunk(dl_doc=dl_doc, **kwargs)  # type: ignore
+        res = [x for c in res for x in self._split_by_doc_items(c)]
+        res = [x for c in res for x in self._split_using_plain_text(c)]
+        if self.merge_peers:
+            res = self._merge_chunks_with_matching_metadata(res)
+        return iter(res)

docling_core/types/doc/document.py

@@ -798,7 +798,7 @@ class PictureItem(FloatingItem):
 
             if img is not None:
                 imgb64 = self._image_to_base64(img)
-                text = f"\n![Image]({imgb64})\n"
+                text = f"\n![Image](data:image/png;base64,{imgb64})\n"
 
                 return text
             else:

{docling_core-2.7.0.dist-info → docling_core-2.8.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: docling-core
-Version: 2.7.0
+Version: 2.8.0
 Summary: A python library to define and validate data types in Docling.
 Home-page: https://ds4sd.github.io/
 License: MIT
@@ -25,13 +25,16 @@ Classifier: Topic :: Database
 Classifier: Topic :: Scientific/Engineering :: Information Analysis
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Typing :: Typed
+Provides-Extra: chunking
 Requires-Dist: jsonref (>=1.1.0,<2.0.0)
 Requires-Dist: jsonschema (>=4.16.0,<5.0.0)
 Requires-Dist: pandas (>=2.1.4,<3.0.0)
 Requires-Dist: pillow (>=10.3.0,<11.0.0)
 Requires-Dist: pydantic (>=2.6.0,<3.0.0,!=2.10.0,!=2.10.1,!=2.10.2)
 Requires-Dist: pyyaml (>=5.1,<7.0.0)
+Requires-Dist: semchunk (>=2.2.0,<3.0.0) ; extra == "chunking"
 Requires-Dist: tabulate (>=0.9.0,<0.10.0)
+Requires-Dist: transformers (>=4.34.0,<5.0.0) ; extra == "chunking"
 Requires-Dist: typing-extensions (>=4.12.2,<5.0.0)
 Project-URL: Repository, https://github.com/DS4SD/docling-core
 Description-Content-Type: text/markdown

{docling_core-2.7.0.dist-info → docling_core-2.8.0.dist-info}/RECORD

@@ -14,14 +14,15 @@ docling_core/search/mapping.py,sha256=6rqG7LgYSeWmooKNEcRa5gFDLp1ZdzPqDGlwTA5gpO
 docling_core/search/meta.py,sha256=wSurrsqdP1N3gQKx027fVdzVmc33a7Y6rPl-FClQvtA,3318
 docling_core/search/package.py,sha256=Lz2ml2eDy5t0ZimnGTq-DXHAn-f18w0bn4H5xrhs75A,1841
 docling_core/transforms/__init__.py,sha256=P81y_oqkiTN4Ld5crh1gQ6BbHqqR6C6nBt9ACDd57ds,106
-docling_core/transforms/chunker/__init__.py,sha256=sSSTnt7ZCt8Og1e0jhApNTtA0pyyHyzwcl8yXFLb2J8,292
-docling_core/transforms/chunker/base.py,sha256=iPouZOJ3cYWvai4P0Gpd3QmsTKQuY5fFUXzTMk_XNmE,1571
-docling_core/transforms/chunker/hierarchical_chunker.py,sha256=V4FiOYqL0GgBqVB7x6CafAJs3WF5oYifKIiexVggGPE,8086
+docling_core/transforms/chunker/__init__.py,sha256=YdizSKXLmmK9eyYBsarHWr8Mx_AoA0PT7c0absibZMk,306
+docling_core/transforms/chunker/base.py,sha256=PZl6QN41cZseTPkTwPzysDHYYFb6DwDSKw0QVSiFfG0,2541
+docling_core/transforms/chunker/hierarchical_chunker.py,sha256=cy3sE9w_7l-uoIEUcfnZlQweDHUoyAJTQ6IkzxxVjFY,8052
+docling_core/transforms/chunker/hybrid_chunker.py,sha256=LUzlqtTbXfhY40bhBVGtjEMZXFWRz1XH53OGqBh2Z3Y,11224
 docling_core/types/__init__.py,sha256=MVRSgsk5focwGyAplh_TRR3dEecIXpd98g_u3zZ5HXo,260
 docling_core/types/base.py,sha256=PusJskRVL19y-hq0BgXr5e8--QEqSqLnFNJ8UbOqW88,8318
 docling_core/types/doc/__init__.py,sha256=bEL4zKVOG7Wxm6xQrgF58mu-Teds9aSavuEAKVNhrTU,639
 docling_core/types/doc/base.py,sha256=_ttU8QI8wXDTQRUnN5n7L6D9wYFVLSAibxlFoMbgAsk,4557
-docling_core/types/doc/document.py,sha256=LXmDD0qZiB34WTWSTklcdWndetOqumMFN3yJEqifb8M,90500
+docling_core/types/doc/document.py,sha256=FoEm1GFV2JeXdxtj-ZINe7S_b_rZZjSKOSa72J16ork,90522
 docling_core/types/doc/labels.py,sha256=A8vWP82VAeXO1rlCO0oDKo_Hb8uDeQe0myOTY3P03hk,1596
 docling_core/types/doc/tokens.py,sha256=uU_MYW_p7ypf7eYICFBvxdnVaPZ7CQnvZmbJ6oPrtEA,6134
 docling_core/types/doc/utils.py,sha256=YDOh_ZD1Y7OmCEDdCLJ_MO5K3HA67nc_acfhOK6WztU,1439
@@ -52,8 +53,8 @@ docling_core/utils/generate_docs.py,sha256=BdKAoduWXOc7YMvcmlhjoJOFlUxij1ybxglj6
 docling_core/utils/generate_jsonschema.py,sha256=uNX1O5XnjyB5nA66XqZXTt3YbGuR2tyi_OhHepHYtZg,1654
 docling_core/utils/validate.py,sha256=3FmnxnKTDZC5J9OGxCL3U3DGRl0t0bBV1NcySXswdas,2031
 docling_core/utils/validators.py,sha256=azcrndLzhNkTWnbFSu9shJ5D3j_znnLrIFA5R8hzmGU,2798
-docling_core-2.7.0.dist-info/LICENSE,sha256=2M9-6EoQ1sxFztTOkXGAtwUDJvnWaAHdB9BYWVwGkIw,1087
-docling_core-2.7.0.dist-info/METADATA,sha256=ht4UM23KfXIPp2aeUjSr9AUruTANa-kSt9kDwHQyeNk,5547
-docling_core-2.7.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-docling_core-2.7.0.dist-info/entry_points.txt,sha256=jIxlWv3tnO04irlZc0zfhqJIgz1bg9Hha4AkaLWSdUA,177
-docling_core-2.7.0.dist-info/RECORD,,
+docling_core-2.8.0.dist-info/LICENSE,sha256=2M9-6EoQ1sxFztTOkXGAtwUDJvnWaAHdB9BYWVwGkIw,1087
+docling_core-2.8.0.dist-info/METADATA,sha256=HNRaSRjkC-DkeOvguUK82YRbCUDYir4cuSG6-qqKT1U,5703
+docling_core-2.8.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+docling_core-2.8.0.dist-info/entry_points.txt,sha256=jIxlWv3tnO04irlZc0zfhqJIgz1bg9Hha4AkaLWSdUA,177
+docling_core-2.8.0.dist-info/RECORD,,