docling-core 2.31.0__py3-none-any.whl → 2.31.2__py3-none-any.whl

docling_core/transforms/chunker/hybrid_chunker.py

@@ -9,6 +9,7 @@ from functools import cached_property
 from typing import Any, Iterable, Iterator, Optional, Union
 
 from pydantic import BaseModel, ConfigDict, Field, computed_field, model_validator
+from transformers import PreTrainedTokenizerBase
 
 from docling_core.transforms.chunker.hierarchical_chunker import (
     ChunkingSerializerProvider,
@@ -70,23 +71,37 @@ class HybridChunker(BaseChunker):
     @model_validator(mode="before")
     @classmethod
     def _patch(cls, data: Any) -> Any:
-        if isinstance(data, dict) and (tokenizer := data.get("tokenizer")):
+        if isinstance(data, dict):
+            tokenizer = data.get("tokenizer")
             max_tokens = data.get("max_tokens")
-            if isinstance(tokenizer, BaseTokenizer):
-                pass
-            else:
+            if not isinstance(tokenizer, BaseTokenizer) and (
+                # some legacy param passed:
+                tokenizer is not None
+                or max_tokens is not None
+            ):
                 from docling_core.transforms.chunker.tokenizer.huggingface import (
                     HuggingFaceTokenizer,
                 )
 
+                warnings.warn(
+                    "Deprecated initialization parameter types for HybridChunker. "
+                    "For updated usage check out "
+                    "https://docling-project.github.io/docling/examples/hybrid_chunking/",
+                    DeprecationWarning,
+                    stacklevel=3,
+                )
+
                 if isinstance(tokenizer, str):
                     data["tokenizer"] = HuggingFaceTokenizer.from_pretrained(
                         model_name=tokenizer,
                         max_tokens=max_tokens,
                     )
-                else:
-                    # migrate previous HF-based tokenizers
-                    kwargs = {"tokenizer": tokenizer}
+                elif tokenizer is None or isinstance(
+                    tokenizer, PreTrainedTokenizerBase
+                ):
+                    kwargs = {
+                        "tokenizer": tokenizer or _get_default_tokenizer().tokenizer
+                    }
                     if max_tokens is not None:
                         kwargs["max_tokens"] = max_tokens
                     data["tokenizer"] = HuggingFaceTokenizer(**kwargs)

docling_core/transforms/serializer/markdown.py

@@ -535,7 +535,7 @@ class MarkdownDocSerializer(DocSerializer):
     ) -> SerializationResult:
         """Serialize a document out of its parts."""
         text_res = "\n\n".join([p.text for p in parts if p.text])
-        if self.params.page_break_placeholder:
+        if self.requires_page_break():
             page_sep = self.params.page_break_placeholder or ""
             for full_match, _, _ in self._get_page_breaks(text=text_res):
                 text_res = text_res.replace(full_match, page_sep)
@@ -543,6 +543,6 @@ class MarkdownDocSerializer(DocSerializer):
         return create_ser_result(text=text_res, span_source=parts)
 
     @override
-    def requires_page_break(self):
+    def requires_page_break(self) -> bool:
         """Whether to add page breaks."""
         return self.params.page_break_placeholder is not None

{docling_core-2.31.0.dist-info → docling_core-2.31.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: docling-core
-Version: 2.31.0
+Version: 2.31.2
 Summary: A python library to define and validate data types in Docling.
 Home-page: https://github.com/docling-project
 License: MIT

{docling_core-2.31.0.dist-info → docling_core-2.31.2.dist-info}/RECORD

@@ -20,7 +20,7 @@ docling_core/transforms/__init__.py,sha256=P81y_oqkiTN4Ld5crh1gQ6BbHqqR6C6nBt9AC
 docling_core/transforms/chunker/__init__.py,sha256=YdizSKXLmmK9eyYBsarHWr8Mx_AoA0PT7c0absibZMk,306
 docling_core/transforms/chunker/base.py,sha256=kJaRrGQynglG9wpy0IaAYTf4MKheWH5BAPzx4LE9yIg,2824
 docling_core/transforms/chunker/hierarchical_chunker.py,sha256=7Fpwwsn2BoiR12KGPrn8fU1uuhqBLp85MRLMF0aIsL8,8281
-docling_core/transforms/chunker/hybrid_chunker.py,sha256=67Whij6zSPZbVQA-fToyBtTfLtDK6BdnZ-Mhlz0p8ZQ,11886
+docling_core/transforms/chunker/hybrid_chunker.py,sha256=TsAPI7WmvsMbr7Xc6WGmvwrMf4WHZnEeWxtgaf-S0iM,12530
 docling_core/transforms/chunker/tokenizer/__init__.py,sha256=-bhXOTpoI7SYk7vn47z8Ek-RZFjJk4TfZawxsFuNHnE,34
 docling_core/transforms/chunker/tokenizer/base.py,sha256=2gOBQPYJYC0iWXOgMG3DiNP7xEBtii7DYcib0iECq5o,575
 docling_core/transforms/chunker/tokenizer/huggingface.py,sha256=JQ-D3b5vTPQbvu4HaMfYqFzSBLbV_HnmoBGv7d6Kqn4,2220
@@ -31,7 +31,7 @@ docling_core/transforms/serializer/common.py,sha256=TC1EwHIp9PYcI8jeTKeavUAPtoun
 docling_core/transforms/serializer/doctags.py,sha256=mEmRWVuebcG5pZcR1_HX146cyUk0_FjaLQtMXSgh9hs,17870
 docling_core/transforms/serializer/html.py,sha256=Xq9CU5qZTDdwstizYqWNL_TFNDs9NHK_6JvvZk0TP98,34571
 docling_core/transforms/serializer/html_styles.py,sha256=-jBwS4EU7yfKoz0GSoxhwx90OmIKieO6TwPw57IuxcA,4692
-docling_core/transforms/serializer/markdown.py,sha256=YqThAYMsOWSg6nZnnmrUHZohn0QvfZzRqpLrB-Keev8,17873
+docling_core/transforms/serializer/markdown.py,sha256=4thokWJIaF3dvpchjp-Y7NTSzUuXwTmfNey4MQj-c5I,17873
 docling_core/transforms/visualizer/__init__.py,sha256=gUfF25yiJ_KO46ZIUNqZQOZGy2PLx6gnnr6AZYxKHXI,35
 docling_core/transforms/visualizer/base.py,sha256=aEF7b3rHq6DVdX8zDYEPoq55BHDYe4Hh_97lBdcW4lY,555
 docling_core/transforms/visualizer/layout_visualizer.py,sha256=ulXxWGIl69-HMKDPFk_XKgNCgQeDNc969PVt_X0-drA,7823
@@ -73,8 +73,8 @@ docling_core/utils/generate_jsonschema.py,sha256=uNX1O5XnjyB5nA66XqZXTt3YbGuR2ty
 docling_core/utils/legacy.py,sha256=DrI3QGoL755ZCIoKHF74-pTWm8R0zfFo2C2vB5dT2aY,24463
 docling_core/utils/validate.py,sha256=aQ11UbFyl8iD_N7yTTZmm_VVeXz8KcCyn3GLXgkfYRM,2049
 docling_core/utils/validators.py,sha256=azcrndLzhNkTWnbFSu9shJ5D3j_znnLrIFA5R8hzmGU,2798
-docling_core-2.31.0.dist-info/LICENSE,sha256=2M9-6EoQ1sxFztTOkXGAtwUDJvnWaAHdB9BYWVwGkIw,1087
-docling_core-2.31.0.dist-info/METADATA,sha256=OohNxPwKcbRVVKm_kpa3HRFhYh9ZMVyBIlf3apF9hm4,5976
-docling_core-2.31.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-docling_core-2.31.0.dist-info/entry_points.txt,sha256=oClcdb2L2RKx4jdqUykY16Kum_f0_whwWhGzIodyidc,216
-docling_core-2.31.0.dist-info/RECORD,,
+docling_core-2.31.2.dist-info/LICENSE,sha256=2M9-6EoQ1sxFztTOkXGAtwUDJvnWaAHdB9BYWVwGkIw,1087
+docling_core-2.31.2.dist-info/METADATA,sha256=Dd_spe0UyIXloLfK2CHHHtHY08eP7SmM2jAkfdPsj8w,5976
+docling_core-2.31.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+docling_core-2.31.2.dist-info/entry_points.txt,sha256=oClcdb2L2RKx4jdqUykY16Kum_f0_whwWhGzIodyidc,216
+docling_core-2.31.2.dist-info/RECORD,,