docling-core 1.7.0__py3-none-any.whl → 1.7.2__py3-none-any.whl

docling_core/transforms/chunker/base.py

@@ -4,19 +4,41 @@
 #
 
 """Define base classes for chunking."""
+import re
 from abc import ABC, abstractmethod
-from typing import Iterator, Optional
+from typing import Final, Iterator, Optional
 
-from pydantic import BaseModel
+from pydantic import BaseModel, Field, field_validator
 
 from docling_core.types import BoundingBox, Document
+from docling_core.types.base import _JSON_POINTER_REGEX
+
+# (subset of) JSONPath format, e.g. "$.main-text[84]" (for migration purposes)
+_DEPRECATED_JSON_PATH_PATTERN: Final = re.compile(r"^\$\.([\w-]+)\[(\d+)\]$")
+
+
+def _create_path(pos: int, path_prefix: str = "main-text") -> str:
+    return f"#/{path_prefix}/{pos}"
 
 
 class Chunk(BaseModel):
     """Data model for Chunk."""
 
-    path: str
+    path: str = Field(pattern=_JSON_POINTER_REGEX)
     text: str
+    heading: Optional[str] = None
+
+    @field_validator("path", mode="before")
+    @classmethod
+    def _json_pointer_from_json_path(cls, path: str):
+        if (match := _DEPRECATED_JSON_PATH_PATTERN.match(path)) is not None:
+            groups = match.groups()
+            if len(groups) == 2 and groups[0] is not None and groups[1] is not None:
+                return _create_path(
+                    pos=int(groups[1]),
+                    path_prefix=groups[0],
+                )
+        return path
 
 
 class ChunkWithMetadata(Chunk):
@@ -24,7 +46,6 @@ class ChunkWithMetadata(Chunk):
 
     page: Optional[int] = None
     bbox: Optional[BoundingBox] = None
-    heading: Optional[str] = None
 
 
 class BaseChunker(BaseModel, ABC):
@@ -44,3 +65,10 @@ class BaseChunker(BaseModel, ABC):
             Iterator[Chunk]: iterator over extracted chunks
         """
         raise NotImplementedError()
+
+    @classmethod
+    def _create_path(cls, pos: int, path_prefix: str = "main-text") -> str:
+        return _create_path(
+            pos=pos,
+            path_prefix=path_prefix,
+        )

docling_core/transforms/chunker/hierarchical_chunker.py

@@ -12,7 +12,7 @@ from enum import Enum
 from typing import Any, Iterator, Optional, Union
 
 import pandas as pd
-from pydantic import BaseModel, PositiveInt
+from pydantic import BaseModel, Field, PositiveInt
 
 from docling_core.transforms.chunker import BaseChunker, Chunk, ChunkWithMetadata
 from docling_core.types import BaseText
@@ -25,9 +25,17 @@ _logger = logging.getLogger(__name__)
 class HierarchicalChunker(BaseChunker):
     """Chunker implementation leveraging the document layout."""
 
-    include_metadata: bool = True
-    heading_as_metadata: bool = False
-    min_chunk_len: PositiveInt = 64
+    heading_as_metadata: bool = Field(
+        default=False,
+        description="Whether heading should be in metadata (instead of text)",
+    )
+    include_metadata: bool = Field(
+        default=True,
+        description="Whether to include extras in the metadata",
+    )
+    min_chunk_len: PositiveInt = Field(
+        default=64, description="Minimum chunk text length to consider (in chars)"
+    )
 
     class _NodeType(str, Enum):
         PARAGRAPH = "paragraph"
@@ -75,7 +83,7 @@ class HierarchicalChunker(BaseChunker):
             nrows = table_df.shape[0]
             ncols = table_df.shape[1]
             texts = [
-                f"{rows[i]}, {cols[j]} = {table_df.iloc[i, j].strip()}"
+                f"{rows[i]}, {cols[j]} = {str(table_df.iloc[i, j]).strip()}"
                 for i in range(1, nrows)
                 for j in range(1, ncols)
             ]
@@ -83,10 +91,6 @@ class HierarchicalChunker(BaseChunker):
 
         return output_text
 
-    @classmethod
-    def _create_path(cls, pos: int, path_prefix: str = "main-text") -> str:
-        return f"$.{path_prefix}[{pos}]"
-
     class _MainTextItemNode(BaseModel):
         parent: Optional[int] = None
         children: list[int] = []
@@ -304,14 +308,15 @@ class HierarchicalChunker(BaseChunker):
                 return ChunkWithMetadata(
                     text=concat,
                     path=path,
+                    heading=heading,
                     page=item.prov[0].page if item.prov else None,
                     bbox=item.prov[0].bbox if item.prov else None,
-                    heading=heading,
                 )
             else:
                 return Chunk(
                     text=concat,
                     path=path,
+                    heading=heading,
                 )
         else:
             return None
@@ -327,11 +332,6 @@ class HierarchicalChunker(BaseChunker):
         Yields:
             Iterator[Chunk]: iterator over extracted chunks
         """
-        if (not self.include_metadata) and self.heading_as_metadata:
-            raise RuntimeError(
-                "To enable `heading_as_metadata`, also `include_metadata` must be True."
-            )
-
         if dl_doc.main_text:
             # extract doc structure incl. metadata for
             # each item (e.g. parent, children)

docling_core/transforms/metadata_extractor/simple_metadata_extractor.py

@@ -6,20 +6,18 @@
 """Simple metadata extractor module."""
 
 
-from enum import Enum
-from typing import Any
+from typing import Any, Final
 
 from docling_core.transforms.metadata_extractor import BaseMetadataExtractor
 from docling_core.types import Document as DLDocument
 
+_DL_DOC_HASH: Final[str] = "dl_doc_hash"
+_ORIGIN: Final[str] = "origin"
+
 
 class SimpleMetadataExtractor(BaseMetadataExtractor):
     """Simple metadata extractor class."""
 
-    class _Keys(str, Enum):
-        DL_DOC_HASH = "dl_doc_hash"
-        ORIGIN = "origin"
-
     include_origin: bool = False
 
     def get_metadata(
@@ -35,10 +33,10 @@ class SimpleMetadataExtractor(BaseMetadataExtractor):
             dict[str, Any]: the extracted metadata
         """
         meta: dict[str, Any] = {
-            self._Keys.DL_DOC_HASH: doc.file_info.document_hash,
+            _DL_DOC_HASH: doc.file_info.document_hash,
         }
         if self.include_origin:
-            meta[self._Keys.ORIGIN] = origin
+            meta[_ORIGIN] = origin
         return meta
 
     def get_excluded_embed_metadata_keys(self) -> list[str]:
@@ -47,9 +45,9 @@ class SimpleMetadataExtractor(BaseMetadataExtractor):
         Returns:
             list[str]: the metadata to exclude
         """
-        excl_keys: list[str] = [self._Keys.DL_DOC_HASH]
+        excl_keys: list[str] = [_DL_DOC_HASH]
         if self.include_origin:
-            excl_keys.append(self._Keys.ORIGIN)
+            excl_keys.append(_ORIGIN)
         return excl_keys
 
     def get_excluded_llm_metadata_keys(self) -> list[str]:

docling_core/types/base.py

@@ -6,7 +6,7 @@
 """Define common models across types."""
 from datetime import datetime, timezone
 from enum import Enum
-from typing import Generic, Hashable, List, Literal, Optional, TypeVar
+from typing import Final, Generic, Hashable, List, Literal, Optional, TypeVar
 
 from pydantic import (
     AfterValidator,
@@ -28,6 +28,9 @@ from docling_core.search.package import VERSION_PATTERN
 from docling_core.utils.alias import AliasModel
 from docling_core.utils.validators import validate_datetime, validate_unique_list
 
+# (subset of) JSON Pointer URI fragment id format, e.g. "#/main-text/84":
+_JSON_POINTER_REGEX: Final[str] = r"^#(?:/([\w-]+)(?:/(\d+))?)?$"
+
 LanguageT = TypeVar("LanguageT", bound=str)
 IdentifierTypeT = TypeVar("IdentifierTypeT", bound=str)
 DescriptionAdvancedT = TypeVar("DescriptionAdvancedT", bound=BaseModel)

docling_core/types/experimental/document.py

@@ -20,6 +20,7 @@ from tabulate import tabulate
 from typing_extensions import Annotated
 
 from docling_core.search.package import VERSION_PATTERN
+from docling_core.types.base import _JSON_POINTER_REGEX
 from docling_core.types.doc.tokens import DocumentToken
 from docling_core.types.experimental import BoundingBox, Size
 from docling_core.types.experimental.labels import DocItemLabel, GroupLabel
@@ -28,9 +29,6 @@ Uint64 = typing.Annotated[int, Field(ge=0, le=(2**64 - 1))]
 LevelNumber = typing.Annotated[int, Field(ge=1, le=100)]
 CURRENT_VERSION: Final = "1.0.0"
 
-# (subset of) JSON Pointer URI fragment identifier format:
-_JSON_POINTER_REGEX = r"^#(/[\w\-]+(/\d+)?)?$"
-
 
 class BasePictureData(BaseModel):  # TBD
     """BasePictureData."""

{docling_core-1.7.0.dist-info → docling_core-1.7.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: docling-core
-Version: 1.7.0
+Version: 1.7.2
 Summary: A python library to define and validate data types in Docling.
 Home-page: https://ds4sd.github.io/
 License: MIT
@@ -28,7 +28,7 @@ Classifier: Typing :: Typed
 Requires-Dist: json-schema-for-humans (>=1.0.0,<2.0.0)
 Requires-Dist: jsonref (>=1.1.0,<2.0.0)
 Requires-Dist: jsonschema (>=4.16.0,<5.0.0)
-Requires-Dist: pandas (>=2.2.2,<3.0.0)
+Requires-Dist: pandas (>=2.1.4,<3.0.0)
 Requires-Dist: pydantic (>=2.6.0,<3.0.0)
 Requires-Dist: tabulate (>=0.9.0,<0.10.0)
 Project-URL: Repository, https://github.com/DS4SD/docling-core

{docling_core-1.7.0.dist-info → docling_core-1.7.2.dist-info}/RECORD

@@ -15,17 +15,17 @@ docling_core/search/meta.py,sha256=wSurrsqdP1N3gQKx027fVdzVmc33a7Y6rPl-FClQvtA,3
 docling_core/search/package.py,sha256=Lz2ml2eDy5t0ZimnGTq-DXHAn-f18w0bn4H5xrhs75A,1841
 docling_core/transforms/__init__.py,sha256=P81y_oqkiTN4Ld5crh1gQ6BbHqqR6C6nBt9ACDd57ds,106
 docling_core/transforms/chunker/__init__.py,sha256=xZ5ELOB8tbCoJY1dKUvOrFqxYyoHmmCNUSHxrrRi8a4,317
-docling_core/transforms/chunker/base.py,sha256=8tueNosEHt5J49olPOaw8KgZ_5WXBTBhtHBkNMR0MjA,1018
-docling_core/transforms/chunker/hierarchical_chunker.py,sha256=GkUn17E4CG8MNqIdyVmjQz6_Tdy22utBpUs-_aIMrrA,12676
+docling_core/transforms/chunker/base.py,sha256=5EW89CZf4SMB6Eh4yNjzYoNjn8S7oHH8NEpMck3Lcio,2078
+docling_core/transforms/chunker/hierarchical_chunker.py,sha256=DexBEPMR5rqnwrCMi-g98AtLDG7PKyZuf7u3NuXo-tA,12682
 docling_core/transforms/id_generator/__init__.py,sha256=7UoSyAcLsvw-RRrNjYXRVS4rIOUXjwqVpaQA-SSeINU,379
 docling_core/transforms/id_generator/base.py,sha256=SufPsaZUfMpuITq7pMv5YtlLmtGTDgA4LWmjmhQuSM0,704
 docling_core/transforms/id_generator/doc_hash_id_generator.py,sha256=SUw4FBhMZtbWCfc7oMucSwYvJTXqIPMn3yCXPRxtPCI,656
 docling_core/transforms/id_generator/uuid_generator.py,sha256=t8Bky_1JQB9myX-PJGWvW_c4-NvtHPHab6b1NdS-bpU,929
 docling_core/transforms/metadata_extractor/__init__.py,sha256=q_eAUcbaToEuYUPco4uiBO8vgTGSmZUC-r0mS7KbWh8,335
 docling_core/transforms/metadata_extractor/base.py,sha256=7h_S6-buCVtvAvKQKLISjDqFV8D3brewiQ-geqlUriI,1467
-docling_core/transforms/metadata_extractor/simple_metadata_extractor.py,sha256=SPRL6H_IvK18MiTLR9APsXh_2SRaC9SzfizDluzwFWY,1702
+docling_core/transforms/metadata_extractor/simple_metadata_extractor.py,sha256=ZRjDdXgFe8jPBNC_0ruJjQanabpkxceVsCJVVWVWlIg,1629
 docling_core/types/__init__.py,sha256=6mrAEKRW85uHJwNQBufwjPcMWCjm3oocA6MaO4_NLgg,805
-docling_core/types/base.py,sha256=fNtfQ20NKa_RBNBWbq0DfO8o0zC1Cec8UAMu0Znsltk,8170
+docling_core/types/base.py,sha256=PusJskRVL19y-hq0BgXr5e8--QEqSqLnFNJ8UbOqW88,8318
 docling_core/types/doc/__init__.py,sha256=Pzj_8rft6SJTVTCHgXRwHtuZjL6LK_6dcBWjikL9biY,125
 docling_core/types/doc/base.py,sha256=ujko-oQKoXw6wjBn0Il2Khu3PyljHqYnUNh3mPDVJF8,14676
 docling_core/types/doc/doc_ann.py,sha256=8pV2efUglw19jxl4_oqB__mSxjWvtGIcllyCdqA-b2s,1196
@@ -35,7 +35,7 @@ docling_core/types/doc/document.py,sha256=AKp1kOo0tncf9FX3q7qRWQ2Jz_hZE44smZpyrt
 docling_core/types/doc/tokens.py,sha256=uU_MYW_p7ypf7eYICFBvxdnVaPZ7CQnvZmbJ6oPrtEA,6134
 docling_core/types/experimental/__init__.py,sha256=mpqa2soTcHHEKqkcSeYBbAHepg0OgVZNReKvPmGz2r4,587
 docling_core/types/experimental/base.py,sha256=k04zvzNI7qo4HfKxLPCePKxCnerzXd582gvrVjF25SI,4225
-docling_core/types/experimental/document.py,sha256=xy-Y_Gf3L4iXigEuQCrEFNsZfBKhAuDaOayq9ZyvmAU,37925
+docling_core/types/experimental/document.py,sha256=X3z4sjRmWytRbEmSCnKat4D9sYxSV7Olm1YNmG3c5Kg,37874
 docling_core/types/experimental/labels.py,sha256=tpmvpmJuQyYMLhxAvJSVuFhDRh_zQNiP1WrQmNXKQzo,1224
 docling_core/types/gen/__init__.py,sha256=C6TuCfvpSnSL5XDOFMcYHUY2-i08vvfOGRcdu6Af0pI,124
 docling_core/types/gen/generic.py,sha256=l4CZ4_Lb8ONG36WNJWbKX5hGKvTh_yU-hXp5hsm7uVU,844
@@ -56,8 +56,8 @@ docling_core/utils/ds_generate_jsonschema.py,sha256=EhNQutqWJFWuN-yl9UUPFZ7DJTvG
 docling_core/utils/file.py,sha256=VQgzjyvmJnAIHB6ex7ikcmbDAR4GA1ALreuO7Ubrp50,1895
 docling_core/utils/validate.py,sha256=3FmnxnKTDZC5J9OGxCL3U3DGRl0t0bBV1NcySXswdas,2031
 docling_core/utils/validators.py,sha256=fBdyWX4PvFh7o_d25ZTs4iwmeo75QTbrxsvXv2kXkTg,2777
-docling_core-1.7.0.dist-info/LICENSE,sha256=2M9-6EoQ1sxFztTOkXGAtwUDJvnWaAHdB9BYWVwGkIw,1087
-docling_core-1.7.0.dist-info/METADATA,sha256=rpzl-k7WEt4anqtZcE-FFjrfTBHIXXP_fhiHrgI7xz4,5383
-docling_core-1.7.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-docling_core-1.7.0.dist-info/entry_points.txt,sha256=XHhtJEkdUuLxXSNxLdFIzx_siQ3z2UFQEKp-P8VYAE4,189
-docling_core-1.7.0.dist-info/RECORD,,
+docling_core-1.7.2.dist-info/LICENSE,sha256=2M9-6EoQ1sxFztTOkXGAtwUDJvnWaAHdB9BYWVwGkIw,1087
+docling_core-1.7.2.dist-info/METADATA,sha256=OfpdHwn-55a4Z-61sx7SZ1yD7jMDaiZ_LEmEIoYKa9I,5383
+docling_core-1.7.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+docling_core-1.7.2.dist-info/entry_points.txt,sha256=XHhtJEkdUuLxXSNxLdFIzx_siQ3z2UFQEKp-P8VYAE4,189
+docling_core-1.7.2.dist-info/RECORD,,