docling-core 2.23.3__py3-none-any.whl → 2.24.0__py3-none-any.whl

docling_core/types/doc/page.py

@@ -546,7 +546,7 @@ class SegmentedPdfPage(SegmentedPage):
 
     def save_as_json(
         self,
-        filename: Path,
+        filename: Union[str, Path],
         indent: int = 2,
     ):
         """Save the page data as a JSON file.
@@ -555,12 +555,14 @@ class SegmentedPdfPage(SegmentedPage):
             filename: Path to save the JSON file
             indent: Indentation level for JSON formatting
         """
+        if isinstance(filename, str):
+            filename = Path(filename)
         out = self.export_to_dict()
         with open(filename, "w", encoding="utf-8") as fw:
             json.dump(out, fw, indent=indent)
 
     @classmethod
-    def load_from_json(cls, filename: Path) -> "SegmentedPdfPage":
+    def load_from_json(cls, filename: Union[str, Path]) -> "SegmentedPdfPage":
         """Load page data from a JSON file.
 
         Args:
@@ -569,6 +571,8 @@ class SegmentedPdfPage(SegmentedPage):
         Returns:
             Instantiated SegmentedPdfPage object
         """
+        if isinstance(filename, str):
+            filename = Path(filename)
         with open(filename, "r", encoding="utf-8") as f:
             return cls.model_validate_json(f.read())
 
@@ -1155,19 +1159,21 @@ class PdfTableOfContents(BaseModel):
         """
         return self.model_dump(mode=mode, by_alias=True, exclude_none=True)
 
-    def save_as_json(self, filename: Path, indent: int = 2):
+    def save_as_json(self, filename: Union[str, Path], indent: int = 2):
         """Save the table of contents as a JSON file.
 
         Args:
             filename: Path to save the JSON file
             indent: Indentation level for JSON formatting
         """
+        if isinstance(filename, str):
+            filename = Path(filename)
         out = self.export_to_dict()
         with open(filename, "w", encoding="utf-8") as fw:
             json.dump(out, fw, indent=indent)
 
     @classmethod
-    def load_from_json(cls, filename: Path) -> "PdfTableOfContents":
+    def load_from_json(cls, filename: Union[str, Path]) -> "PdfTableOfContents":
         """Load table of contents from a JSON file.
 
         Args:
@@ -1176,6 +1182,8 @@ class PdfTableOfContents(BaseModel):
         Returns:
             Instantiated PdfTableOfContents object
         """
+        if isinstance(filename, str):
+            filename = Path(filename)
         with open(filename, "r", encoding="utf-8") as f:
             return cls.model_validate_json(f.read())
 
@@ -1213,19 +1221,21 @@ class ParsedPdfDocument(BaseModel):
         """
         return self.model_dump(mode=mode, by_alias=True, exclude_none=True)
 
-    def save_as_json(self, filename: Path, indent: int = 2):
+    def save_as_json(self, filename: Union[str, Path], indent: int = 2):
         """Save the document as a JSON file.
 
         Args:
             filename: Path to save the JSON file
             indent: Indentation level for JSON formatting
         """
+        if isinstance(filename, str):
+            filename = Path(filename)
         out = self.export_to_dict()
         with open(filename, "w", encoding="utf-8") as fw:
             json.dump(out, fw, indent=indent)
 
     @classmethod
-    def load_from_json(cls, filename: Path) -> "ParsedPdfDocument":
+    def load_from_json(cls, filename: Union[str, Path]) -> "ParsedPdfDocument":
         """Load document from a JSON file.
 
         Args:
@@ -1234,5 +1244,7 @@ class ParsedPdfDocument(BaseModel):
         Returns:
             Instantiated ParsedPdfDocument object
         """
+        if isinstance(filename, str):
+            filename = Path(filename)
         with open(filename, "r", encoding="utf-8") as f:
             return cls.model_validate_json(f.read())

docling_core/types/doc/tokens.py

@@ -8,10 +8,10 @@
 from enum import Enum
 from typing import Tuple
 
-from docling_core.types.doc.labels import PictureClassificationLabel
+from docling_core.types.doc.labels import DocItemLabel
 
 
-class TableToken(Enum):
+class TableToken(str, Enum):
     """Class to represent an LLM friendly representation of a Table."""
 
     CELL_LABEL_COLUMN_HEADER = "<column_header>"
@@ -41,41 +41,207 @@ class TableToken(Enum):
         return label in TableToken.get_special_tokens()
 
 
-class DocumentToken(Enum):
+_LOC_PREFIX = "loc_"
+_SECTION_HEADER_PREFIX = "section_header_level_"
+
+
+class _PictureClassificationToken(str, Enum):
+    """PictureClassificationToken."""
+
+    OTHER = "<other>"
+
+    # If more than one picture is grouped together, it
+    # is generally not possible to assign a label
+    PICTURE_GROUP = "<picture_group>"
+
+    # General
+    PIE_CHART = "<pie_chart>"
+    BAR_CHART = "<bar_chart>"
+    LINE_CHART = "<line_chart>"
+    FLOW_CHART = "<flow_chart>"
+    SCATTER_CHART = "<scatter_chart>"
+    HEATMAP = "<heatmap>"
+    REMOTE_SENSING = "<remote_sensing>"
+
+    NATURAL_IMAGE = "<natural_image>"
+
+    # Chemistry
+    MOLECULAR_STRUCTURE = "<chemistry_molecular_structure>"
+    MARKUSH_STRUCTURE = "<chemistry_markush_structure>"
+
+    # Company
+    ICON = "<icon>"
+    LOGO = "<logo>"
+    SIGNATURE = "<signature>"
+    STAMP = "<stamp>"
+    QR_CODE = "<qr_code>"
+    BAR_CODE = "<bar_code>"
+    SCREENSHOT = "<screenshot>"
+
+    # Geology/Geography
+    GEOGRAPHIC_MAP = "<map>"
+    STRATIGRAPHIC_CHART = "<stratigraphic_chart>"
+
+    # Engineering
+    CAD_DRAWING = "<cad_drawing>"
+    ELECTRICAL_DIAGRAM = "<electrical_diagram>"
+
+
+class _CodeLanguageToken(str, Enum):
+    """CodeLanguageToken."""
+
+    ADA = "<_Ada_>"
+    AWK = "<_Awk_>"
+    BASH = "<_Bash_>"
+    BC = "<_bc_>"
+    C = "<_C_>"
+    C_SHARP = "<_C#_>"
+    C_PLUS_PLUS = "<_C++_>"
+    CMAKE = "<_CMake_>"
+    COBOL = "<_COBOL_>"
+    CSS = "<_CSS_>"
+    CEYLON = "<_Ceylon_>"
+    CLOJURE = "<_Clojure_>"
+    CRYSTAL = "<_Crystal_>"
+    CUDA = "<_Cuda_>"
+    CYTHON = "<_Cython_>"
+    D = "<_D_>"
+    DART = "<_Dart_>"
+    DC = "<_dc_>"
+    DOCKERFILE = "<_Dockerfile_>"
+    ELIXIR = "<_Elixir_>"
+    ERLANG = "<_Erlang_>"
+    FORTRAN = "<_FORTRAN_>"
+    FORTH = "<_Forth_>"
+    GO = "<_Go_>"
+    HTML = "<_HTML_>"
+    HASKELL = "<_Haskell_>"
+    HAXE = "<_Haxe_>"
+    JAVA = "<_Java_>"
+    JAVASCRIPT = "<_JavaScript_>"
+    JULIA = "<_Julia_>"
+    KOTLIN = "<_Kotlin_>"
+    LISP = "<_Lisp_>"
+    LUA = "<_Lua_>"
+    MATLAB = "<_Matlab_>"
+    MOONSCRIPT = "<_MoonScript_>"
+    NIM = "<_Nim_>"
+    OCAML = "<_OCaml_>"
+    OBJECTIVEC = "<_ObjectiveC_>"
+    OCTAVE = "<_Octave_>"
+    PHP = "<_PHP_>"
+    PASCAL = "<_Pascal_>"
+    PERL = "<_Perl_>"
+    PROLOG = "<_Prolog_>"
+    PYTHON = "<_Python_>"
+    RACKET = "<_Racket_>"
+    RUBY = "<_Ruby_>"
+    RUST = "<_Rust_>"
+    SML = "<_SML_>"
+    SQL = "<_SQL_>"
+    SCALA = "<_Scala_>"
+    SCHEME = "<_Scheme_>"
+    SWIFT = "<_Swift_>"
+    TYPESCRIPT = "<_TypeScript_>"
+    UNKNOWN = "<_unknown_>"
+    VISUALBASIC = "<_VisualBasic_>"
+    XML = "<_XML_>"
+    YAML = "<_YAML_>"
+
+
+class DocumentToken(str, Enum):
     """Class to represent an LLM friendly representation of a Document."""
 
     DOCUMENT = "doctag"
     OTSL = "otsl"
     ORDERED_LIST = "ordered_list"
     UNORDERED_LIST = "unordered_list"
-    LOC = "loc_"
     PAGE_BREAK = "page_break"
     SMILES = "smiles"
+    INLINE = "inline"
+
+    CAPTION = "caption"
+    FOOTNOTE = "footnote"
+    FORMULA = "formula"
+    LIST_ITEM = "list_item"
+    PAGE_FOOTER = "page_footer"
+    PAGE_HEADER = "page_header"
+    PICTURE = "picture"
+    TABLE = "table"
+    TEXT = "text"
+    TITLE = "title"
+    DOCUMENT_INDEX = "document_index"
+    CODE = "code"
+    CHECKBOX_SELECTED = "checkbox_selected"
+    CHECKBOX_UNSELECTED = "checkbox_unselected"
+    FORM = "form"
+    KEY_VALUE_REGION = "key_value_region"
+
+    PARAGRAPH = "paragraph"
+    REFERENCE = "reference"
 
     @classmethod
     def get_special_tokens(
         cls,
-        page_dimension: Tuple[int, int] = (100, 100),
+        page_dimension: Tuple[int, int] = (500, 500),
     ):
         """Function to get all special document tokens."""
-        special_tokens = [token.value for token in cls]
+        special_tokens: list[str] = []
+        for token in cls:
+            special_tokens.append(f"<{token.value}>")
+            special_tokens.append(f"</{token.value}>")
 
         for i in range(6):
             special_tokens += [
-                f"<section_header_level_{i}>",
-                f"</section_header_level_{i}>",
+                f"<{_SECTION_HEADER_PREFIX}{i}>",
+                f"</{_SECTION_HEADER_PREFIX}{i}>",
             ]
 
-        # Add dynamically picture classification tokens
-        for _, member in PictureClassificationLabel.__members__.items():
-            special_tokens.append(f"<{member}>")
+        special_tokens.extend([t.value for t in _PictureClassificationToken])
+        special_tokens.extend([t.value for t in _CodeLanguageToken])
+
+        special_tokens.extend(TableToken.get_special_tokens())
 
         # Adding dynamically generated location-tokens
-        for i in range(0, max(page_dimension[0] + 1, page_dimension[1] + 1)):
-            special_tokens.append(f"<loc_{i}>")
+        for i in range(0, max(page_dimension[0], page_dimension[1])):
+            special_tokens.append(f"<{_LOC_PREFIX}{i}>")
 
         return special_tokens
 
+    @classmethod
+    def create_token_name_from_doc_item_label(cls, label: str, level: int = 1) -> str:
+        """Get token corresponding to passed doc item label."""
+        doc_token_by_item_label = {
+            DocItemLabel.CAPTION: DocumentToken.CAPTION,
+            DocItemLabel.FOOTNOTE: DocumentToken.FOOTNOTE,
+            DocItemLabel.FORMULA: DocumentToken.FORMULA,
+            DocItemLabel.LIST_ITEM: DocumentToken.LIST_ITEM,
+            DocItemLabel.PAGE_FOOTER: DocumentToken.PAGE_FOOTER,
+            DocItemLabel.PAGE_HEADER: DocumentToken.PAGE_HEADER,
+            DocItemLabel.PICTURE: DocumentToken.PICTURE,
+            DocItemLabel.TABLE: DocumentToken.TABLE,
+            DocItemLabel.TEXT: DocumentToken.TEXT,
+            DocItemLabel.TITLE: DocumentToken.TITLE,
+            DocItemLabel.DOCUMENT_INDEX: DocumentToken.DOCUMENT_INDEX,
+            DocItemLabel.CODE: DocumentToken.CODE,
+            DocItemLabel.CHECKBOX_SELECTED: DocumentToken.CHECKBOX_SELECTED,
+            DocItemLabel.CHECKBOX_UNSELECTED: DocumentToken.CHECKBOX_UNSELECTED,
+            DocItemLabel.FORM: DocumentToken.FORM,
+            DocItemLabel.KEY_VALUE_REGION: DocumentToken.KEY_VALUE_REGION,
+            DocItemLabel.PARAGRAPH: DocumentToken.PARAGRAPH,
+            DocItemLabel.REFERENCE: DocumentToken.REFERENCE,
+        }
+
+        res: str
+        if label == DocItemLabel.SECTION_HEADER:
+            res = f"{_SECTION_HEADER_PREFIX}{level}"
+        else:
+            try:
+                res = doc_token_by_item_label[DocItemLabel(label)].value
+            except KeyError as e:
+                raise RuntimeError(f"Unexpected DocItemLabel: {label}") from e
+        return res
+
     @staticmethod
     def is_known_token(label):
         """Function to check if label is in tokens."""
@@ -83,29 +249,29 @@ class DocumentToken(Enum):
 
     @staticmethod
     def get_picture_classification_token(classification: str) -> str:
-        """Function to get picture classification tokens."""
-        return f"<{classification}>"
+        """Function to get the token for a given picture classification value."""
+        return _PictureClassificationToken(f"<{classification}>").value
+
+    @staticmethod
+    def get_code_language_token(code_language: str) -> str:
+        """Function to get the token for a given code language."""
+        return _CodeLanguageToken(f"<_{code_language}_>").value
 
     @staticmethod
-    def get_location_token(val: float, rnorm: int = 100):
+    def get_location_token(val: float, rnorm: int = 500):  # TODO review
         """Function to get location tokens."""
         val_ = round(rnorm * val)
-
-        if val_ < 0:
-            return "<loc_0>"
-
-        if val_ > rnorm:
-            return f"<loc_{rnorm}>"
-
-        return f"<loc_{val_}>"
+        val_ = max(val_, 0)
+        val_ = min(val_, rnorm - 1)
+        return f"<{_LOC_PREFIX}{val_}>"
 
     @staticmethod
     def get_location(
         bbox: tuple[float, float, float, float],
         page_w: float,
         page_h: float,
-        xsize: int = 100,
-        ysize: int = 100,
+        xsize: int = 500,  # TODO review
+        ysize: int = 500,  # TODO review
     ):
         """Get the location string give bbox and page-dim."""
         assert bbox[0] <= bbox[2], f"bbox[0]<=bbox[2] => {bbox[0]}<={bbox[2]}"

{docling_core-2.23.3.dist-info → docling_core-2.24.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: docling-core
-Version: 2.23.3
+Version: 2.24.0
 Summary: A python library to define and validate data types in Docling.
 Home-page: https://github.com/docling-project
 License: MIT

{docling_core-2.23.3.dist-info → docling_core-2.24.0.dist-info}/RECORD

@@ -3,9 +3,10 @@ docling_core/cli/__init__.py,sha256=C63yWifzpA0IV7YWDatpAdrhoV8zjqxAKv0xMf09VdM,
 docling_core/cli/view.py,sha256=gwxSBYhGqwznMR8pdXaEuAh2bjFD5X_g11xFYSgFgtM,1764
 docling_core/experimental/__init__.py,sha256=XnAVSUHbA6OFhNSpoYqSD3u83-xVaUaki1DIKFw69Ew,99
 docling_core/experimental/serializer/__init__.py,sha256=CECQlMoCDUxkg4RAUdC3itA3I3qFhKhe2HcYghN6_xw,105
-docling_core/experimental/serializer/base.py,sha256=3rMQajYerAMMBJpW7dzzmRvGJ9LTdjpu0ucrK75KTVY,5142
-docling_core/experimental/serializer/common.py,sha256=AbIYG2Dh5C2KtAKaqLHfffOFlKa4MYNNxqVjO1rQx8o,11615
-docling_core/experimental/serializer/markdown.py,sha256=J0enJuW7oGVHs038CSME5KBSaFylrCCh1rdpS6EIfzc,14764
+docling_core/experimental/serializer/base.py,sha256=avNYy8Lgv45Gm0jfO1OV4wSRsv-O9Eeow2PkUAPY1pA,5152
+docling_core/experimental/serializer/common.py,sha256=g_o-wSQONXIZM7YJF_ghlwc3W3_VkePpM6pDS4ZjrhI,13701
+docling_core/experimental/serializer/doctags.py,sha256=bNUd5vOj1JnvIYFfSc_TSzQKQ7eQ34TY7NAUNK3C604,15953
+docling_core/experimental/serializer/markdown.py,sha256=oEzuPXiooJPVL7yTbXPPFhWF8Phstmzm3mev3yqcqbo,15950
 docling_core/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 docling_core/resources/schemas/doc/ANN.json,sha256=04U5j-PU9m5w7IagJ_rHcAx7qUtLkUuaWZO9GuYHnTA,4202
 docling_core/resources/schemas/doc/DOC.json,sha256=9tVKpCqDGGq3074Nn5qlUCdTN-5k1Q0ri_scJblwnLE,6686
@@ -29,10 +30,10 @@ docling_core/types/__init__.py,sha256=MVRSgsk5focwGyAplh_TRR3dEecIXpd98g_u3zZ5HX
 docling_core/types/base.py,sha256=PusJskRVL19y-hq0BgXr5e8--QEqSqLnFNJ8UbOqW88,8318
 docling_core/types/doc/__init__.py,sha256=bysJn2iwjAHwThSWDPXEdVUUij7p_ax12_nx2_0CMdg,653
 docling_core/types/doc/base.py,sha256=22U1qDlD-2ICmgzbdZrjNayoPHnq4S1ks1GRoqB7y1Q,12542
-docling_core/types/doc/document.py,sha256=j3v1hL2O6_DzN9n8Ak0Ho46sRhElqmRXU_Gd4zqThLA,128422
+docling_core/types/doc/document.py,sha256=_FJtmp0yh6F_3AVLVN4Xpo7E1hz50gvS_-HrJmp8FOA,128806
 docling_core/types/doc/labels.py,sha256=0J9Gsqz-jQ4FP2yxs9wOxoTr3qg97BniFX7MJVziUmk,5684
-docling_core/types/doc/page.py,sha256=8A9sM-6mNad_JzaoaIXlfsBoo6zbw29uk7fp6j24omg,39461
-docling_core/types/doc/tokens.py,sha256=Z2FuzHWinYQzWZdTvOBsEQACAKPcBiSf777w5S9NJms,3947
+docling_core/types/doc/page.py,sha256=qCXp_s0cY3N1WWkICv6fjH52OVYYbjYiqRQit86FxG4,39989
+docling_core/types/doc/tokens.py,sha256=fpPtVHfO5RXk8mkqZ7YrW5LyHipg697kbFBNqn6jXQU,9159
 docling_core/types/doc/utils.py,sha256=SaiQD-WMMooFm1bMqwatU-IGhtG048iKJb-ppnJit_k,2250
 docling_core/types/gen/__init__.py,sha256=C6TuCfvpSnSL5XDOFMcYHUY2-i08vvfOGRcdu6Af0pI,124
 docling_core/types/gen/generic.py,sha256=l4CZ4_Lb8ONG36WNJWbKX5hGKvTh_yU-hXp5hsm7uVU,844
@@ -62,8 +63,8 @@ docling_core/utils/generate_jsonschema.py,sha256=uNX1O5XnjyB5nA66XqZXTt3YbGuR2ty
 docling_core/utils/legacy.py,sha256=SqNQAxl97aHfoJEsC9vZcMJg5FNkmqKPFi-wdSrnfI0,24442
 docling_core/utils/validate.py,sha256=aQ11UbFyl8iD_N7yTTZmm_VVeXz8KcCyn3GLXgkfYRM,2049
 docling_core/utils/validators.py,sha256=azcrndLzhNkTWnbFSu9shJ5D3j_znnLrIFA5R8hzmGU,2798
-docling_core-2.23.3.dist-info/LICENSE,sha256=2M9-6EoQ1sxFztTOkXGAtwUDJvnWaAHdB9BYWVwGkIw,1087
-docling_core-2.23.3.dist-info/METADATA,sha256=JSY_qNdtZqYS_9pflWQncaxDlisQdzq_DtTiaCTfcWY,5843
-docling_core-2.23.3.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-docling_core-2.23.3.dist-info/entry_points.txt,sha256=oClcdb2L2RKx4jdqUykY16Kum_f0_whwWhGzIodyidc,216
-docling_core-2.23.3.dist-info/RECORD,,
+docling_core-2.24.0.dist-info/LICENSE,sha256=2M9-6EoQ1sxFztTOkXGAtwUDJvnWaAHdB9BYWVwGkIw,1087
+docling_core-2.24.0.dist-info/METADATA,sha256=ycw0ioISQ7Uv0rL9_RU5zpsimerhh35wfKv0bul1e9g,5843
+docling_core-2.24.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+docling_core-2.24.0.dist-info/entry_points.txt,sha256=oClcdb2L2RKx4jdqUykY16Kum_f0_whwWhGzIodyidc,216
+docling_core-2.24.0.dist-info/RECORD,,