docling-core 2.21.1__py3-none-any.whl → 2.21.2__py3-none-any.whl

docling_core/transforms/chunker/hybrid_chunker.py

@@ -73,7 +73,7 @@ class HybridChunker(BaseChunker):
             for t in text:
                 total += self._count_text_tokens(t)
             return total
-        return len(self._tokenizer.tokenize(text, max_length=None))
+        return len(self._tokenizer.tokenize(text))
 
     class _ChunkLengthInfo(BaseModel):
         total_len: int
@@ -82,7 +82,7 @@ class HybridChunker(BaseChunker):
 
     def _count_chunk_tokens(self, doc_chunk: DocChunk):
         ser_txt = self.serialize(chunk=doc_chunk)
-        return len(self._tokenizer.tokenize(text=ser_txt, max_length=None))
+        return len(self._tokenizer.tokenize(text=ser_txt))
 
     def _doc_chunk_length(self, doc_chunk: DocChunk):
         text_length = self._count_text_tokens(doc_chunk.text)

docling_core/types/doc/document.py

@@ -800,7 +800,7 @@ class CodeItem(FloatingItem, TextItem):
         :param add_content: bool:  (Default value = True)
 
         """
-        body = f"<{self.label.value}{new_line}"
+        body = f"<{self.label.value}>{new_line}"
 
         if add_location:
             body += self.get_location_tokens(
@@ -813,7 +813,7 @@ class CodeItem(FloatingItem, TextItem):
         if add_content and self.text is not None:
             body += f"<_{self.code_language.value}_>{self.text}{new_line}"
 
-        body += f"</{self.label.value}\n"
+        body += f"</{self.label.value}>\n"
 
         return body
 
@@ -2487,7 +2487,6 @@ class DoclingDocument(BaseModel):
                         is_inline_scope=is_inline_scope,
                         visited=visited,
                     )
-                    # NOTE: assumes unordered (flag & marker currently in ListItem)
                     indent_str = list_level * indent * " "
                     is_ol = item.label == GroupLabel.ORDERED_LIST
                     text = "\n".join(
@@ -2501,7 +2500,12 @@ class DoclingDocument(BaseModel):
                             for i, c in enumerate(comps)
                         ]
                     )
-                    _ingest_text(text=text)
+                    _ingest_text(
+                        text=text,
+                        # special chars have already been escaped as needed
+                        do_escape_html=False,
+                        do_escape_underscores=False,
+                    )
                 elif item.label == GroupLabel.INLINE:
                     comps = self._get_markdown_components(
                         node=item,
@@ -2520,7 +2524,13 @@ class DoclingDocument(BaseModel):
                         is_inline_scope=True,
                         visited=visited,
                     )
-                    _ingest_text(" ".join(comps))
+                    text = " ".join(comps)
+                    _ingest_text(
+                        text=text,
+                        # special chars have already been escaped as needed
+                        do_escape_html=False,
+                        do_escape_underscores=False,
+                    )
                 else:
                     continue
 
@@ -2838,7 +2848,7 @@ class DoclingDocument(BaseModel):
 
                 # Building a math equation in MathML format
                 # ref https://www.w3.org/TR/wai-aria-1.1/#math
-                elif formula_to_mathml:
+                elif formula_to_mathml and len(math_formula) > 0:
                     try:
                         mathml_element = latex2mathml.converter.convert_to_element(
                             math_formula, display="block"
@@ -2860,7 +2870,7 @@ class DoclingDocument(BaseModel):
                             and img_fallback is not None
                         ):
                             text = img_fallback
-                        elif len(math_formula) > 0:
+                        else:
                             text = f"<pre>{math_formula}</pre>"
 
                 elif math_formula != "":

{docling_core-2.21.1.dist-info → docling_core-2.21.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: docling-core
-Version: 2.21.1
+Version: 2.21.2
 Summary: A python library to define and validate data types in Docling.
 Home-page: https://ds4sd.github.io/
 License: MIT

{docling_core-2.21.1.dist-info → docling_core-2.21.2.dist-info}/RECORD

@@ -19,12 +19,12 @@ docling_core/transforms/__init__.py,sha256=P81y_oqkiTN4Ld5crh1gQ6BbHqqR6C6nBt9AC
 docling_core/transforms/chunker/__init__.py,sha256=YdizSKXLmmK9eyYBsarHWr8Mx_AoA0PT7c0absibZMk,306
 docling_core/transforms/chunker/base.py,sha256=BSWTiFOsF5YaZaZJZY8nwIdOXb9uufJMRIds7LxRNh8,2546
 docling_core/transforms/chunker/hierarchical_chunker.py,sha256=MStDUDtzFGc6j8v9AkcAnnSHTDxdoiVrp8FTmRdGqU8,8138
-docling_core/transforms/chunker/hybrid_chunker.py,sha256=kokjDdxjc_gygOokQwYFVnHv2NjWTgf9uex8o0ole7w,9876
+docling_core/transforms/chunker/hybrid_chunker.py,sha256=v-HpFg-HvQLi0gQtHm-6KlMfcMWupkBjwr5qF-rfr4E,9842
 docling_core/types/__init__.py,sha256=MVRSgsk5focwGyAplh_TRR3dEecIXpd98g_u3zZ5HXo,260
 docling_core/types/base.py,sha256=PusJskRVL19y-hq0BgXr5e8--QEqSqLnFNJ8UbOqW88,8318
 docling_core/types/doc/__init__.py,sha256=bysJn2iwjAHwThSWDPXEdVUUij7p_ax12_nx2_0CMdg,653
 docling_core/types/doc/base.py,sha256=22U1qDlD-2ICmgzbdZrjNayoPHnq4S1ks1GRoqB7y1Q,12542
-docling_core/types/doc/document.py,sha256=P8dx5lP3oVrdlrXJx-Y-nk-UM7llDF6ZwOqs046HAM4,110451
+docling_core/types/doc/document.py,sha256=ajb-E6ZNcOWEY0ngEDhlUDx5oHBjvV9aQELJ616nU94,110854
 docling_core/types/doc/labels.py,sha256=0J9Gsqz-jQ4FP2yxs9wOxoTr3qg97BniFX7MJVziUmk,5684
 docling_core/types/doc/tokens.py,sha256=i73PXkmqXCLsQ5SddnJX8L9e_Ub2_K_DYSE-VE8NDq0,3925
 docling_core/types/doc/utils.py,sha256=SaiQD-WMMooFm1bMqwatU-IGhtG048iKJb-ppnJit_k,2250
@@ -56,8 +56,8 @@ docling_core/utils/generate_jsonschema.py,sha256=uNX1O5XnjyB5nA66XqZXTt3YbGuR2ty
 docling_core/utils/legacy.py,sha256=SqNQAxl97aHfoJEsC9vZcMJg5FNkmqKPFi-wdSrnfI0,24442
 docling_core/utils/validate.py,sha256=aQ11UbFyl8iD_N7yTTZmm_VVeXz8KcCyn3GLXgkfYRM,2049
 docling_core/utils/validators.py,sha256=azcrndLzhNkTWnbFSu9shJ5D3j_znnLrIFA5R8hzmGU,2798
-docling_core-2.21.1.dist-info/LICENSE,sha256=2M9-6EoQ1sxFztTOkXGAtwUDJvnWaAHdB9BYWVwGkIw,1087
-docling_core-2.21.1.dist-info/METADATA,sha256=qz2AeXj0vfiBu24oWyMDiQSPvKM0yUn1Rj85JaUd7Yg,5803
-docling_core-2.21.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-docling_core-2.21.1.dist-info/entry_points.txt,sha256=oClcdb2L2RKx4jdqUykY16Kum_f0_whwWhGzIodyidc,216
-docling_core-2.21.1.dist-info/RECORD,,
+docling_core-2.21.2.dist-info/LICENSE,sha256=2M9-6EoQ1sxFztTOkXGAtwUDJvnWaAHdB9BYWVwGkIw,1087
+docling_core-2.21.2.dist-info/METADATA,sha256=qIh8v5l84ULFf25DQ70FqwxHdMDu_06oKYR9ZFr1HJg,5803
+docling_core-2.21.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+docling_core-2.21.2.dist-info/entry_points.txt,sha256=oClcdb2L2RKx4jdqUykY16Kum_f0_whwWhGzIodyidc,216
+docling_core-2.21.2.dist-info/RECORD,,