docling-core 2.21.1__tar.gz → 2.21.2__tar.gz

{docling_core-2.21.1 → docling_core-2.21.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: docling-core
-Version: 2.21.1
+Version: 2.21.2
 Summary: A python library to define and validate data types in Docling.
 Home-page: https://ds4sd.github.io/
 License: MIT

{docling_core-2.21.1 → docling_core-2.21.2}/docling_core/transforms/chunker/hybrid_chunker.py

@@ -73,7 +73,7 @@ class HybridChunker(BaseChunker):
             for t in text:
                 total += self._count_text_tokens(t)
             return total
-        return len(self._tokenizer.tokenize(text, max_length=None))
+        return len(self._tokenizer.tokenize(text))
 
     class _ChunkLengthInfo(BaseModel):
         total_len: int
@@ -82,7 +82,7 @@ class HybridChunker(BaseChunker):
 
     def _count_chunk_tokens(self, doc_chunk: DocChunk):
         ser_txt = self.serialize(chunk=doc_chunk)
-        return len(self._tokenizer.tokenize(text=ser_txt, max_length=None))
+        return len(self._tokenizer.tokenize(text=ser_txt))
 
     def _doc_chunk_length(self, doc_chunk: DocChunk):
         text_length = self._count_text_tokens(doc_chunk.text)

{docling_core-2.21.1 → docling_core-2.21.2}/docling_core/types/doc/document.py

@@ -800,7 +800,7 @@ class CodeItem(FloatingItem, TextItem):
         :param add_content: bool:  (Default value = True)
 
         """
-        body = f"<{self.label.value}{new_line}"
+        body = f"<{self.label.value}>{new_line}"
 
         if add_location:
             body += self.get_location_tokens(
@@ -813,7 +813,7 @@ class CodeItem(FloatingItem, TextItem):
         if add_content and self.text is not None:
             body += f"<_{self.code_language.value}_>{self.text}{new_line}"
 
-        body += f"</{self.label.value}\n"
+        body += f"</{self.label.value}>\n"
 
         return body
 
@@ -2487,7 +2487,6 @@ class DoclingDocument(BaseModel):
                         is_inline_scope=is_inline_scope,
                         visited=visited,
                     )
-                    # NOTE: assumes unordered (flag & marker currently in ListItem)
                     indent_str = list_level * indent * " "
                     is_ol = item.label == GroupLabel.ORDERED_LIST
                     text = "\n".join(
@@ -2501,7 +2500,12 @@ class DoclingDocument(BaseModel):
                             for i, c in enumerate(comps)
                         ]
                     )
-                    _ingest_text(text=text)
+                    _ingest_text(
+                        text=text,
+                        # special chars have already been escaped as needed
+                        do_escape_html=False,
+                        do_escape_underscores=False,
+                    )
                 elif item.label == GroupLabel.INLINE:
                     comps = self._get_markdown_components(
                         node=item,
@@ -2520,7 +2524,13 @@ class DoclingDocument(BaseModel):
                         is_inline_scope=True,
                         visited=visited,
                     )
-                    _ingest_text(" ".join(comps))
+                    text = " ".join(comps)
+                    _ingest_text(
+                        text=text,
+                        # special chars have already been escaped as needed
+                        do_escape_html=False,
+                        do_escape_underscores=False,
+                    )
                 else:
                     continue
 
@@ -2838,7 +2848,7 @@ class DoclingDocument(BaseModel):
 
                 # Building a math equation in MathML format
                 # ref https://www.w3.org/TR/wai-aria-1.1/#math
-                elif formula_to_mathml:
+                elif formula_to_mathml and len(math_formula) > 0:
                     try:
                         mathml_element = latex2mathml.converter.convert_to_element(
                             math_formula, display="block"
@@ -2860,7 +2870,7 @@ class DoclingDocument(BaseModel):
                             and img_fallback is not None
                         ):
                             text = img_fallback
-                        elif len(math_formula) > 0:
+                        else:
                             text = f"<pre>{math_formula}</pre>"
 
                 elif math_formula != "":

{docling_core-2.21.1 → docling_core-2.21.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "docling-core"
-version = "2.21.1"
+version = "2.21.2"
 description = "A python library to define and validate data types in Docling."
 license = "MIT"
 authors = [