dot-parser 2.1.1__tar.gz → 2.1.2__tar.gz

{dot_parser-2.1.1 → dot_parser-2.1.2}/CHANGELOG.md

@@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [2.1.2] - 2026-10-27
+
+### Fixed
+
+- `chunk()` keeps every chunk within `max_tokens`. A paragraph larger than
+  the limit on its own (a long table, text without blank lines) is now split
+  instead of kept whole, and an undersized chunk is absorbed into its neighbour
+  only when the result still fits: a long run of heading-only sections (a table
+  of contents, an index) no longer piles up into a single chunk.
+
 ## [2.1.1] - 2026-09-30
 
 ### Changed
@@ -90,7 +100,8 @@ this release.
 - `parse_with_images(backend=...)` accepts DOCX- and PPTX-capable backends in
   its type signature, matching what the function already supported at runtime.
 
-[Unreleased]: https://gitlab.com/deepika6190303/deepika-open-toolbox/dot-parser/-/compare/v2.1.1...main
+[Unreleased]: https://gitlab.com/deepika6190303/deepika-open-toolbox/dot-parser/-/compare/v2.1.2...main
+[2.1.2]: https://gitlab.com/deepika6190303/deepika-open-toolbox/dot-parser/-/compare/v2.1.1...v2.1.2
 [2.1.1]: https://gitlab.com/deepika6190303/deepika-open-toolbox/dot-parser/-/compare/v2.1.0...v2.1.1
 [2.1.0]: https://gitlab.com/deepika6190303/deepika-open-toolbox/dot-parser/-/compare/v2.0.0...v2.1.0
 [2.0.0]: https://gitlab.com/deepika6190303/deepika-open-toolbox/dot-parser/-/tags/v2.0.0

{dot_parser-2.1.1 → dot_parser-2.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: dot-parser
-Version: 2.1.1
+Version: 2.1.2
 Summary: Document-to-markdown parser and chunker for RAG pipelines
 Project-URL: Homepage, https://gitlab.com/deepika6190303/deepika-open-toolbox/dot-parser
 Project-URL: Repository, https://gitlab.com/deepika6190303/deepika-open-toolbox/dot-parser

{dot_parser-2.1.1 → dot_parser-2.1.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dot-parser"
-version = "2.1.1"
+version = "2.1.2"
 description = "Document-to-markdown parser and chunker for RAG pipelines"
 readme = "README.md"
 authors = [{ name = "Kannon For Deep Tech", email = "louis.letarnec@deepika.ai" }]

{dot_parser-2.1.1 → dot_parser-2.1.2}/src/dot_parser/chunking.py

@@ -3,6 +3,7 @@
 
 import re
 import statistics
+from typing import cast
 
 import semchunk
 
@@ -63,17 +64,16 @@ def _split_oversized(section: dict, max_tokens: int) -> list[dict]:
     body = section["body"]
     path = section["section_path"]
 
-    full_text = f"{heading}\n\n{body}" if heading else body
-    if count_tokens(full_text) <= max_tokens:
+    prefix = f"{heading}\n\n" if heading else ""
+    if count_tokens(prefix + body) <= max_tokens:
         return [section]
 
     # Try paragraph splitting first
     paragraphs = re.split(r"\n\n+", body)
     if len(paragraphs) > 1:
-        sub_chunks = _group_paragraphs(paragraphs, max_tokens, heading)
+        sub_chunks = _group_paragraphs(paragraphs, max_tokens, prefix)
     else:
-        # Semantic splitting via semchunk
-        sub_chunks = semchunk.chunk(body, chunk_size=max_tokens, token_counter=count_tokens)
+        sub_chunks = _split_text(body, max_tokens, prefix)
 
     results = []
     for i, text in enumerate(sub_chunks):
@@ -96,31 +96,37 @@ def _split_oversized(section: dict, max_tokens: int) -> list[dict]:
     return results
 
 
-def _group_paragraphs(paragraphs: list[str], max_tokens: int, heading: str | None) -> list[str]:
-    """Group paragraphs into chunks that fit within max_tokens."""
+def _split_text(text: str, max_tokens: int, prefix: str) -> list[str]:
+    """Split text semantically (semchunk) into pieces that fit within max_tokens
+    even with `prefix` (the heading the first piece carries) before them.
+    semchunk returns a bare list unless asked for offsets."""
+    pieces = semchunk.chunk(
+        text, chunk_size=max_tokens, token_counter=lambda t: count_tokens(prefix + t)
+    )
+    return cast("list[str]", pieces)
+
+
+def _group_paragraphs(paragraphs: list[str], max_tokens: int, prefix: str) -> list[str]:
+    """Group paragraphs into chunks that fit within max_tokens, the first one
+    with `prefix` (its heading) before it. A paragraph too large on its own is
+    split semantically first."""
     groups: list[str] = []
     current: list[str] = []
-    current_tokens = 0
 
-    # Account for heading in first group
-    heading_tokens = count_tokens(heading + "\n\n") if heading else 0
-    first_group = True
+    def fits(texts: list[str]) -> bool:
+        lead = "" if groups else prefix
+        return count_tokens(lead + "\n\n".join(texts)) <= max_tokens
 
     for para in paragraphs:
         para = para.strip()
         if not para:
             continue
-        para_tokens = count_tokens(para)
-        overhead = heading_tokens if first_group and not current else 0
-
-        if current and current_tokens + overhead + para_tokens > max_tokens:
-            groups.append("\n\n".join(current))
-            current = [para]
-            current_tokens = para_tokens
-            first_group = False
-        else:
-            current.append(para)
-            current_tokens += para_tokens
+        pieces = [para] if fits([para]) else _split_text(para, max_tokens, prefix)
+        for piece in pieces:
+            if current and not fits([*current, piece]):
+                groups.append("\n\n".join(current))
+                current = []
+            current.append(piece)
 
     if current:
         groups.append("\n\n".join(current))
@@ -168,15 +174,19 @@ def _merge_small(sections: list[dict], max_tokens: int) -> list[dict]:
 
 
 def _absorb_undersized(sections: list[dict], max_tokens: int) -> list[dict]:
-    """Absorb chunks smaller than 15% of max_tokens into neighbors."""
+    """Absorb chunks smaller than 15% of max_tokens into neighbors, when the
+    result still fits within max_tokens."""
     threshold = max_tokens * 0.15
     result: list[dict] = []
 
+    def fits(first: str, second: str) -> bool:
+        return count_tokens(first + "\n\n" + second) <= max_tokens
+
     for section in sections:
         text = _section_text(section)
         tokens = count_tokens(text)
 
-        if tokens < threshold and result:
+        if tokens < threshold and result and fits(_section_text(result[-1]), text):
             # Append to previous
             prev = result[-1]
             prev_text = _section_text(prev)
@@ -192,7 +202,7 @@ def _absorb_undersized(sections: list[dict], max_tokens: int) -> list[dict]:
     # Check if first chunk is undersized and can be prepended to next
     if len(result) >= 2:
         first_text = _section_text(result[0])
-        if count_tokens(first_text) < threshold:
+        if count_tokens(first_text) < threshold and fits(first_text, _section_text(result[1])):
             second = result[1]
             second_text = _section_text(second)
             result[1] = {

{dot_parser-2.1.1 → dot_parser-2.1.2}/tests/test_chunking.py

@@ -1,7 +1,10 @@
 # SPDX-FileCopyrightText: Kannon For Deep Tech
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
+import pytest
+
 from dot_parser import Chunk, chunk
+from dot_parser.tokens import count_tokens
 
 
 class TestChunkBasic:
@@ -62,6 +65,28 @@ class TestSplitOversized:
             assert c.section_path == ["# Big"]
 
 
+class TestMaxTokensBound:
+    @pytest.mark.parametrize("merge", [True, False])
+    def test_an_oversized_paragraph_among_others_is_split(self, merge):
+        md = "# Big\n\nIntro.\n\n" + "word " * 2000 + "\n\nOutro."
+        chunks = chunk(md, max_tokens=64, merge=merge)
+        assert all(count_tokens(c.content) <= 64 for c in chunks)
+        assert chunks[0].heading == "# Big"
+
+    @pytest.mark.parametrize("merge", [True, False])
+    def test_a_table_without_blank_lines_is_split(self, merge):
+        table = "\n".join(f"| row {i} | 0.{i:04d} | 12.5 |" for i in range(500))
+        md = f"# Results\n\nThe table below.\n\n{table}\n\nEnd."
+        chunks = chunk(md, max_tokens=64, merge=merge)
+        assert all(count_tokens(c.content) <= 64 for c in chunks)
+
+    def test_a_run_of_heading_only_sections_stays_within_the_bound(self):
+        md = "# Index\n\n" + "\n\n".join(f"## Entry {i}" for i in range(300))
+        chunks = chunk(md, max_tokens=64)
+        assert all(count_tokens(c.content) <= 64 for c in chunks)
+        assert "## Entry 299" in chunks[-1].content
+
+
 class TestMerge:
     def test_small_sections_merged(self):
         md = "# A\n\n## B\n\nShort.\n\n## C\n\nAlso short."

{dot_parser-2.1.1 → dot_parser-2.1.2}/uv.lock

@@ -654,7 +654,7 @@ standard = [
 
 [[package]]
 name = "dot-parser"
-version = "2.1.1"
+version = "2.1.2"
 source = { editable = "." }
 dependencies = [
     { name = "beautifulsoup4" },