chunklet-py 2.3.1__tar.gz → 2.4.0__tar.gz

{chunklet_py-2.3.1 → chunklet_py-2.4.0}/LICENSE

@@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+SOFTWARE.

{chunklet_py-2.3.1/src/chunklet_py.egg-info → chunklet_py-2.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: chunklet-py
-Version: 2.3.1
+Version: 2.4.0
 Summary: High-fidelity context-aware chunking and interactive visualization for RAG. Advanced segmentation for code and documents, because your LLM is only as smart as the fragments you feed it.
 Author-email: speedyk_005 <speedy40115719@gmail.com>
 License-Expression: MIT
@@ -9,7 +9,7 @@ Project-URL: Repository, https://github.com/speedyk-005/chunklet-py
 Project-URL: Documentation, https://speedyk-005.github.io/chunklet-py/
 Project-URL: Issues, https://github.com/speedyk-005/chunklet-py/issues
 Project-URL: Changelog, https://github.com/speedyk-005/chunklet-py/blob/main/CHANGELOG.md
-Keywords: nlp,chunking,text-splitting,sentence-splitting,llm,rag,ai,multilingual,text processing,natural language processing,data processing,information retrieval,semantic search,document processing,document-chunker,code chunking,code-chunker,source code analysis,programming languages,code structure,chunk-visualization
+Keywords: nlp,chunking,chunker,text-chunking,text-splitting,sentence-splitting,llm,rag,ai,multilingual,text processing,natural language processing,data processing,information retrieval,semantic search,document processing,document-chunker,code chunking,code-chunker,programming languages,code structure,chunk-visualization
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
@@ -26,20 +26,17 @@ Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: loguru<1.0,>=0.7.3
-Requires-Dist: pysbd<1.0,>=0.3.4
-Requires-Dist: sentsplit<2.0,>=1.0.8
+Requires-Dist: yasbd-lib<2.0,>=0.12.0
 Requires-Dist: sentencex<=0.6.1; sys_platform == "linux" and (platform_machine == "aarch64" or platform_machine == "armv7l")
 Requires-Dist: sentencex<2.0,>=1.0.0; sys_platform != "linux" or (platform_machine != "aarch64" and platform_machine != "armv7l")
 Requires-Dist: indic-nlp-library<1.0,>=0.92
 Requires-Dist: py3langid<1.0,>=0.3.0
 Requires-Dist: mpire<3.0,>=2.10.2
-Requires-Dist: dotdict3<2.0,>=1.1.0
 Requires-Dist: more-itertools<12.0,>=10.6.0
 Requires-Dist: regex>=2025.7.29
 Requires-Dist: pydantic<3.0,>=2.11.0
 Requires-Dist: typer<1.0,>=0.19.0
 Requires-Dist: charset-normalizer<4.0,>=3.4.2
-Requires-Dist: setuptools<81
 Provides-Extra: structured-document
 Requires-Dist: pdfminer.six>=20250324; extra == "structured-document"
 Requires-Dist: python-docx<2.0,>=1.2.0; extra == "structured-document"
@@ -48,10 +45,11 @@ Requires-Dist: ebooklib<1.0,>=0.19; extra == "structured-document"
 Requires-Dist: docutils<1.0,>=0.21.2; extra == "structured-document"
 Requires-Dist: markdownify<2.0,>=1.1.0; extra == "structured-document"
 Requires-Dist: striprtf<1.0,>=0.0.29; extra == "structured-document"
-Requires-Dist: pylatexenc==2.10; extra == "structured-document"
+Requires-Dist: mailparse<2.0,>=1.0.1; extra == "structured-document"
+Requires-Dist: python-pptx<2.0,>=1.0.0; extra == "structured-document"
+Requires-Dist: pylatexenc==2.11; extra == "structured-document"
 Requires-Dist: odfpy<2.0,>=1.4.1; extra == "structured-document"
 Requires-Dist: openpyxl<4.0,>=3.1.5; extra == "structured-document"
-Requires-Dist: tabulate2<2.0,>=1.10.2; extra == "structured-document"
 Provides-Extra: document
 Requires-Dist: chunklet-py[structured-document]; extra == "document"
 Provides-Extra: code
@@ -71,7 +69,6 @@ Provides-Extra: dev
 Requires-Dist: pytest>=8.3.5; extra == "dev"
 Requires-Dist: pytest-cov>=6.2.1; extra == "dev"
 Requires-Dist: pytest-mock>=3.14.1; extra == "dev"
-Requires-Dist: pytest-timeout>=2.3.1; extra == "dev"
 Requires-Dist: ruff>=0.14.14; extra == "dev"
 Provides-Extra: docs
 Requires-Dist: mike>=2.1.2; extra == "docs"
@@ -129,7 +126,7 @@ Dumb splitting causes problems:
 Smart chunking solves this by:
 
 - **Smart limits** — Respects both natural boundaries (sentences, paragraphs, sections) AND configurable limits (tokens, lines, functions)
-- **Language-aware** — Detects language automatically and applies the right rules (50+ languages supported)
+- **Language-aware** — Detects language automatically and applies the right rules (60+ languages supported)
 - **Context preservation** — Overlap between chunks, rich metadata (source, span, document structure)
 
 ## 🤔 So What's Chunklet-py Anyway? (And Why Should You Care?)
@@ -158,7 +155,7 @@ Perfect for prepping data for LLMs, building RAG systems, or powering AI search
 | 🪶 **Featherlight Footprint** | Designed to be lightweight and memory-efficient, ensuring optimal performance without unnecessary overhead. |
 | 🗂️ **Rich Metadata for RAG** | Enriches chunks with valuable, context-aware metadata (source, span, document properties, code AST details) crucial for advanced RAG and LLM applications. |
 | 🔧 **Infinitely Customizable** | Offers extensive customization options, from pluggable token counters to custom sentence splitters and processors. |
-| 🌐 **Multilingual Mastery** | Supports over 50 natural languages for text and document chunking with intelligent detection and language-specific algorithms. |
+| 🌐 **Multilingual Mastery** | Supports over 60 natural languages for text and document chunking with intelligent detection and language-specific algorithms. |
 | 🧑‍💻 **Code-Aware Intelligence** | Language-agnostic code chunking that understands and preserves the structural integrity of your source code. |
 | 🎯 **Precision Chunking** | Flexible chunking with configurable limits based on sentences, tokens, sections, lines, and functions. |
 | 📄 **Document Format Mastery** | Processes a wide array of document formats including `.pdf`, `.docx`, `.epub`, `.txt`, `.tex`, `.html`, `.hml`, `.md`, `.rst`, `.rtf`, `.odt`, `.csv`, and `.xlsx`. |
@@ -185,19 +182,21 @@ The simplest way to get started is with pip:
 
 ```bash
 # Install and check it's working
-pip install chunklet-py
+pip install chunklet-py -U
 chunklet --version
 ```
 
 > [!TIP]
 > <b>Termux (Android)</b>
 >
-> No rust toolchain on Termux (especially python 3.13)? Install pydantic-core pre-built wheels first then retry installing chunklet-py:
+> No rust toolchain on Termux (especially python 3.13)? Pydantic-core and cryptography won't build from source.
+> Install pre-built wheels and dependencies:
 >
 > ```bash
 > pip install typing-extensions
 > pip install pydantic-core --index-url https://termux-user-repository.github.io/pypi/
 > pip install "pydantic>=2.12.4,<2.13"
+> pkg install python-cryptography
 > ```
 
 That's it! You're all set to start chunking.
@@ -217,6 +216,8 @@ Want to unlock more Chunklet-py superpowers? Add these optional dependencies bas
 *   **Visualization:** For the interactive web-based chunk visualizer:
     ```bash
     pip install "chunklet-py[visualization]"
+    # Or
+    pip install "chunklet-py[viz]"
     ```
 * **All Extras:** To install all optional dependencies:
     ```bash
@@ -352,6 +353,13 @@ splitter = SentenceSplitter()
 sentences = splitter.split_text(text, lang="auto")
 ```
 
+> [!TIP]
+> **Advanced text splitting**
+>
+> Chunklet-py quietly delegates sentence splitting to [yasbd-lib](https://github.com/speedyk-005/yasbd-lib), our high-accuracy SBD supporting **39 languages**.
+>
+> Curious why we built it? Read **[yasbd-lib vs PySBD: Two Philosophies of Sentence Boundary Detection](https://dev.to/speed_k_7e1b449706e59e433/yasbd-lib-vs-pysbd-two-philosophies-of-sentence-boundary-detection-i88)**.
+
 **CLI (Command Line Interface)**
 
 If you prefer the terminal to an IDE, the CLI is packed with features. Just ask for help.

{chunklet_py-2.3.1 → chunklet_py-2.4.0}/README.md

@@ -44,7 +44,7 @@ Dumb splitting causes problems:
 Smart chunking solves this by:
 
 - **Smart limits** — Respects both natural boundaries (sentences, paragraphs, sections) AND configurable limits (tokens, lines, functions)
-- **Language-aware** — Detects language automatically and applies the right rules (50+ languages supported)
+- **Language-aware** — Detects language automatically and applies the right rules (60+ languages supported)
 - **Context preservation** — Overlap between chunks, rich metadata (source, span, document structure)
 
 ## 🤔 So What's Chunklet-py Anyway? (And Why Should You Care?)
@@ -73,7 +73,7 @@ Perfect for prepping data for LLMs, building RAG systems, or powering AI search
 | 🪶 **Featherlight Footprint** | Designed to be lightweight and memory-efficient, ensuring optimal performance without unnecessary overhead. |
 | 🗂️ **Rich Metadata for RAG** | Enriches chunks with valuable, context-aware metadata (source, span, document properties, code AST details) crucial for advanced RAG and LLM applications. |
 | 🔧 **Infinitely Customizable** | Offers extensive customization options, from pluggable token counters to custom sentence splitters and processors. |
-| 🌐 **Multilingual Mastery** | Supports over 50 natural languages for text and document chunking with intelligent detection and language-specific algorithms. |
+| 🌐 **Multilingual Mastery** | Supports over 60 natural languages for text and document chunking with intelligent detection and language-specific algorithms. |
 | 🧑‍💻 **Code-Aware Intelligence** | Language-agnostic code chunking that understands and preserves the structural integrity of your source code. |
 | 🎯 **Precision Chunking** | Flexible chunking with configurable limits based on sentences, tokens, sections, lines, and functions. |
 | 📄 **Document Format Mastery** | Processes a wide array of document formats including `.pdf`, `.docx`, `.epub`, `.txt`, `.tex`, `.html`, `.hml`, `.md`, `.rst`, `.rtf`, `.odt`, `.csv`, and `.xlsx`. |
@@ -100,19 +100,21 @@ The simplest way to get started is with pip:
 
 ```bash
 # Install and check it's working
-pip install chunklet-py
+pip install chunklet-py -U
 chunklet --version
 ```
 
 > [!TIP]
 > <b>Termux (Android)</b>
 >
-> No rust toolchain on Termux (especially python 3.13)? Install pydantic-core pre-built wheels first then retry installing chunklet-py:
+> No rust toolchain on Termux (especially python 3.13)? Pydantic-core and cryptography won't build from source.
+> Install pre-built wheels and dependencies:
 >
 > ```bash
 > pip install typing-extensions
 > pip install pydantic-core --index-url https://termux-user-repository.github.io/pypi/
 > pip install "pydantic>=2.12.4,<2.13"
+> pkg install python-cryptography
 > ```
 
 That's it! You're all set to start chunking.
@@ -132,6 +134,8 @@ Want to unlock more Chunklet-py superpowers? Add these optional dependencies bas
 *   **Visualization:** For the interactive web-based chunk visualizer:
     ```bash
     pip install "chunklet-py[visualization]"
+    # Or
+    pip install "chunklet-py[viz]"
     ```
 * **All Extras:** To install all optional dependencies:
     ```bash
@@ -267,6 +271,13 @@ splitter = SentenceSplitter()
 sentences = splitter.split_text(text, lang="auto")
 ```
 
+> [!TIP]
+> **Advanced text splitting**
+>
+> Chunklet-py quietly delegates sentence splitting to [yasbd-lib](https://github.com/speedyk-005/yasbd-lib), our high-accuracy SBD supporting **39 languages**.
+>
+> Curious why we built it? Read **[yasbd-lib vs PySBD: Two Philosophies of Sentence Boundary Detection](https://dev.to/speed_k_7e1b449706e59e433/yasbd-lib-vs-pysbd-two-philosophies-of-sentence-boundary-detection-i88)**.
+
 **CLI (Command Line Interface)**
 
 If you prefer the terminal to an IDE, the CLI is packed with features. Just ask for help.

{chunklet_py-2.3.1 → chunklet_py-2.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "chunklet-py"
-version = "2.3.1"
+version = "2.4.0"
 description = "High-fidelity context-aware chunking and interactive visualization for RAG. Advanced segmentation for code and documents, because your LLM is only as smart as the fragments you feed it."
 authors = [
   { name = "speedyk_005", email = "speedy40115719@gmail.com" }
@@ -26,17 +26,16 @@ classifiers = [
 ]
 
 keywords = [
-  "nlp", "chunking", "text-splitting", "sentence-splitting", "llm", "rag", "ai", "multilingual",
+  "nlp", "chunking", "chunker", "text-chunking", "text-splitting", "sentence-splitting", "llm", "rag", "ai", "multilingual",
   "text processing", "natural language processing", "data processing",
   "information retrieval", "semantic search", "document processing", "document-chunker",
-  "code chunking", "code-chunker", "source code analysis", "programming languages",
+  "code chunking", "code-chunker", "programming languages",
   "code structure", "chunk-visualization",
 ]
 
 dependencies = [
   "loguru>=0.7.3,<1.0",
-  "pysbd>=0.3.4,<1.0",
-  "sentsplit>=1.0.8,<2.0",
+  "yasbd-lib>=0.12.0,<2.0",
 
   # Rust binding (1.0+) for faster performance on supported platforms, legacy for Android temporarily.
   "sentencex<=0.6.1; sys_platform == 'linux' and (platform_machine == 'aarch64' or platform_machine == 'armv7l')",
@@ -45,15 +44,11 @@ dependencies = [
   "indic-nlp-library>=0.92,<1.0",
   "py3langid>=0.3.0,<1.0",
   "mpire>=2.10.2,<3.0",
-  "dotdict3>=1.1.0,<2.0",
   "more-itertools>=10.6.0,<12.0",
   "regex>=2025.7.29",
   "pydantic>=2.11.0,<3.0",
   "typer>=0.19.0,<1.0",
   "charset-normalizer>=3.4.2,<4.0",
-
-  # sentsplit depends on deprecated pkg_resources removed in setuptools 81.0.0
-  "setuptools<81",
 ]
 
 [project.optional-dependencies]
@@ -65,13 +60,14 @@ structured-document = [
   "docutils>=0.21.2,<1.0",
   "markdownify>=1.1.0,<2.0",
   "striprtf>=0.0.29,<1.0",
+  "mailparse>=1.0.1,<2.0",
+  "python-pptx>=1.0.0,<2.0",
 
   # Lastest version of 2.x.x series
-  "pylatexenc==2.10",
+  "pylatexenc==2.11",
   
   "odfpy>=1.4.1,<2.0",
   "openpyxl>=3.1.5,<4.0",
-  "tabulate2>=1.10.2,<2.0",
 ]
 
 # Kept for compatibility
@@ -100,7 +96,6 @@ dev = [
   "pytest>=8.3.5",
   "pytest-cov>=6.2.1",
   "pytest-mock>=3.14.1",
-  "pytest-timeout>=2.3.1",
   "ruff>=0.14.14",
 ]
 
@@ -152,14 +147,13 @@ chunklet = ["visualizer/static/**/*"]
 minversion = "7.4"
 filterwarnings = [
     "ignore::DeprecationWarning:websockets.*",
-    "ignore::DeprecationWarning:pkg_resources.*",
+    "ignore::DeprecationWarning:multiprocessing.*",
 ]
 
 # disable coverage report to prevent racing with multiprocessing
 # addopts = "--cov=chunklet --cov-report=term-missing:skip-covered --durations=10"
-addopts = "--timeout=60"
-
-testpaths = ["tests"]
+addopts = "--doctest-modules"
+testpaths = ["tests", "src/chunklet/common/dotdict.py"]
 python_files = "test_*.py"
 
 [tool.coverage.run]
@@ -170,7 +164,6 @@ omit = [
     "src/chunklet/**/__init__.py",
 ]
 
-
 [tool.ruff]
 target-version = "py310"
 line-length = 88
@@ -194,9 +187,6 @@ ignore = ["E501", "E203"]
 # Ignore warnings about File(...)
 "src/chunklet/visualizer/visualizer.py" = ["B008"]
 
-# Ignore E402 - warnings.filterwarnings() must be called before imports to suppress pkg_resources
-"src/chunklet/sentence_splitter/sentence_splitter.py" = ["E402"]
-
 [tool.ruff.format]
 # Replaces Black exclude logic
 exclude = [

{chunklet_py-2.3.1 → chunklet_py-2.4.0}/src/chunklet/base_chunker.py

@@ -7,8 +7,7 @@ Defines the interface for chunkers.
 from abc import ABC, abstractmethod
 from collections.abc import Generator
 
-from dotdict3 import DotDict
-from loguru import logger
+from chunklet.common.dotdict import DotDict
 
 
 class BaseChunker(ABC):

{chunklet_py-2.3.1 → chunklet_py-2.4.0}/src/chunklet/cli.py

@@ -328,7 +328,7 @@ def split_command(
             destination.write_text(output_str, encoding="utf-8")
             typer.echo(
                 f"Successfully split and wrote {len(sentences)} sentences "
-                f"{source_display} to {destination} (Confidence: {confidence})",
+                f"{source_display} to {destination} (Lang: {lang_detected}, Confidence: {confidence})",
                 err=True,
             )
         except Exception as e:
@@ -339,7 +339,7 @@ def split_command(
 
         typer.echo(
             f"--- Sentences ({len(sentences)}): "
-            f" [{source_display} | Lang: {lang.upper()} | Confidence: {confidence}] ---"
+            f" [{source_display} | Lang: {lang_detected} | Confidence: {confidence}] ---"
         )
 
         for sentence in sentences:

{chunklet_py-2.3.1 → chunklet_py-2.4.0}/src/chunklet/code_chunker/_code_structure_extractor.py

@@ -19,7 +19,6 @@ try:
 except ImportError:  # pragma: no cover
     Node, ET = None, None
 
-from loguru import logger
 
 from chunklet.code_chunker.patterns import (
     ALL_SINGLE_LINE_COMM,
@@ -34,9 +33,8 @@ from chunklet.code_chunker.patterns import (
     NAMESPACE_DECLARATION,
     OPENER,
 )
-from chunklet.common.validation import validate_input
 from chunklet.common.logging_utils import log_info
-
+from chunklet.common.validation import validate_input
 
 CodeLine = namedtuple(
     "CodeLine", ["line_number", "content", "indent_level", "func_partial_signature"]
@@ -316,7 +314,9 @@ class CodeStructureExtractor:
         if not (curr_struct or annotated_lines_buffer):
             return
 
-        candidates = [entry for v in annotated_lines_buffer.values() for entry in v] + curr_struct
+        candidates = [
+            entry for v in annotated_lines_buffer.values() for entry in v
+        ] + curr_struct
         sorted_candidates = sorted(candidates, key=lambda x: x.line_number)
 
         if not sorted_candidates:
@@ -372,13 +372,18 @@ class CodeStructureExtractor:
         indent_level = len(deannotated_line) - len(deannotated_line.lstrip())
         first_metadata = tag == "META" and not annotated_lines_buffer["META"]
         consecutive_docstrings = (
-            annotated_lines_buffer["DOC"] and annotated_lines_buffer["DOC"][-1].line_number == line_no - 1
+            annotated_lines_buffer["DOC"]
+            and annotated_lines_buffer["DOC"][-1].line_number == line_no - 1
         )
 
         if first_metadata or not consecutive_docstrings:
-            self._flush_snippet(state["curr_struct"], state["snippet_dicts"], annotated_lines_buffer)
+            self._flush_snippet(
+                state["curr_struct"], state["snippet_dicts"], annotated_lines_buffer
+            )
 
-        annotated_lines_buffer[tag].append(CodeLine(line_no, deannotated_line, indent_level, None))
+        annotated_lines_buffer[tag].append(
+            CodeLine(line_no, deannotated_line, indent_level, None)
+        )
 
     def _handle_block_start(
         self,
@@ -431,12 +436,16 @@ class CodeStructureExtractor:
             if state["curr_struct"]:
                 if is_python_code:
                     self._flush_snippet(
-                        state["curr_struct"], state["snippet_dicts"], annotated_lines_buffer
+                        state["curr_struct"],
+                        state["snippet_dicts"],
+                        annotated_lines_buffer,
                     )
                 else:
                     doc = annotated_lines_buffer.pop("DOC", [])
                     self._flush_snippet(
-                        state["curr_struct"], state["snippet_dicts"], annotated_lines_buffer
+                        state["curr_struct"],
+                        state["snippet_dicts"],
+                        annotated_lines_buffer,
                     )
                     annotated_lines_buffer.clear()
                     annotated_lines_buffer["doc"] = doc
@@ -536,7 +545,9 @@ class CodeStructureExtractor:
 
         # Append last snippet
         if state["curr_struct"]:
-            self._flush_snippet(state["curr_struct"], state["snippet_dicts"], annotated_lines_buffer)
+            self._flush_snippet(
+                state["curr_struct"], state["snippet_dicts"], annotated_lines_buffer
+            )
 
         snippet_dicts = self._post_processing(state["snippet_dicts"])
         log_info(

{chunklet_py-2.3.1 → chunklet_py-2.4.0}/src/chunklet/code_chunker/code_chunker.py

@@ -29,10 +29,11 @@ from itertools import chain
 from pathlib import Path
 from typing import Annotated, Any, Callable, Generator, Literal
 
-from dotdict3 import DotDict
 from more_itertools import unique_everseen
 from pydantic import Field
 
+from chunklet.common.dotdict import DotDict
+
 try:
     import defusedxml.ElementTree as ET
     from charset_normalizer import from_path
@@ -50,7 +51,7 @@ from chunklet.common.deprecation import deprecated_callable
 from chunklet.common.logging_utils import log_info
 from chunklet.common.path_utils import is_path_like, read_text_file
 from chunklet.common.token_utils import count_tokens
-from chunklet.common.validation import IterableOfStr, IterableOfPath, validate_input
+from chunklet.common.validation import IterableOfPath, IterableOfStr, validate_input
 from chunklet.exceptions import (
     InvalidInputError,
     MissingTokenCounterError,
@@ -125,6 +126,7 @@ class CodeChunker(BaseChunker):
         # Deduplicate relations
         def relation_key(relation: dict):
             return tuple(sorted(relation.items()))
+
         unique_relations = list(unique_everseen(all_relations_flat, key=relation_key))
 
         if not unique_relations:
@@ -203,7 +205,7 @@ class CodeChunker(BaseChunker):
                         }
                     )
                 )
-                curr_chunk = [line]  # Add the overflow line!
+                curr_chunk = []
                 token_count = 0
                 line_count = 0
 
@@ -255,7 +257,6 @@ class CodeChunker(BaseChunker):
         Args:
             box_tokens: Actual token count in the block
             max_tokens: Maximum allowed tokens
-            box_lines: Actual line count in the block
             max_lines: Maximum allowed lines
             function_count: Actual function count in the block
             max_functions: Maximum allowed functions
@@ -278,6 +279,62 @@ class CodeChunker(BaseChunker):
             f"Content starting with: \n```\n{content_preview}...\n```"
         )
 
+    def _handle_oversized_snippet(
+        self,
+        snippet_dict: dict,
+        **kwargs: Any,
+    ) -> None:
+        """
+        Handle a snippet that exceeds constraints and cannot be merged.
+
+        Expects ``kwargs`` to contain the same keys as the local variables of
+        ``_group_by_chunk`` that are relevant to oversized handling:
+        ``max_tokens``, ``max_lines``, ``max_functions``, ``strict``,
+        ``source``, ``token_counter``, ``cumulative_lengths``,
+        ``result_chunks``, ``box_tokens``, ``box_lines``, ``is_func``.
+
+        Either raises a ``TokenLimitError`` (strict mode) or splits the
+        oversized block into sub-chunks appended to ``result_chunks``.
+        """
+        limit_msg = self._format_limit_msg(
+            kwargs.get("box_tokens", 0),
+            kwargs["max_tokens"],
+            kwargs.get("box_lines", 0),
+            kwargs["max_lines"],
+            int(kwargs.get("is_func", False)),
+            kwargs["max_functions"],
+            snippet_dict["content"][:100],
+        )
+
+        strict = kwargs["strict"]
+        result_chunks = kwargs["result_chunks"]
+
+        if strict:
+            raise TokenLimitError(
+                f"Structural block exceeds maximum limit.\n{limit_msg}\n"
+                "Reason: Prevent splitting inside interest points (function, class, region, ...)\n"
+                "💡Hint: Consider increasing 'max_tokens', 'max_lines', or 'max_functions', "
+                "refactoring the oversized block, or setting 'strict=False' to allow automatic splitting of oversized blocks."
+            )
+
+        logger.warning(
+            "Splitting oversized block into sub-chunks.\n(%s)",
+            limit_msg,
+        )
+
+        sub_chunks = self._split_oversized(
+            snippet_dict,
+            kwargs["max_tokens"],
+            kwargs["max_lines"],
+            kwargs["source"],
+            kwargs["token_counter"],
+            kwargs["cumulative_lengths"],
+        )
+
+        for sub_chunk in sub_chunks:
+            sub_chunk.metadata.chunk_num = len(result_chunks) + 1
+            result_chunks.append(sub_chunk)
+
     def _group_by_chunk(
         self,
         snippet_dicts: list[dict],
@@ -359,41 +416,23 @@ class CodeChunker(BaseChunker):
 
             elif not merged_content:
                 # Too big and nothing merged yet: handle oversize
-                limit_msg = self._format_limit_msg(
-                    box_tokens,
-                    max_tokens,
-                    box_lines,
-                    max_lines,
-                    function_count,
-                    max_functions,
-                    snippet_dict["content"][:100],
+                self._handle_oversized_snippet(
+                    snippet_dict,
+                    **{
+                        "max_tokens": max_tokens,
+                        "max_lines": max_lines,
+                        "max_functions": max_functions,
+                        "strict": strict,
+                        "source": source,
+                        "token_counter": token_counter,
+                        "cumulative_lengths": cumulative_lengths,
+                        "result_chunks": result_chunks,
+                        "box_tokens": box_tokens,
+                        "box_lines": box_lines,
+                        "is_func": is_function,
+                    },
                 )
-                if strict:
-                    raise TokenLimitError(
-                        f"Structural block exceeds maximum limit.\n{limit_msg}\n"
-                        "Reason: Prevent splitting inside interest points (function, class, region, ...)\n"
-                        "💡Hint: Consider increasing 'max_tokens', 'max_lines', or 'max_functions', "
-                        "refactoring the oversized block, or setting 'strict=False' to allow automatic splitting of oversized blocks."
-                    )
-                else:  # Else split further
-                    logger.warning(
-                        "Splitting oversized block into sub-chunks.\n(%s)",
-                        limit_msg,
-                    )
-
-                    sub_chunks = self._split_oversized(
-                        snippet_dict,
-                        max_tokens,
-                        max_lines,
-                        source,
-                        token_counter,
-                        cumulative_lengths,
-                    )
-
-                    for sub_chunk in sub_chunks:
-                        sub_chunk.metadata.chunk_num = len(result_chunks) + 1
-                        result_chunks.append(sub_chunk)
-                    index += 1
+                index += 1
             else:
                 # Flush current merged content as a chunk
                 start_span = cumulative_lengths[start_line - 1]

{chunklet_py-2.3.1 → chunklet_py-2.4.0}/src/chunklet/common/deprecation.py

@@ -1,9 +1,10 @@
 import functools
 import warnings
 from importlib.metadata import PackageNotFoundError, version
-from packaging.version import InvalidVersion, Version
 from typing import Any, Callable
 
+from packaging.version import Version
+
 try:
     CURRENT_VERSION = version("chunklet-py")
 except PackageNotFoundError:  # pragma: no cover