chunklet-py 2.3.2__tar.gz → 2.4.0__tar.gz

{chunklet_py-2.3.2/src/chunklet_py.egg-info → chunklet_py-2.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: chunklet-py
-Version: 2.3.2
+Version: 2.4.0
 Summary: High-fidelity context-aware chunking and interactive visualization for RAG. Advanced segmentation for code and documents, because your LLM is only as smart as the fragments you feed it.
 Author-email: speedyk_005 <speedy40115719@gmail.com>
 License-Expression: MIT
@@ -9,7 +9,7 @@ Project-URL: Repository, https://github.com/speedyk-005/chunklet-py
 Project-URL: Documentation, https://speedyk-005.github.io/chunklet-py/
 Project-URL: Issues, https://github.com/speedyk-005/chunklet-py/issues
 Project-URL: Changelog, https://github.com/speedyk-005/chunklet-py/blob/main/CHANGELOG.md
-Keywords: nlp,chunking,text-splitting,sentence-splitting,llm,rag,ai,multilingual,text processing,natural language processing,data processing,information retrieval,semantic search,document processing,document-chunker,code chunking,code-chunker,source code analysis,programming languages,code structure,chunk-visualization
+Keywords: nlp,chunking,chunker,text-chunking,text-splitting,sentence-splitting,llm,rag,ai,multilingual,text processing,natural language processing,data processing,information retrieval,semantic search,document processing,document-chunker,code chunking,code-chunker,programming languages,code structure,chunk-visualization
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
@@ -26,8 +26,7 @@ Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: loguru<1.0,>=0.7.3
-Requires-Dist: pysbd<1.0,>=0.3.4
-Requires-Dist: sentsplit<2.0,>=1.0.8
+Requires-Dist: yasbd-lib<2.0,>=0.12.0
 Requires-Dist: sentencex<=0.6.1; sys_platform == "linux" and (platform_machine == "aarch64" or platform_machine == "armv7l")
 Requires-Dist: sentencex<2.0,>=1.0.0; sys_platform != "linux" or (platform_machine != "aarch64" and platform_machine != "armv7l")
 Requires-Dist: indic-nlp-library<1.0,>=0.92
@@ -38,7 +37,6 @@ Requires-Dist: regex>=2025.7.29
 Requires-Dist: pydantic<3.0,>=2.11.0
 Requires-Dist: typer<1.0,>=0.19.0
 Requires-Dist: charset-normalizer<4.0,>=3.4.2
-Requires-Dist: setuptools<81
 Provides-Extra: structured-document
 Requires-Dist: pdfminer.six>=20250324; extra == "structured-document"
 Requires-Dist: python-docx<2.0,>=1.2.0; extra == "structured-document"
@@ -47,10 +45,11 @@ Requires-Dist: ebooklib<1.0,>=0.19; extra == "structured-document"
 Requires-Dist: docutils<1.0,>=0.21.2; extra == "structured-document"
 Requires-Dist: markdownify<2.0,>=1.1.0; extra == "structured-document"
 Requires-Dist: striprtf<1.0,>=0.0.29; extra == "structured-document"
-Requires-Dist: pylatexenc==2.10; extra == "structured-document"
+Requires-Dist: mailparse<2.0,>=1.0.1; extra == "structured-document"
+Requires-Dist: python-pptx<2.0,>=1.0.0; extra == "structured-document"
+Requires-Dist: pylatexenc==2.11; extra == "structured-document"
 Requires-Dist: odfpy<2.0,>=1.4.1; extra == "structured-document"
 Requires-Dist: openpyxl<4.0,>=3.1.5; extra == "structured-document"
-Requires-Dist: tabulate2<2.0,>=1.10.2; extra == "structured-document"
 Provides-Extra: document
 Requires-Dist: chunklet-py[structured-document]; extra == "document"
 Provides-Extra: code
@@ -70,7 +69,6 @@ Provides-Extra: dev
 Requires-Dist: pytest>=8.3.5; extra == "dev"
 Requires-Dist: pytest-cov>=6.2.1; extra == "dev"
 Requires-Dist: pytest-mock>=3.14.1; extra == "dev"
-Requires-Dist: pytest-timeout>=2.3.1; extra == "dev"
 Requires-Dist: ruff>=0.14.14; extra == "dev"
 Provides-Extra: docs
 Requires-Dist: mike>=2.1.2; extra == "docs"
@@ -128,7 +126,7 @@ Dumb splitting causes problems:
 Smart chunking solves this by:
 
 - **Smart limits** — Respects both natural boundaries (sentences, paragraphs, sections) AND configurable limits (tokens, lines, functions)
-- **Language-aware** — Detects language automatically and applies the right rules (50+ languages supported)
+- **Language-aware** — Detects language automatically and applies the right rules (60+ languages supported)
 - **Context preservation** — Overlap between chunks, rich metadata (source, span, document structure)
 
 ## 🤔 So What's Chunklet-py Anyway? (And Why Should You Care?)
@@ -157,7 +155,7 @@ Perfect for prepping data for LLMs, building RAG systems, or powering AI search
 | 🪶 **Featherlight Footprint** | Designed to be lightweight and memory-efficient, ensuring optimal performance without unnecessary overhead. |
 | 🗂️ **Rich Metadata for RAG** | Enriches chunks with valuable, context-aware metadata (source, span, document properties, code AST details) crucial for advanced RAG and LLM applications. |
 | 🔧 **Infinitely Customizable** | Offers extensive customization options, from pluggable token counters to custom sentence splitters and processors. |
-| 🌐 **Multilingual Mastery** | Supports over 50 natural languages for text and document chunking with intelligent detection and language-specific algorithms. |
+| 🌐 **Multilingual Mastery** | Supports over 60 natural languages for text and document chunking with intelligent detection and language-specific algorithms. |
 | 🧑‍💻 **Code-Aware Intelligence** | Language-agnostic code chunking that understands and preserves the structural integrity of your source code. |
 | 🎯 **Precision Chunking** | Flexible chunking with configurable limits based on sentences, tokens, sections, lines, and functions. |
 | 📄 **Document Format Mastery** | Processes a wide array of document formats including `.pdf`, `.docx`, `.epub`, `.txt`, `.tex`, `.html`, `.hml`, `.md`, `.rst`, `.rtf`, `.odt`, `.csv`, and `.xlsx`. |
@@ -184,19 +182,21 @@ The simplest way to get started is with pip:
 
 ```bash
 # Install and check it's working
-pip install chunklet-py
+pip install chunklet-py -U
 chunklet --version
 ```
 
 > [!TIP]
 > <b>Termux (Android)</b>
 >
-> No rust toolchain on Termux (especially python 3.13)? Install pydantic-core pre-built wheels first then retry installing chunklet-py:
+> No rust toolchain on Termux (especially python 3.13)? Pydantic-core and cryptography won't build from source.
+> Install pre-built wheels and dependencies:
 >
 > ```bash
 > pip install typing-extensions
 > pip install pydantic-core --index-url https://termux-user-repository.github.io/pypi/
 > pip install "pydantic>=2.12.4,<2.13"
+> pkg install python-cryptography
 > ```
 
 That's it! You're all set to start chunking.
@@ -216,6 +216,8 @@ Want to unlock more Chunklet-py superpowers? Add these optional dependencies bas
 *   **Visualization:** For the interactive web-based chunk visualizer:
     ```bash
     pip install "chunklet-py[visualization]"
+    # Or
+    pip install "chunklet-py[viz]"
     ```
 * **All Extras:** To install all optional dependencies:
     ```bash
@@ -351,6 +353,13 @@ splitter = SentenceSplitter()
 sentences = splitter.split_text(text, lang="auto")
 ```
 
+> [!TIP]
+> **Advanced text splitting**
+>
+> Chunklet-py quietly delegates sentence splitting to [yasbd-lib](https://github.com/speedyk-005/yasbd-lib), our high-accuracy SBD supporting **39 languages**.
+>
+> Curious why we built it? Read **[yasbd-lib vs PySBD: Two Philosophies of Sentence Boundary Detection](https://dev.to/speed_k_7e1b449706e59e433/yasbd-lib-vs-pysbd-two-philosophies-of-sentence-boundary-detection-i88)**.
+
 **CLI (Command Line Interface)**
 
 If you prefer the terminal to an IDE, the CLI is packed with features. Just ask for help.

{chunklet_py-2.3.2 → chunklet_py-2.4.0}/README.md

@@ -44,7 +44,7 @@ Dumb splitting causes problems:
 Smart chunking solves this by:
 
 - **Smart limits** — Respects both natural boundaries (sentences, paragraphs, sections) AND configurable limits (tokens, lines, functions)
-- **Language-aware** — Detects language automatically and applies the right rules (50+ languages supported)
+- **Language-aware** — Detects language automatically and applies the right rules (60+ languages supported)
 - **Context preservation** — Overlap between chunks, rich metadata (source, span, document structure)
 
 ## 🤔 So What's Chunklet-py Anyway? (And Why Should You Care?)
@@ -73,7 +73,7 @@ Perfect for prepping data for LLMs, building RAG systems, or powering AI search
 | 🪶 **Featherlight Footprint** | Designed to be lightweight and memory-efficient, ensuring optimal performance without unnecessary overhead. |
 | 🗂️ **Rich Metadata for RAG** | Enriches chunks with valuable, context-aware metadata (source, span, document properties, code AST details) crucial for advanced RAG and LLM applications. |
 | 🔧 **Infinitely Customizable** | Offers extensive customization options, from pluggable token counters to custom sentence splitters and processors. |
-| 🌐 **Multilingual Mastery** | Supports over 50 natural languages for text and document chunking with intelligent detection and language-specific algorithms. |
+| 🌐 **Multilingual Mastery** | Supports over 60 natural languages for text and document chunking with intelligent detection and language-specific algorithms. |
 | 🧑‍💻 **Code-Aware Intelligence** | Language-agnostic code chunking that understands and preserves the structural integrity of your source code. |
 | 🎯 **Precision Chunking** | Flexible chunking with configurable limits based on sentences, tokens, sections, lines, and functions. |
 | 📄 **Document Format Mastery** | Processes a wide array of document formats including `.pdf`, `.docx`, `.epub`, `.txt`, `.tex`, `.html`, `.hml`, `.md`, `.rst`, `.rtf`, `.odt`, `.csv`, and `.xlsx`. |
@@ -100,19 +100,21 @@ The simplest way to get started is with pip:
 
 ```bash
 # Install and check it's working
-pip install chunklet-py
+pip install chunklet-py -U
 chunklet --version
 ```
 
 > [!TIP]
 > <b>Termux (Android)</b>
 >
-> No rust toolchain on Termux (especially python 3.13)? Install pydantic-core pre-built wheels first then retry installing chunklet-py:
+> No rust toolchain on Termux (especially python 3.13)? Pydantic-core and cryptography won't build from source.
+> Install pre-built wheels and dependencies:
 >
 > ```bash
 > pip install typing-extensions
 > pip install pydantic-core --index-url https://termux-user-repository.github.io/pypi/
 > pip install "pydantic>=2.12.4,<2.13"
+> pkg install python-cryptography
 > ```
 
 That's it! You're all set to start chunking.
@@ -132,6 +134,8 @@ Want to unlock more Chunklet-py superpowers? Add these optional dependencies bas
 *   **Visualization:** For the interactive web-based chunk visualizer:
     ```bash
     pip install "chunklet-py[visualization]"
+    # Or
+    pip install "chunklet-py[viz]"
     ```
 * **All Extras:** To install all optional dependencies:
     ```bash
@@ -267,6 +271,13 @@ splitter = SentenceSplitter()
 sentences = splitter.split_text(text, lang="auto")
 ```
 
+> [!TIP]
+> **Advanced text splitting**
+>
+> Chunklet-py quietly delegates sentence splitting to [yasbd-lib](https://github.com/speedyk-005/yasbd-lib), our high-accuracy SBD supporting **39 languages**.
+>
+> Curious why we built it? Read **[yasbd-lib vs PySBD: Two Philosophies of Sentence Boundary Detection](https://dev.to/speed_k_7e1b449706e59e433/yasbd-lib-vs-pysbd-two-philosophies-of-sentence-boundary-detection-i88)**.
+
 **CLI (Command Line Interface)**
 
 If you prefer the terminal to an IDE, the CLI is packed with features. Just ask for help.

{chunklet_py-2.3.2 → chunklet_py-2.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "chunklet-py"
-version = "2.3.2"
+version = "2.4.0"
 description = "High-fidelity context-aware chunking and interactive visualization for RAG. Advanced segmentation for code and documents, because your LLM is only as smart as the fragments you feed it."
 authors = [
   { name = "speedyk_005", email = "speedy40115719@gmail.com" }
@@ -26,17 +26,16 @@ classifiers = [
 ]
 
 keywords = [
-  "nlp", "chunking", "text-splitting", "sentence-splitting", "llm", "rag", "ai", "multilingual",
+  "nlp", "chunking", "chunker", "text-chunking", "text-splitting", "sentence-splitting", "llm", "rag", "ai", "multilingual",
   "text processing", "natural language processing", "data processing",
   "information retrieval", "semantic search", "document processing", "document-chunker",
-  "code chunking", "code-chunker", "source code analysis", "programming languages",
+  "code chunking", "code-chunker", "programming languages",
   "code structure", "chunk-visualization",
 ]
 
 dependencies = [
   "loguru>=0.7.3,<1.0",
-  "pysbd>=0.3.4,<1.0",
-  "sentsplit>=1.0.8,<2.0",
+  "yasbd-lib>=0.12.0,<2.0",
 
   # Rust binding (1.0+) for faster performance on supported platforms, legacy for Android temporarily.
   "sentencex<=0.6.1; sys_platform == 'linux' and (platform_machine == 'aarch64' or platform_machine == 'armv7l')",
@@ -50,9 +49,6 @@ dependencies = [
   "pydantic>=2.11.0,<3.0",
   "typer>=0.19.0,<1.0",
   "charset-normalizer>=3.4.2,<4.0",
-
-  # sentsplit depends on deprecated pkg_resources removed in setuptools 81.0.0
-  "setuptools<81",
 ]
 
 [project.optional-dependencies]
@@ -64,13 +60,14 @@ structured-document = [
   "docutils>=0.21.2,<1.0",
   "markdownify>=1.1.0,<2.0",
   "striprtf>=0.0.29,<1.0",
+  "mailparse>=1.0.1,<2.0",
+  "python-pptx>=1.0.0,<2.0",
 
   # Lastest version of 2.x.x series
-  "pylatexenc==2.10",
+  "pylatexenc==2.11",
   
   "odfpy>=1.4.1,<2.0",
   "openpyxl>=3.1.5,<4.0",
-  "tabulate2>=1.10.2,<2.0",
 ]
 
 # Kept for compatibility
@@ -99,7 +96,6 @@ dev = [
   "pytest>=8.3.5",
   "pytest-cov>=6.2.1",
   "pytest-mock>=3.14.1",
-  "pytest-timeout>=2.3.1",
   "ruff>=0.14.14",
 ]
 
@@ -151,14 +147,13 @@ chunklet = ["visualizer/static/**/*"]
 minversion = "7.4"
 filterwarnings = [
     "ignore::DeprecationWarning:websockets.*",
-    "ignore::DeprecationWarning:pkg_resources.*",
+    "ignore::DeprecationWarning:multiprocessing.*",
 ]
 
 # disable coverage report to prevent racing with multiprocessing
 # addopts = "--cov=chunklet --cov-report=term-missing:skip-covered --durations=10"
-addopts = "--timeout=60"
-
-testpaths = ["tests"]
+addopts = "--doctest-modules"
+testpaths = ["tests", "src/chunklet/common/dotdict.py"]
 python_files = "test_*.py"
 
 [tool.coverage.run]
@@ -169,7 +164,6 @@ omit = [
     "src/chunklet/**/__init__.py",
 ]
 
-
 [tool.ruff]
 target-version = "py310"
 line-length = 88
@@ -193,9 +187,6 @@ ignore = ["E501", "E203"]
 # Ignore warnings about File(...)
 "src/chunklet/visualizer/visualizer.py" = ["B008"]
 
-# Ignore E402 - warnings.filterwarnings() must be called before imports to suppress pkg_resources
-"src/chunklet/sentence_splitter/sentence_splitter.py" = ["E402"]
-
 [tool.ruff.format]
 # Replaces Black exclude logic
 exclude = [

{chunklet_py-2.3.2 → chunklet_py-2.4.0}/src/chunklet/base_chunker.py

@@ -8,7 +8,6 @@ from abc import ABC, abstractmethod
 from collections.abc import Generator
 
 from chunklet.common.dotdict import DotDict
-from loguru import logger
 
 
 class BaseChunker(ABC):

{chunklet_py-2.3.2 → chunklet_py-2.4.0}/src/chunklet/cli.py

@@ -328,7 +328,7 @@ def split_command(
             destination.write_text(output_str, encoding="utf-8")
             typer.echo(
                 f"Successfully split and wrote {len(sentences)} sentences "
-                f"{source_display} to {destination} (Confidence: {confidence})",
+                f"{source_display} to {destination} (Lang: {lang_detected}, Confidence: {confidence})",
                 err=True,
             )
         except Exception as e:
@@ -339,7 +339,7 @@ def split_command(
 
         typer.echo(
             f"--- Sentences ({len(sentences)}): "
-            f" [{source_display} | Lang: {lang.upper()} | Confidence: {confidence}] ---"
+            f" [{source_display} | Lang: {lang_detected} | Confidence: {confidence}] ---"
         )
 
         for sentence in sentences:

{chunklet_py-2.3.2 → chunklet_py-2.4.0}/src/chunklet/code_chunker/_code_structure_extractor.py

@@ -19,7 +19,6 @@ try:
 except ImportError:  # pragma: no cover
     Node, ET = None, None
 
-from loguru import logger
 
 from chunklet.code_chunker.patterns import (
     ALL_SINGLE_LINE_COMM,
@@ -34,9 +33,8 @@ from chunklet.code_chunker.patterns import (
     NAMESPACE_DECLARATION,
     OPENER,
 )
-from chunklet.common.validation import validate_input
 from chunklet.common.logging_utils import log_info
-
+from chunklet.common.validation import validate_input
 
 CodeLine = namedtuple(
     "CodeLine", ["line_number", "content", "indent_level", "func_partial_signature"]
@@ -316,7 +314,9 @@ class CodeStructureExtractor:
         if not (curr_struct or annotated_lines_buffer):
             return
 
-        candidates = [entry for v in annotated_lines_buffer.values() for entry in v] + curr_struct
+        candidates = [
+            entry for v in annotated_lines_buffer.values() for entry in v
+        ] + curr_struct
         sorted_candidates = sorted(candidates, key=lambda x: x.line_number)
 
         if not sorted_candidates:
@@ -372,13 +372,18 @@ class CodeStructureExtractor:
         indent_level = len(deannotated_line) - len(deannotated_line.lstrip())
         first_metadata = tag == "META" and not annotated_lines_buffer["META"]
         consecutive_docstrings = (
-            annotated_lines_buffer["DOC"] and annotated_lines_buffer["DOC"][-1].line_number == line_no - 1
+            annotated_lines_buffer["DOC"]
+            and annotated_lines_buffer["DOC"][-1].line_number == line_no - 1
         )
 
         if first_metadata or not consecutive_docstrings:
-            self._flush_snippet(state["curr_struct"], state["snippet_dicts"], annotated_lines_buffer)
+            self._flush_snippet(
+                state["curr_struct"], state["snippet_dicts"], annotated_lines_buffer
+            )
 
-        annotated_lines_buffer[tag].append(CodeLine(line_no, deannotated_line, indent_level, None))
+        annotated_lines_buffer[tag].append(
+            CodeLine(line_no, deannotated_line, indent_level, None)
+        )
 
     def _handle_block_start(
         self,
@@ -431,12 +436,16 @@ class CodeStructureExtractor:
             if state["curr_struct"]:
                 if is_python_code:
                     self._flush_snippet(
-                        state["curr_struct"], state["snippet_dicts"], annotated_lines_buffer
+                        state["curr_struct"],
+                        state["snippet_dicts"],
+                        annotated_lines_buffer,
                     )
                 else:
                     doc = annotated_lines_buffer.pop("DOC", [])
                     self._flush_snippet(
-                        state["curr_struct"], state["snippet_dicts"], annotated_lines_buffer
+                        state["curr_struct"],
+                        state["snippet_dicts"],
+                        annotated_lines_buffer,
                     )
                     annotated_lines_buffer.clear()
                     annotated_lines_buffer["doc"] = doc
@@ -536,7 +545,9 @@ class CodeStructureExtractor:
 
         # Append last snippet
         if state["curr_struct"]:
-            self._flush_snippet(state["curr_struct"], state["snippet_dicts"], annotated_lines_buffer)
+            self._flush_snippet(
+                state["curr_struct"], state["snippet_dicts"], annotated_lines_buffer
+            )
 
         snippet_dicts = self._post_processing(state["snippet_dicts"])
         log_info(

{chunklet_py-2.3.2 → chunklet_py-2.4.0}/src/chunklet/code_chunker/code_chunker.py

@@ -29,10 +29,11 @@ from itertools import chain
 from pathlib import Path
 from typing import Annotated, Any, Callable, Generator, Literal
 
-from chunklet.common.dotdict import DotDict
 from more_itertools import unique_everseen
 from pydantic import Field
 
+from chunklet.common.dotdict import DotDict
+
 try:
     import defusedxml.ElementTree as ET
     from charset_normalizer import from_path
@@ -50,7 +51,7 @@ from chunklet.common.deprecation import deprecated_callable
 from chunklet.common.logging_utils import log_info
 from chunklet.common.path_utils import is_path_like, read_text_file
 from chunklet.common.token_utils import count_tokens
-from chunklet.common.validation import IterableOfStr, IterableOfPath, validate_input
+from chunklet.common.validation import IterableOfPath, IterableOfStr, validate_input
 from chunklet.exceptions import (
     InvalidInputError,
     MissingTokenCounterError,
@@ -125,6 +126,7 @@ class CodeChunker(BaseChunker):
         # Deduplicate relations
         def relation_key(relation: dict):
             return tuple(sorted(relation.items()))
+
         unique_relations = list(unique_everseen(all_relations_flat, key=relation_key))
 
         if not unique_relations:
@@ -203,7 +205,7 @@ class CodeChunker(BaseChunker):
                         }
                     )
                 )
-                curr_chunk = [line]  # Add the overflow line!
+                curr_chunk = []
                 token_count = 0
                 line_count = 0
 
@@ -255,7 +257,6 @@ class CodeChunker(BaseChunker):
         Args:
             box_tokens: Actual token count in the block
             max_tokens: Maximum allowed tokens
-            box_lines: Actual line count in the block
             max_lines: Maximum allowed lines
             function_count: Actual function count in the block
             max_functions: Maximum allowed functions
@@ -278,6 +279,62 @@ class CodeChunker(BaseChunker):
             f"Content starting with: \n```\n{content_preview}...\n```"
         )
 
+    def _handle_oversized_snippet(
+        self,
+        snippet_dict: dict,
+        **kwargs: Any,
+    ) -> None:
+        """
+        Handle a snippet that exceeds constraints and cannot be merged.
+
+        Expects ``kwargs`` to contain the same keys as the local variables of
+        ``_group_by_chunk`` that are relevant to oversized handling:
+        ``max_tokens``, ``max_lines``, ``max_functions``, ``strict``,
+        ``source``, ``token_counter``, ``cumulative_lengths``,
+        ``result_chunks``, ``box_tokens``, ``box_lines``, ``is_func``.
+
+        Either raises a ``TokenLimitError`` (strict mode) or splits the
+        oversized block into sub-chunks appended to ``result_chunks``.
+        """
+        limit_msg = self._format_limit_msg(
+            kwargs.get("box_tokens", 0),
+            kwargs["max_tokens"],
+            kwargs.get("box_lines", 0),
+            kwargs["max_lines"],
+            int(kwargs.get("is_func", False)),
+            kwargs["max_functions"],
+            snippet_dict["content"][:100],
+        )
+
+        strict = kwargs["strict"]
+        result_chunks = kwargs["result_chunks"]
+
+        if strict:
+            raise TokenLimitError(
+                f"Structural block exceeds maximum limit.\n{limit_msg}\n"
+                "Reason: Prevent splitting inside interest points (function, class, region, ...)\n"
+                "💡Hint: Consider increasing 'max_tokens', 'max_lines', or 'max_functions', "
+                "refactoring the oversized block, or setting 'strict=False' to allow automatic splitting of oversized blocks."
+            )
+
+        logger.warning(
+            "Splitting oversized block into sub-chunks.\n(%s)",
+            limit_msg,
+        )
+
+        sub_chunks = self._split_oversized(
+            snippet_dict,
+            kwargs["max_tokens"],
+            kwargs["max_lines"],
+            kwargs["source"],
+            kwargs["token_counter"],
+            kwargs["cumulative_lengths"],
+        )
+
+        for sub_chunk in sub_chunks:
+            sub_chunk.metadata.chunk_num = len(result_chunks) + 1
+            result_chunks.append(sub_chunk)
+
     def _group_by_chunk(
         self,
         snippet_dicts: list[dict],
@@ -359,41 +416,23 @@ class CodeChunker(BaseChunker):
 
             elif not merged_content:
                 # Too big and nothing merged yet: handle oversize
-                limit_msg = self._format_limit_msg(
-                    box_tokens,
-                    max_tokens,
-                    box_lines,
-                    max_lines,
-                    function_count,
-                    max_functions,
-                    snippet_dict["content"][:100],
+                self._handle_oversized_snippet(
+                    snippet_dict,
+                    **{
+                        "max_tokens": max_tokens,
+                        "max_lines": max_lines,
+                        "max_functions": max_functions,
+                        "strict": strict,
+                        "source": source,
+                        "token_counter": token_counter,
+                        "cumulative_lengths": cumulative_lengths,
+                        "result_chunks": result_chunks,
+                        "box_tokens": box_tokens,
+                        "box_lines": box_lines,
+                        "is_func": is_function,
+                    },
                 )
-                if strict:
-                    raise TokenLimitError(
-                        f"Structural block exceeds maximum limit.\n{limit_msg}\n"
-                        "Reason: Prevent splitting inside interest points (function, class, region, ...)\n"
-                        "💡Hint: Consider increasing 'max_tokens', 'max_lines', or 'max_functions', "
-                        "refactoring the oversized block, or setting 'strict=False' to allow automatic splitting of oversized blocks."
-                    )
-                else:  # Else split further
-                    logger.warning(
-                        "Splitting oversized block into sub-chunks.\n(%s)",
-                        limit_msg,
-                    )
-
-                    sub_chunks = self._split_oversized(
-                        snippet_dict,
-                        max_tokens,
-                        max_lines,
-                        source,
-                        token_counter,
-                        cumulative_lengths,
-                    )
-
-                    for sub_chunk in sub_chunks:
-                        sub_chunk.metadata.chunk_num = len(result_chunks) + 1
-                        result_chunks.append(sub_chunk)
-                    index += 1
+                index += 1
             else:
                 # Flush current merged content as a chunk
                 start_span = cumulative_lengths[start_line - 1]

{chunklet_py-2.3.2 → chunklet_py-2.4.0}/src/chunklet/common/deprecation.py

@@ -1,9 +1,10 @@
 import functools
 import warnings
 from importlib.metadata import PackageNotFoundError, version
-from packaging.version import InvalidVersion, Version
 from typing import Any, Callable
 
+from packaging.version import Version
+
 try:
     CURRENT_VERSION = version("chunklet-py")
 except PackageNotFoundError:  # pragma: no cover