chunklet-py 2.3.1__tar.gz → 2.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/LICENSE +1 -1
- {chunklet_py-2.3.1/src/chunklet_py.egg-info → chunklet_py-2.3.2}/PKG-INFO +1 -2
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/pyproject.toml +1 -2
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/base_chunker.py +1 -1
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/code_chunker/code_chunker.py +1 -1
- chunklet_py-2.3.2/src/chunklet/common/dotdict.py +106 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/document_chunker/_plain_text_chunker.py +1 -1
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/document_chunker/document_chunker.py +1 -1
- {chunklet_py-2.3.1 → chunklet_py-2.3.2/src/chunklet_py.egg-info}/PKG-INFO +1 -2
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet_py.egg-info/SOURCES.txt +1 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet_py.egg-info/requires.txt +0 -1
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/README.md +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/setup.cfg +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/__init__.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/cli.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/code_chunker/__init__.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/code_chunker/_code_structure_extractor.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/code_chunker/patterns.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/code_chunker/utils.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/common/__init__.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/common/batch_runner.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/common/deprecation.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/common/logging_utils.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/common/path_utils.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/common/token_utils.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/common/validation.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/document_chunker/__init__.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/document_chunker/converters/__init__.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/document_chunker/converters/html_2_md.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/document_chunker/converters/latex_2_md.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/document_chunker/converters/rst_2_md.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/document_chunker/converters/table_2_md.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/document_chunker/processors/__init__.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/document_chunker/processors/base_processor.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/document_chunker/processors/docx_processor.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/document_chunker/processors/epub_processor.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/document_chunker/processors/odt_processor.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/document_chunker/processors/pdf_processor.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/document_chunker/registry.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/document_chunker/span_finder.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/exceptions.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/sentence_splitter/__init__.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/sentence_splitter/_universal_splitter.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/sentence_splitter/languages.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/sentence_splitter/registry.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/sentence_splitter/sentence_splitter.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/sentence_splitter/terminators.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/visualizer/__init__.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/visualizer/static/css/style.css +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/visualizer/static/index.html +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/visualizer/static/js/app.js +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/visualizer/static/js/utils.js +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/visualizer/visualizer.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet_py.egg-info/dependency_links.txt +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet_py.egg-info/entry_points.txt +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet_py.egg-info/top_level.txt +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/tests/test_code_chunking.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/tests/test_document_chunking.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/tests/test_plain_text_chunking.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/tests/test_sentence_splitting.py +0 -0
- {chunklet_py-2.3.1 → chunklet_py-2.3.2}/tests/test_visualization.py +0 -0
|
@@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
|
18
18
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
19
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
20
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
-
SOFTWARE.
|
|
21
|
+
SOFTWARE.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: chunklet-py
|
|
3
|
-
Version: 2.3.
|
|
3
|
+
Version: 2.3.2
|
|
4
4
|
Summary: High-fidelity context-aware chunking and interactive visualization for RAG. Advanced segmentation for code and documents, because your LLM is only as smart as the fragments you feed it.
|
|
5
5
|
Author-email: speedyk_005 <speedy40115719@gmail.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -33,7 +33,6 @@ Requires-Dist: sentencex<2.0,>=1.0.0; sys_platform != "linux" or (platform_machi
|
|
|
33
33
|
Requires-Dist: indic-nlp-library<1.0,>=0.92
|
|
34
34
|
Requires-Dist: py3langid<1.0,>=0.3.0
|
|
35
35
|
Requires-Dist: mpire<3.0,>=2.10.2
|
|
36
|
-
Requires-Dist: dotdict3<2.0,>=1.1.0
|
|
37
36
|
Requires-Dist: more-itertools<12.0,>=10.6.0
|
|
38
37
|
Requires-Dist: regex>=2025.7.29
|
|
39
38
|
Requires-Dist: pydantic<3.0,>=2.11.0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "chunklet-py"
|
|
3
|
-
version = "2.3.
|
|
3
|
+
version = "2.3.2"
|
|
4
4
|
description = "High-fidelity context-aware chunking and interactive visualization for RAG. Advanced segmentation for code and documents, because your LLM is only as smart as the fragments you feed it."
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "speedyk_005", email = "speedy40115719@gmail.com" }
|
|
@@ -45,7 +45,6 @@ dependencies = [
|
|
|
45
45
|
"indic-nlp-library>=0.92,<1.0",
|
|
46
46
|
"py3langid>=0.3.0,<1.0",
|
|
47
47
|
"mpire>=2.10.2,<3.0",
|
|
48
|
-
"dotdict3>=1.1.0,<2.0",
|
|
49
48
|
"more-itertools>=10.6.0,<12.0",
|
|
50
49
|
"regex>=2025.7.29",
|
|
51
50
|
"pydantic>=2.11.0,<3.0",
|
|
@@ -29,7 +29,7 @@ from itertools import chain
|
|
|
29
29
|
from pathlib import Path
|
|
30
30
|
from typing import Annotated, Any, Callable, Generator, Literal
|
|
31
31
|
|
|
32
|
-
from
|
|
32
|
+
from chunklet.common.dotdict import DotDict
|
|
33
33
|
from more_itertools import unique_everseen
|
|
34
34
|
from pydantic import Field
|
|
35
35
|
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""
|
|
2
|
+
DotDict/DotList with Box-compatible serialization.
|
|
3
|
+
|
|
4
|
+
This module embeds code originally from ``dotdict3`` (MIT License).
|
|
5
|
+
Co-authored by:
|
|
6
|
+
- speedyk-005
|
|
7
|
+
- Patrick Elmer <patrick@elmer.ws> (https://github.com/dotdict/dotdict3)
|
|
8
|
+
|
|
9
|
+
Modifications: added ``to_dict()``, ``to_json()``, ``to_yaml()``,
|
|
10
|
+
``to_msgpack()``, ``to_toml()``, ``to_csv()`` for backward compatibility
|
|
11
|
+
with the python-box API.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class DotDict(dict):
|
|
18
|
+
def __init__(self, data=None):
|
|
19
|
+
if data is not None:
|
|
20
|
+
for key, value in data.items():
|
|
21
|
+
self[key] = value
|
|
22
|
+
|
|
23
|
+
def __setitem__(self, key, value):
|
|
24
|
+
return super().__setitem__(key, _convert(value))
|
|
25
|
+
|
|
26
|
+
# Redirect attribute operations to dictionary methods
|
|
27
|
+
__delattr__ = dict.__delitem__
|
|
28
|
+
__getattr__ = dict.__getitem__
|
|
29
|
+
__setattr__ = __setitem__
|
|
30
|
+
|
|
31
|
+
def to_dict(self):
|
|
32
|
+
return {k: _to_plain(v) for k, v in self.items()}
|
|
33
|
+
|
|
34
|
+
def to_json(self, filename=None, **kwargs):
|
|
35
|
+
if filename:
|
|
36
|
+
with open(filename, "w", encoding="utf-8") as f:
|
|
37
|
+
json.dump(self.to_dict(), f, ensure_ascii=False, **kwargs)
|
|
38
|
+
else:
|
|
39
|
+
return json.dumps(self.to_dict(), ensure_ascii=False, **kwargs)
|
|
40
|
+
|
|
41
|
+
def to_yaml(self, filename=None, default_flow_style=False, **kwargs):
|
|
42
|
+
try:
|
|
43
|
+
import yaml
|
|
44
|
+
except ImportError:
|
|
45
|
+
raise ImportError("pyyaml required: pip install pyyaml") from None
|
|
46
|
+
if filename:
|
|
47
|
+
with open(filename, "w", encoding="utf-8") as f:
|
|
48
|
+
yaml.dump(self.to_dict(), f, default_flow_style=default_flow_style, **kwargs)
|
|
49
|
+
else:
|
|
50
|
+
return yaml.dump(self.to_dict(), default_flow_style=default_flow_style, **kwargs)
|
|
51
|
+
|
|
52
|
+
def to_toml(self, filename=None):
|
|
53
|
+
try:
|
|
54
|
+
import toml
|
|
55
|
+
except ImportError:
|
|
56
|
+
raise ImportError("toml required: pip install toml") from None
|
|
57
|
+
if filename:
|
|
58
|
+
with open(filename, "w", encoding="utf-8") as f:
|
|
59
|
+
toml.dump(self.to_dict(), f)
|
|
60
|
+
else:
|
|
61
|
+
return toml.dumps(self.to_dict())
|
|
62
|
+
|
|
63
|
+
def to_msgpack(self, filename=None, **kwargs):
|
|
64
|
+
try:
|
|
65
|
+
import msgpack
|
|
66
|
+
except ImportError:
|
|
67
|
+
raise ImportError("msgpack required: pip install msgpack") from None
|
|
68
|
+
if filename:
|
|
69
|
+
with open(filename, "wb") as f:
|
|
70
|
+
msgpack.pack(self.to_dict(), f, **kwargs)
|
|
71
|
+
else:
|
|
72
|
+
return msgpack.packb(self.to_dict(), **kwargs)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class DotList(list):
|
|
76
|
+
def __init__(self, items=None):
|
|
77
|
+
if items is not None:
|
|
78
|
+
for item in items:
|
|
79
|
+
self.append(item)
|
|
80
|
+
|
|
81
|
+
def append(self, items):
|
|
82
|
+
return super().append(_convert(items))
|
|
83
|
+
|
|
84
|
+
def insert(self, index, items):
|
|
85
|
+
return super().insert(index, _convert(items))
|
|
86
|
+
|
|
87
|
+
def to_dict(self):
|
|
88
|
+
return [_to_plain(v) for v in self]
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _convert(obj):
|
|
92
|
+
"""Recursively converts dicts/lists to DotDict/DotList if not already converted."""
|
|
93
|
+
if isinstance(obj, dict) and not isinstance(obj, DotDict):
|
|
94
|
+
return DotDict(obj)
|
|
95
|
+
if isinstance(obj, list) and not isinstance(obj, DotList):
|
|
96
|
+
return DotList(obj)
|
|
97
|
+
return obj
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _to_plain(obj):
|
|
101
|
+
"""Recursively converts DotDict/DotList back to plain dict/list."""
|
|
102
|
+
if isinstance(obj, DotDict):
|
|
103
|
+
return {k: _to_plain(v) for k, v in obj.items()}
|
|
104
|
+
if isinstance(obj, (DotList, tuple)):
|
|
105
|
+
return [_to_plain(v) for v in obj]
|
|
106
|
+
return obj
|
{chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/document_chunker/_plain_text_chunker.py
RENAMED
|
@@ -5,7 +5,7 @@ from collections.abc import Iterable
|
|
|
5
5
|
from functools import partial
|
|
6
6
|
from typing import Annotated, Any, Callable, Generator, Literal
|
|
7
7
|
|
|
8
|
-
from
|
|
8
|
+
from chunklet.common.dotdict import DotDict
|
|
9
9
|
from loguru import logger
|
|
10
10
|
from pydantic import Field
|
|
11
11
|
|
|
@@ -3,7 +3,7 @@ from itertools import chain, tee
|
|
|
3
3
|
from pathlib import Path
|
|
4
4
|
from typing import Annotated, Any, Callable, Generator, Iterable, Literal
|
|
5
5
|
|
|
6
|
-
from
|
|
6
|
+
from chunklet.common.dotdict import DotDict
|
|
7
7
|
from loguru import logger
|
|
8
8
|
from more_itertools import ilen, split_at
|
|
9
9
|
from pydantic import Field
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: chunklet-py
|
|
3
|
-
Version: 2.3.
|
|
3
|
+
Version: 2.3.2
|
|
4
4
|
Summary: High-fidelity context-aware chunking and interactive visualization for RAG. Advanced segmentation for code and documents, because your LLM is only as smart as the fragments you feed it.
|
|
5
5
|
Author-email: speedyk_005 <speedy40115719@gmail.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -33,7 +33,6 @@ Requires-Dist: sentencex<2.0,>=1.0.0; sys_platform != "linux" or (platform_machi
|
|
|
33
33
|
Requires-Dist: indic-nlp-library<1.0,>=0.92
|
|
34
34
|
Requires-Dist: py3langid<1.0,>=0.3.0
|
|
35
35
|
Requires-Dist: mpire<3.0,>=2.10.2
|
|
36
|
-
Requires-Dist: dotdict3<2.0,>=1.1.0
|
|
37
36
|
Requires-Dist: more-itertools<12.0,>=10.6.0
|
|
38
37
|
Requires-Dist: regex>=2025.7.29
|
|
39
38
|
Requires-Dist: pydantic<3.0,>=2.11.0
|
|
@@ -13,6 +13,7 @@ src/chunklet/code_chunker/utils.py
|
|
|
13
13
|
src/chunklet/common/__init__.py
|
|
14
14
|
src/chunklet/common/batch_runner.py
|
|
15
15
|
src/chunklet/common/deprecation.py
|
|
16
|
+
src/chunklet/common/dotdict.py
|
|
16
17
|
src/chunklet/common/logging_utils.py
|
|
17
18
|
src/chunklet/common/path_utils.py
|
|
18
19
|
src/chunklet/common/token_utils.py
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/code_chunker/_code_structure_extractor.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/document_chunker/converters/__init__.py
RENAMED
|
File without changes
|
{chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/document_chunker/converters/html_2_md.py
RENAMED
|
File without changes
|
{chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/document_chunker/converters/latex_2_md.py
RENAMED
|
File without changes
|
{chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/document_chunker/converters/rst_2_md.py
RENAMED
|
File without changes
|
{chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/document_chunker/converters/table_2_md.py
RENAMED
|
File without changes
|
{chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/document_chunker/processors/__init__.py
RENAMED
|
File without changes
|
{chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/document_chunker/processors/base_processor.py
RENAMED
|
File without changes
|
{chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/document_chunker/processors/docx_processor.py
RENAMED
|
File without changes
|
{chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/document_chunker/processors/epub_processor.py
RENAMED
|
File without changes
|
{chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/document_chunker/processors/odt_processor.py
RENAMED
|
File without changes
|
{chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/document_chunker/processors/pdf_processor.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{chunklet_py-2.3.1 → chunklet_py-2.3.2}/src/chunklet/sentence_splitter/_universal_splitter.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|