tokenshrink 0.2.1__py3-none-any.whl → 0.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tokenshrink/__init__.py +8 -2
- tokenshrink/cli.py +4 -1
- {tokenshrink-0.2.1.dist-info → tokenshrink-0.2.2.dist-info}/METADATA +1 -1
- tokenshrink-0.2.2.dist-info/RECORD +9 -0
- tokenshrink-0.2.1.dist-info/RECORD +0 -9
- {tokenshrink-0.2.1.dist-info → tokenshrink-0.2.2.dist-info}/WHEEL +0 -0
- {tokenshrink-0.2.1.dist-info → tokenshrink-0.2.2.dist-info}/entry_points.txt +0 -0
- {tokenshrink-0.2.1.dist-info → tokenshrink-0.2.2.dist-info}/licenses/LICENSE +0 -0
tokenshrink/__init__.py
CHANGED
|
@@ -23,7 +23,13 @@ CLI:
|
|
|
23
23
|
tokenshrink stats
|
|
24
24
|
"""
|
|
25
25
|
|
|
26
|
-
|
|
26
|
+
__version__ = "0.2.2"
|
|
27
|
+
|
|
28
|
+
def __getattr__(name):
|
|
29
|
+
if name in ("TokenShrink", "ShrinkResult", "ChunkScore"):
|
|
30
|
+
from tokenshrink.pipeline import TokenShrink, ShrinkResult, ChunkScore
|
|
31
|
+
globals().update({"TokenShrink": TokenShrink, "ShrinkResult": ShrinkResult, "ChunkScore": ChunkScore})
|
|
32
|
+
return globals()[name]
|
|
33
|
+
raise AttributeError(f"module 'tokenshrink' has no attribute {name!r}")
|
|
27
34
|
|
|
28
|
-
__version__ = "0.2.1"
|
|
29
35
|
__all__ = ["TokenShrink", "ShrinkResult", "ChunkScore"]
|
tokenshrink/cli.py
CHANGED
|
@@ -13,7 +13,7 @@ import sys
|
|
|
13
13
|
import json
|
|
14
14
|
from pathlib import Path
|
|
15
15
|
|
|
16
|
-
from tokenshrink import
|
|
16
|
+
from tokenshrink import __version__
|
|
17
17
|
|
|
18
18
|
|
|
19
19
|
def main():
|
|
@@ -134,6 +134,9 @@ def main():
|
|
|
134
134
|
logging.getLogger("huggingface_hub").setLevel(logging.ERROR)
|
|
135
135
|
warnings.filterwarnings("ignore", message=".*unauthenticated.*")
|
|
136
136
|
|
|
137
|
+
# Lazy import to avoid loading ML models for --help/--version
|
|
138
|
+
from tokenshrink import TokenShrink
|
|
139
|
+
|
|
137
140
|
# Determine compression setting
|
|
138
141
|
compression = True
|
|
139
142
|
if hasattr(args, 'no_compress') and args.no_compress:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tokenshrink
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: Cut your AI costs 50-80%. FAISS retrieval + LLMLingua compression + REFRAG-inspired adaptive optimization.
|
|
5
5
|
Project-URL: Homepage, https://tokenshrink.dev
|
|
6
6
|
Project-URL: Repository, https://github.com/MusashiMiyamoto1-cloud/tokenshrink
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
tokenshrink/__init__.py,sha256=ve3m9BZp1ti-U76YWmI63vPU8_C2wx8ZckJTr2Quh70,1195
|
|
2
|
+
tokenshrink/__main__.py,sha256=yEx25Vj4itzYPdLxHsJP3MVjKM75CGdP97WIETxGljQ,91
|
|
3
|
+
tokenshrink/cli.py,sha256=5F6YB_ZLPyf77AsDKKz0IPLsmgbSt0ivAsElzB6yKqU,9058
|
|
4
|
+
tokenshrink/pipeline.py,sha256=ayFJVr_esO-L0Z0JWerLcYuf83te5umAEJ6e6l-TDns,24056
|
|
5
|
+
tokenshrink-0.2.2.dist-info/METADATA,sha256=cbm_6-wOyuCd7D42wNO1NWDtJpZJka4ARSw2lMHwrlo,10615
|
|
6
|
+
tokenshrink-0.2.2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
|
|
7
|
+
tokenshrink-0.2.2.dist-info/entry_points.txt,sha256=vwr3PMC25J8f-ppDVngO3MmXuY_cdR2rNM_syUmT7lc,53
|
|
8
|
+
tokenshrink-0.2.2.dist-info/licenses/LICENSE,sha256=LsUNAvKJnhwbhmOWCjLq-Zf0HllrifthQ9TZkv1UUig,1064
|
|
9
|
+
tokenshrink-0.2.2.dist-info/RECORD,,
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
tokenshrink/__init__.py,sha256=y4zMuRBrZVHYh36ZDLuf_ofZLdiXhn1FjaodfjtX4Mk,883
|
|
2
|
-
tokenshrink/__main__.py,sha256=yEx25Vj4itzYPdLxHsJP3MVjKM75CGdP97WIETxGljQ,91
|
|
3
|
-
tokenshrink/cli.py,sha256=lf7FmwIBlGCWmkYs3Dae_xNWEtwlIi5_cgQlUy7KeiY,8960
|
|
4
|
-
tokenshrink/pipeline.py,sha256=ayFJVr_esO-L0Z0JWerLcYuf83te5umAEJ6e6l-TDns,24056
|
|
5
|
-
tokenshrink-0.2.1.dist-info/METADATA,sha256=3Gc_wI9maw803HfTHvet69xISpYkWZ1iOuZidBjPso4,10615
|
|
6
|
-
tokenshrink-0.2.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
|
|
7
|
-
tokenshrink-0.2.1.dist-info/entry_points.txt,sha256=vwr3PMC25J8f-ppDVngO3MmXuY_cdR2rNM_syUmT7lc,53
|
|
8
|
-
tokenshrink-0.2.1.dist-info/licenses/LICENSE,sha256=LsUNAvKJnhwbhmOWCjLq-Zf0HllrifthQ9TZkv1UUig,1064
|
|
9
|
-
tokenshrink-0.2.1.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|