tokenshrink 0.2.0__py3-none-any.whl → 0.2.2__py3-none-any.whl

tokenshrink/__init__.py

@@ -23,7 +23,13 @@ CLI:
     tokenshrink stats
 """
 
-from tokenshrink.pipeline import TokenShrink, ShrinkResult, ChunkScore
+__version__ = "0.2.2"
+
+def __getattr__(name):
+    if name in ("TokenShrink", "ShrinkResult", "ChunkScore"):
+        from tokenshrink.pipeline import TokenShrink, ShrinkResult, ChunkScore
+        globals().update({"TokenShrink": TokenShrink, "ShrinkResult": ShrinkResult, "ChunkScore": ChunkScore})
+        return globals()[name]
+    raise AttributeError(f"module 'tokenshrink' has no attribute {name!r}")
 
-__version__ = "0.2.0"
 __all__ = ["TokenShrink", "ShrinkResult", "ChunkScore"]

tokenshrink/__main__.py

@@ -0,0 +1,4 @@
+"""Allow running with `python -m tokenshrink`."""
+from tokenshrink.cli import main
+
+main()

tokenshrink/cli.py

@@ -13,7 +13,7 @@ import sys
 import json
 from pathlib import Path
 
-from tokenshrink import TokenShrink, __version__
+from tokenshrink import __version__
 
 
 def main():
@@ -32,6 +32,11 @@ def main():
         action="store_true",
         help="Output as JSON",
     )
+    parser.add_argument(
+        "--quiet",
+        action="store_true",
+        help="Suppress model loading messages",
+    )
     
     subparsers = parser.add_subparsers(dest="command", help="Commands")
     
@@ -118,6 +123,20 @@ def main():
         parser.print_help()
         sys.exit(0)
     
+    # Suppress noisy output when --quiet or --json
+    if args.quiet or args.json:
+        import os, logging, warnings
+        os.environ["TRANSFORMERS_VERBOSITY"] = "error"
+        os.environ["HF_HUB_DISABLE_PROGRESS_BARS"] = "1"
+        os.environ["TOKENIZERS_PARALLELISM"] = "false"
+        logging.getLogger("sentence_transformers").setLevel(logging.ERROR)
+        logging.getLogger("transformers").setLevel(logging.ERROR)
+        logging.getLogger("huggingface_hub").setLevel(logging.ERROR)
+        warnings.filterwarnings("ignore", message=".*unauthenticated.*")
+    
+    # Lazy import to avoid loading ML models for --help/--version
+    from tokenshrink import TokenShrink
+    
     # Determine compression setting
     compression = True
     if hasattr(args, 'no_compress') and args.no_compress:
@@ -195,6 +214,9 @@ def main():
                 print(f"Sources: {', '.join(Path(s).name for s in result.sources)}")
                 print(f"Stats: {result.savings}")
                 
+                if result.savings_pct == 0.0:
+                    print("  Tip: Install llmlingua for compression: pip install llmlingua")
+                
                 if getattr(args, 'scores', False) and result.chunk_scores:
                     print("\nChunk Importance Scores:")
                     for cs in result.chunk_scores:

tokenshrink/pipeline.py

@@ -613,7 +613,7 @@ class TokenShrink:
             "ratio": total_compressed / total_original if total_original else 1.0,
         }
 
-    def search(self, question: str, k: int = 5, min_score: float = 0.3) -> list[dict]:
+    def search(self, question: str, k: int = 5, min_score: float = 0.15) -> list[dict]:
         """Search without compression. Returns raw chunks with scores."""
         if self._index.ntotal == 0:
             return []

{tokenshrink-0.2.0.dist-info → tokenshrink-0.2.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tokenshrink
-Version: 0.2.0
+Version: 0.2.2
 Summary: Cut your AI costs 50-80%. FAISS retrieval + LLMLingua compression + REFRAG-inspired adaptive optimization.
 Project-URL: Homepage, https://tokenshrink.dev
 Project-URL: Repository, https://github.com/MusashiMiyamoto1-cloud/tokenshrink

tokenshrink-0.2.2.dist-info/RECORD

@@ -0,0 +1,9 @@
+tokenshrink/__init__.py,sha256=ve3m9BZp1ti-U76YWmI63vPU8_C2wx8ZckJTr2Quh70,1195
+tokenshrink/__main__.py,sha256=yEx25Vj4itzYPdLxHsJP3MVjKM75CGdP97WIETxGljQ,91
+tokenshrink/cli.py,sha256=5F6YB_ZLPyf77AsDKKz0IPLsmgbSt0ivAsElzB6yKqU,9058
+tokenshrink/pipeline.py,sha256=ayFJVr_esO-L0Z0JWerLcYuf83te5umAEJ6e6l-TDns,24056
+tokenshrink-0.2.2.dist-info/METADATA,sha256=cbm_6-wOyuCd7D42wNO1NWDtJpZJka4ARSw2lMHwrlo,10615
+tokenshrink-0.2.2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+tokenshrink-0.2.2.dist-info/entry_points.txt,sha256=vwr3PMC25J8f-ppDVngO3MmXuY_cdR2rNM_syUmT7lc,53
+tokenshrink-0.2.2.dist-info/licenses/LICENSE,sha256=LsUNAvKJnhwbhmOWCjLq-Zf0HllrifthQ9TZkv1UUig,1064
+tokenshrink-0.2.2.dist-info/RECORD,,

tokenshrink-0.2.0.dist-info/RECORD

@@ -1,8 +0,0 @@
-tokenshrink/__init__.py,sha256=ybxGRkBPQTLIBckBYSZxebHKlAilMOoWVJxOyhv1Hgw,883
-tokenshrink/cli.py,sha256=dmP1BPbMow_NBm8fFXo05vJlU4vgyhDuzxL5q1a6n20,8102
-tokenshrink/pipeline.py,sha256=H3T3UlvHOIc1VOVyNFL-HEP0Cf_v7fVlAY-BFVT4V4w,24055
-tokenshrink-0.2.0.dist-info/METADATA,sha256=quql1c1tRTp7lF4t1YOVEFGx1dT8tf3bXD04KvK0TW4,10615
-tokenshrink-0.2.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-tokenshrink-0.2.0.dist-info/entry_points.txt,sha256=vwr3PMC25J8f-ppDVngO3MmXuY_cdR2rNM_syUmT7lc,53
-tokenshrink-0.2.0.dist-info/licenses/LICENSE,sha256=LsUNAvKJnhwbhmOWCjLq-Zf0HllrifthQ9TZkv1UUig,1064
-tokenshrink-0.2.0.dist-info/RECORD,,