tokenshrink 0.2.0__py3-none-any.whl → 0.2.1__py3-none-any.whl

tokenshrink/__init__.py

@@ -25,5 +25,5 @@ CLI:
 
 from tokenshrink.pipeline import TokenShrink, ShrinkResult, ChunkScore
 
-__version__ = "0.2.0"
+__version__ = "0.2.1"
 __all__ = ["TokenShrink", "ShrinkResult", "ChunkScore"]

tokenshrink/__main__.py

@@ -0,0 +1,4 @@
+"""Allow running with `python -m tokenshrink`."""
+from tokenshrink.cli import main
+
+main()

tokenshrink/cli.py

@@ -32,6 +32,11 @@ def main():
         action="store_true",
         help="Output as JSON",
     )
+    parser.add_argument(
+        "--quiet",
+        action="store_true",
+        help="Suppress model loading messages",
+    )
     
     subparsers = parser.add_subparsers(dest="command", help="Commands")
     
@@ -118,6 +123,17 @@ def main():
         parser.print_help()
         sys.exit(0)
     
+    # Suppress noisy output when --quiet or --json
+    if args.quiet or args.json:
+        import os, logging, warnings
+        os.environ["TRANSFORMERS_VERBOSITY"] = "error"
+        os.environ["HF_HUB_DISABLE_PROGRESS_BARS"] = "1"
+        os.environ["TOKENIZERS_PARALLELISM"] = "false"
+        logging.getLogger("sentence_transformers").setLevel(logging.ERROR)
+        logging.getLogger("transformers").setLevel(logging.ERROR)
+        logging.getLogger("huggingface_hub").setLevel(logging.ERROR)
+        warnings.filterwarnings("ignore", message=".*unauthenticated.*")
+    
     # Determine compression setting
     compression = True
     if hasattr(args, 'no_compress') and args.no_compress:
@@ -195,6 +211,9 @@ def main():
                 print(f"Sources: {', '.join(Path(s).name for s in result.sources)}")
                 print(f"Stats: {result.savings}")
                 
+                if result.savings_pct == 0.0:
+                    print("  Tip: Install llmlingua for compression: pip install llmlingua")
+                
                 if getattr(args, 'scores', False) and result.chunk_scores:
                     print("\nChunk Importance Scores:")
                     for cs in result.chunk_scores:

tokenshrink/pipeline.py

@@ -613,7 +613,7 @@ class TokenShrink:
             "ratio": total_compressed / total_original if total_original else 1.0,
         }
 
-    def search(self, question: str, k: int = 5, min_score: float = 0.3) -> list[dict]:
+    def search(self, question: str, k: int = 5, min_score: float = 0.15) -> list[dict]:
         """Search without compression. Returns raw chunks with scores."""
         if self._index.ntotal == 0:
             return []

{tokenshrink-0.2.0.dist-info → tokenshrink-0.2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tokenshrink
-Version: 0.2.0
+Version: 0.2.1
 Summary: Cut your AI costs 50-80%. FAISS retrieval + LLMLingua compression + REFRAG-inspired adaptive optimization.
 Project-URL: Homepage, https://tokenshrink.dev
 Project-URL: Repository, https://github.com/MusashiMiyamoto1-cloud/tokenshrink

tokenshrink-0.2.1.dist-info/RECORD

@@ -0,0 +1,9 @@
+tokenshrink/__init__.py,sha256=y4zMuRBrZVHYh36ZDLuf_ofZLdiXhn1FjaodfjtX4Mk,883
+tokenshrink/__main__.py,sha256=yEx25Vj4itzYPdLxHsJP3MVjKM75CGdP97WIETxGljQ,91
+tokenshrink/cli.py,sha256=lf7FmwIBlGCWmkYs3Dae_xNWEtwlIi5_cgQlUy7KeiY,8960
+tokenshrink/pipeline.py,sha256=ayFJVr_esO-L0Z0JWerLcYuf83te5umAEJ6e6l-TDns,24056
+tokenshrink-0.2.1.dist-info/METADATA,sha256=3Gc_wI9maw803HfTHvet69xISpYkWZ1iOuZidBjPso4,10615
+tokenshrink-0.2.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+tokenshrink-0.2.1.dist-info/entry_points.txt,sha256=vwr3PMC25J8f-ppDVngO3MmXuY_cdR2rNM_syUmT7lc,53
+tokenshrink-0.2.1.dist-info/licenses/LICENSE,sha256=LsUNAvKJnhwbhmOWCjLq-Zf0HllrifthQ9TZkv1UUig,1064
+tokenshrink-0.2.1.dist-info/RECORD,,

tokenshrink-0.2.0.dist-info/RECORD

@@ -1,8 +0,0 @@
-tokenshrink/__init__.py,sha256=ybxGRkBPQTLIBckBYSZxebHKlAilMOoWVJxOyhv1Hgw,883
-tokenshrink/cli.py,sha256=dmP1BPbMow_NBm8fFXo05vJlU4vgyhDuzxL5q1a6n20,8102
-tokenshrink/pipeline.py,sha256=H3T3UlvHOIc1VOVyNFL-HEP0Cf_v7fVlAY-BFVT4V4w,24055
-tokenshrink-0.2.0.dist-info/METADATA,sha256=quql1c1tRTp7lF4t1YOVEFGx1dT8tf3bXD04KvK0TW4,10615
-tokenshrink-0.2.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-tokenshrink-0.2.0.dist-info/entry_points.txt,sha256=vwr3PMC25J8f-ppDVngO3MmXuY_cdR2rNM_syUmT7lc,53
-tokenshrink-0.2.0.dist-info/licenses/LICENSE,sha256=LsUNAvKJnhwbhmOWCjLq-Zf0HllrifthQ9TZkv1UUig,1064
-tokenshrink-0.2.0.dist-info/RECORD,,