lopace 0.1.6.dev2__tar.gz → 0.1.6.dev3__tar.gz

{lopace-0.1.6.dev2 → lopace-0.1.6.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lopace
-Version: 0.1.6.dev2
+Version: 0.1.6.dev3
 Summary: Lossless Optimized Prompt Accurate Compression Engine
 Home-page: https://github.com/connectaman/LoPace
 Author: Aman Ulla

{lopace-0.1.6.dev2 → lopace-0.1.6.dev3}/lopace/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.1.6.dev2'
-__version_tuple__ = version_tuple = (0, 1, 6, 'dev2')
+__version__ = version = '0.1.6.dev3'
+__version_tuple__ = version_tuple = (0, 1, 6, 'dev3')
 
-__commit_id__ = commit_id = 'g18b00919a'
+__commit_id__ = commit_id = 'gb8dad2406'

{lopace-0.1.6.dev2 → lopace-0.1.6.dev3}/lopace/compressor.py

@@ -126,7 +126,8 @@ class PromptCompressor:
             >>> original = compressor.decompress_token(compressed)
         """
         # Step 1: Convert text to list of token IDs
-        token_ids = list(self.tokenizer.encode(text))  # Ensure it's a list
+        # Allow special tokens to be encoded as normal text (disable the check)
+        token_ids = list(self.tokenizer.encode(text, disallowed_special=()))  # Ensure it's a list
         
         if not token_ids:
             # Empty token list - return just format byte
@@ -221,7 +222,8 @@ class PromptCompressor:
             >>> original = compressor.decompress_hybrid(compressed)
         """
         # Step 1: Tokenize
-        tokens = list(self.tokenizer.encode(text))  # Ensure it's a list
+        # Allow special tokens to be encoded as normal text (disable the check)
+        tokens = list(self.tokenizer.encode(text, disallowed_special=()))  # Ensure it's a list
         
         if not tokens:
             # Empty token list - return compressed empty data
@@ -393,7 +395,7 @@ class PromptCompressor:
         original_size = len(text.encode('utf-8'))
         stats = {
             'original_size_bytes': original_size,
-            'original_size_tokens': len(self.tokenizer.encode(text)),
+            'original_size_tokens': len(self.tokenizer.encode(text, disallowed_special=())),
             'methods': {}
         }
         

{lopace-0.1.6.dev2 → lopace-0.1.6.dev3}/lopace.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lopace
-Version: 0.1.6.dev2
+Version: 0.1.6.dev3
 Summary: Lossless Optimized Prompt Accurate Compression Engine
 Home-page: https://github.com/connectaman/LoPace
 Author: Aman Ulla

{lopace-0.1.6.dev2 → lopace-0.1.6.dev3}/lopace.egg-info/SOURCES.txt

@@ -38,6 +38,12 @@ screenshots/comprehensive_comparison.svg
 screenshots/compression-pipeline.png
 screenshots/compression_ratio.png
 screenshots/compression_ratio.svg
+screenshots/dataset_analysis.csv
+screenshots/dataset_eda.png
+screenshots/dataset_eda.svg
+screenshots/dataset_percentiles.png
+screenshots/dataset_percentiles.svg
+screenshots/dataset_statistics.json
 screenshots/disk_size_comparison.png
 screenshots/disk_size_comparison.svg
 screenshots/logo-text.png