lopace 0.1.0__py3-none-any.whl

lopace/__init__.py

@@ -0,0 +1,11 @@
+"""
+LoPace - Lossless Optimized Prompt Accurate Compression Engine
+
+A professional Python package for compressing and decompressing prompts
+using multiple techniques: Zstd, Token-based (BPE), and Hybrid methods.
+"""
+
+from .compressor import PromptCompressor, CompressionMethod
+
+__version__ = "0.1.0"
+__all__ = ["PromptCompressor", "CompressionMethod"]

lopace/compressor.py

@@ -0,0 +1,517 @@
+"""
+Main compression module implementing Zstd, Token-based, and Hybrid compression methods.
+
+The compression algorithms used:
+- Zstd: Uses LZ77 (sliding window) and FSE (Finite State Entropy, a variant of Huffman coding)
+  internally via the zstandard library
+- Token-based: Uses BPE (Byte-Pair Encoding) via tiktoken
+- Hybrid: Combines tokenization + Zstd compression
+"""
+
+import struct
+import math
+from collections import Counter
+from enum import Enum
+from typing import Union, Tuple, Optional, Dict
+
+try:
+    import zstandard as zstd
+except ImportError:
+    zstd = None
+
+try:
+    import tiktoken
+except ImportError:
+    tiktoken = None
+
+
+class CompressionMethod(Enum):
+    """Compression methods available."""
+    ZSTD = "zstd"
+    TOKEN = "token"
+    HYBRID = "hybrid"
+
+
+class PromptCompressor:
+    """
+    Professional prompt compressor supporting multiple compression techniques.
+    
+    Methods:
+        - Zstd: Dictionary-based compression using Zstandard
+        - Token: Byte-Pair Encoding (BPE) tokenization with binary packing
+        - Hybrid: Combination of tokenization and Zstd compression
+    
+    Args:
+        model: Tokenizer model name (default: "cl100k_base")
+            Options: "cl100k_base", "p50k_base", "r50k_base", "gpt2", etc.
+        zstd_level: Zstd compression level (1-22, default: 15)
+            Higher levels provide better compression but are slower.
+    """
+    
+    def __init__(
+        self, 
+        model: str = "cl100k_base",
+        zstd_level: int = 15
+    ):
+        if zstd is None:
+            raise ImportError(
+                "zstandard is required. Install it with: pip install zstandard"
+            )
+        
+        if tiktoken is None:
+            raise ImportError(
+                "tiktoken is required. Install it with: pip install tiktoken"
+            )
+        
+        self.tokenizer = tiktoken.get_encoding(model)
+        self.zstd_level = zstd_level
+        self.model = model
+        
+        # Validate zstd_level
+        if not (1 <= zstd_level <= 22):
+            raise ValueError("zstd_level must be between 1 and 22")
+    
+    def compress_zstd(self, text: str) -> bytes:
+        """
+        Compress prompt using Zstandard algorithm.
+        
+        Args:
+            text: Original prompt string
+            
+        Returns:
+            Compressed bytes
+            
+        Example:
+            >>> compressor = PromptCompressor()
+            >>> compressed = compressor.compress_zstd("Your prompt here")
+            >>> original = compressor.decompress_zstd(compressed)
+        """
+        data_bytes = text.encode('utf-8')
+        compressed_blob = zstd.compress(data_bytes, level=self.zstd_level)
+        return compressed_blob
+    
+    def decompress_zstd(self, compressed_blob: bytes) -> str:
+        """
+        Decompress Zstandard-compressed prompt.
+        
+        Args:
+            compressed_blob: Compressed bytes from compress_zstd()
+            
+        Returns:
+            Original prompt string
+        """
+        raw_bytes = zstd.decompress(compressed_blob)
+        return raw_bytes.decode('utf-8')
+    
+    def compress_token(self, text: str) -> bytes:
+        """
+        Compress prompt using BPE tokenization and binary packing.
+        
+        This method:
+        1. Converts text to token IDs using the tokenizer
+        2. Packs token IDs as unsigned integers (uint16 or uint32)
+           - Uses uint16 (2 bytes) if all token IDs <= 65535
+           - Uses uint32 (4 bytes) if any token ID > 65535
+        
+        Args:
+            text: Original prompt string
+            
+        Returns:
+            Compressed bytes (format byte + binary-packed token IDs)
+            Format: [1 byte format flag: 0=uint16, 1=uint32][packed token IDs]
+            
+        Example:
+            >>> compressor = PromptCompressor()
+            >>> compressed = compressor.compress_token("Your prompt here")
+            >>> original = compressor.decompress_token(compressed)
+        """
+        # Step 1: Convert text to list of token IDs
+        token_ids = list(self.tokenizer.encode(text))  # Ensure it's a list
+        
+        if not token_ids:
+            # Empty token list - return just format byte
+            return struct.pack('B', 0)  # uint16 format
+        
+        # Step 2: Determine if we can use uint16 or need uint32
+        # Check if ANY token ID exceeds uint16 range (0-65535)
+        max_token_id = max(token_ids)
+        min_token_id = min(token_ids)
+        use_uint32 = (max_token_id > 65535) or (min_token_id < 0)
+        
+        # Step 3: Pack token IDs (format byte + token data)
+        format_byte = 1 if use_uint32 else 0  # 0 = uint16, 1 = uint32
+        
+        try:
+            if use_uint32:
+                # Use uint32 (4 bytes per token) - format 'I'
+                token_data = struct.pack(f'{len(token_ids)}I', *token_ids)
+            else:
+                # Use uint16 (2 bytes per token) - format 'H'
+                # Double-check all IDs fit in uint16 range
+                if max(token_ids) > 65535:
+                    # Fallback to uint32 if somehow we got here
+                    format_byte = 1
+                    token_data = struct.pack(f'{len(token_ids)}I', *token_ids)
+                else:
+                    token_data = struct.pack(f'{len(token_ids)}H', *token_ids)
+        except (struct.error, OverflowError) as e:
+            # If packing fails, fallback to uint32
+            format_byte = 1
+            token_data = struct.pack(f'{len(token_ids)}I', *token_ids)
+        
+        # Combine format byte with token data
+        binary_payload = struct.pack('B', format_byte) + token_data
+        
+        return binary_payload
+    
+    def decompress_token(self, binary_payload: bytes) -> str:
+        """
+        Decompress token-based compressed prompt.
+        
+        Args:
+            binary_payload: Compressed bytes from compress_token()
+            Format: [1 byte format flag: 0=uint16, 1=uint32][packed token IDs]
+            
+        Returns:
+            Original prompt string
+        """
+        if len(binary_payload) < 1:
+            raise ValueError("Invalid compressed data: missing format byte")
+        
+        # Step 1: Read format byte
+        format_byte = struct.unpack('B', binary_payload[0:1])[0]
+        token_data = binary_payload[1:]
+        
+        if format_byte == 1:
+            # uint32 format (4 bytes per token)
+            if len(token_data) % 4 != 0:
+                raise ValueError("Invalid compressed data: uint32 data length not divisible by 4")
+            num_tokens = len(token_data) // 4
+            token_ids = struct.unpack(f'{num_tokens}I', token_data)
+        else:
+            # uint16 format (2 bytes per token)
+            if len(token_data) % 2 != 0:
+                raise ValueError("Invalid compressed data: uint16 data length not divisible by 2")
+            num_tokens = len(token_data) // 2
+            token_ids = struct.unpack(f'{num_tokens}H', token_data)
+        
+        # Step 2: Decode token IDs back to string
+        return self.tokenizer.decode(list(token_ids))
+    
+    def compress_hybrid(self, text: str) -> bytes:
+        """
+        Compress prompt using hybrid approach (Token + Zstd).
+        
+        This is the most efficient method:
+        1. Tokenizes text to reduce redundancy
+        2. Packs tokens as binary
+        3. Applies Zstd compression on the binary data
+        
+        Provides the best compression ratio for database storage.
+        
+        Args:
+            text: Original prompt string
+            
+        Returns:
+            Compressed bytes
+            
+        Example:
+            >>> compressor = PromptCompressor()
+            >>> compressed = compressor.compress_hybrid("Your prompt here")
+            >>> original = compressor.decompress_hybrid(compressed)
+        """
+        # Step 1: Tokenize
+        tokens = list(self.tokenizer.encode(text))  # Ensure it's a list
+        
+        if not tokens:
+            # Empty token list - return compressed empty data
+            empty_data = struct.pack('B', 0)  # uint16 format
+            return zstd.compress(empty_data, level=self.zstd_level)
+        
+        # Step 2: Convert to binary (determine uint16 or uint32)
+        max_token_id = max(tokens)
+        min_token_id = min(tokens)
+        use_uint32 = (max_token_id > 65535) or (min_token_id < 0)
+        
+        format_byte = 1 if use_uint32 else 0  # 0 = uint16, 1 = uint32
+        
+        try:
+            if use_uint32:
+                # Use uint32 (4 bytes per token)
+                token_data = struct.pack('B', format_byte) + struct.pack(f'{len(tokens)}I', *tokens)
+            else:
+                # Use uint16 (2 bytes per token)
+                # Double-check all IDs fit in uint16 range
+                if max(tokens) > 65535:
+                    # Fallback to uint32 if somehow we got here
+                    format_byte = 1
+                    token_data = struct.pack('B', format_byte) + struct.pack(f'{len(tokens)}I', *tokens)
+                else:
+                    token_data = struct.pack('B', format_byte) + struct.pack(f'{len(tokens)}H', *tokens)
+        except (struct.error, OverflowError) as e:
+            # If packing fails, fallback to uint32
+            format_byte = 1
+            token_data = struct.pack('B', format_byte) + struct.pack(f'{len(tokens)}I', *tokens)
+        
+        # Step 3: Final Zstd compression
+        compressed_blob = zstd.compress(token_data, level=self.zstd_level)
+        
+        return compressed_blob
+    
+    def decompress_hybrid(self, blob: bytes) -> str:
+        """
+        Decompress hybrid-compressed prompt.
+        
+        Args:
+            blob: Compressed bytes from compress_hybrid()
+            
+        Returns:
+            Original prompt string
+        """
+        # Step 1: Decompress Zstd
+        token_data = zstd.decompress(blob)
+        
+        if len(token_data) < 1:
+            raise ValueError("Invalid compressed data: missing format byte")
+        
+        # Step 2: Read format byte and unpack token IDs
+        format_byte = struct.unpack('B', token_data[0:1])[0]
+        packed_data = token_data[1:]
+        
+        if format_byte == 1:
+            # uint32 format (4 bytes per token)
+            if len(packed_data) % 4 != 0:
+                raise ValueError("Invalid compressed data: uint32 data length not divisible by 4")
+            num_tokens = len(packed_data) // 4
+            tokens = struct.unpack(f'{num_tokens}I', packed_data)
+        else:
+            # uint16 format (2 bytes per token)
+            if len(packed_data) % 2 != 0:
+                raise ValueError("Invalid compressed data: uint16 data length not divisible by 2")
+            num_tokens = len(packed_data) // 2
+            tokens = struct.unpack(f'{num_tokens}H', packed_data)
+        
+        # Step 3: Decode to text
+        return self.tokenizer.decode(list(tokens))
+    
+    def compress(
+        self, 
+        text: str, 
+        method: CompressionMethod = CompressionMethod.HYBRID
+    ) -> bytes:
+        """
+        Compress prompt using the specified method.
+        
+        Args:
+            text: Original prompt string
+            method: Compression method to use (default: HYBRID)
+            
+        Returns:
+            Compressed bytes
+            
+        Example:
+            >>> compressor = PromptCompressor()
+            >>> compressed = compressor.compress("Your prompt", CompressionMethod.HYBRID)
+            >>> original = compressor.decompress(compressed, CompressionMethod.HYBRID)
+        """
+        if method == CompressionMethod.ZSTD:
+            return self.compress_zstd(text)
+        elif method == CompressionMethod.TOKEN:
+            return self.compress_token(text)
+        elif method == CompressionMethod.HYBRID:
+            return self.compress_hybrid(text)
+        else:
+            raise ValueError(f"Unknown compression method: {method}")
+    
+    def decompress(
+        self, 
+        compressed_data: bytes, 
+        method: CompressionMethod = CompressionMethod.HYBRID
+    ) -> str:
+        """
+        Decompress prompt using the specified method.
+        
+        Args:
+            compressed_data: Compressed bytes
+            method: Compression method used for compression
+            
+        Returns:
+            Original prompt string
+        """
+        if method == CompressionMethod.ZSTD:
+            return self.decompress_zstd(compressed_data)
+        elif method == CompressionMethod.TOKEN:
+            return self.decompress_token(compressed_data)
+        elif method == CompressionMethod.HYBRID:
+            return self.decompress_hybrid(compressed_data)
+        else:
+            raise ValueError(f"Unknown compression method: {method}")
+    
+    def compress_and_return_both(
+        self, 
+        text: str, 
+        method: CompressionMethod = CompressionMethod.HYBRID
+    ) -> Tuple[str, bytes]:
+        """
+        Compress prompt and return both original and compressed versions.
+        
+        Args:
+            text: Original prompt string
+            method: Compression method to use (default: HYBRID)
+            
+        Returns:
+            Tuple of (original_prompt, compressed_bytes)
+            
+        Example:
+            >>> compressor = PromptCompressor()
+            >>> original, compressed = compressor.compress_and_return_both("Your prompt")
+        """
+        compressed = self.compress(text, method)
+        return (text, compressed)
+    
+    def get_compression_stats(
+        self, 
+        text: str, 
+        method: Optional[CompressionMethod] = None
+    ) -> dict:
+        """
+        Get compression statistics for a given prompt.
+        
+        Args:
+            text: Original prompt string
+            method: Compression method to analyze (None = all methods)
+            
+        Returns:
+            Dictionary with compression statistics
+        """
+        methods = [method] if method else [
+            CompressionMethod.ZSTD,
+            CompressionMethod.TOKEN,
+            CompressionMethod.HYBRID
+        ]
+        
+        original_size = len(text.encode('utf-8'))
+        stats = {
+            'original_size_bytes': original_size,
+            'original_size_tokens': len(self.tokenizer.encode(text)),
+            'methods': {}
+        }
+        
+        for m in methods:
+            compressed = self.compress(text, m)
+            compressed_size = len(compressed)
+            compression_ratio = compressed_size / original_size if original_size > 0 else 0
+            space_saved = 1 - compression_ratio
+            
+            stats['methods'][m.value] = {
+                'compressed_size_bytes': compressed_size,
+                'compression_ratio': compression_ratio,
+                'space_saved_percent': space_saved * 100,
+                'bytes_saved': original_size - compressed_size
+            }
+        
+        return stats
+    
+    def calculate_shannon_entropy(self, text: str, unit: str = 'character') -> float:
+        """
+        Calculate Shannon Entropy of the input text.
+        
+        Shannon Entropy formula: H(X) = -∑ P(x_i) * log₂(P(x_i))
+        
+        This determines the theoretical lower limit of compression based on
+        character/byte frequency distribution.
+        
+        Args:
+            text: Input text to analyze
+            unit: Unit to analyze ('character' or 'byte')
+                - 'character': Analyze individual characters
+                - 'byte': Analyze bytes (for binary data)
+            
+        Returns:
+            Shannon entropy in bits
+            
+        Example:
+            >>> compressor = PromptCompressor()
+            >>> entropy = compressor.calculate_shannon_entropy("Hello world")
+            >>> print(f"Theoretical compression limit: {entropy:.2f} bits per character")
+        """
+        if not text:
+            return 0.0
+        
+        if unit == 'byte':
+            # Analyze bytes
+            data = text.encode('utf-8')
+            frequencies = Counter(data)
+        else:  # unit == 'character'
+            # Analyze characters
+            frequencies = Counter(text)
+        
+        # Calculate probabilities
+        length = len(text) if unit == 'character' else len(data)
+        probabilities = [count / length for count in frequencies.values()]
+        
+        # Calculate Shannon Entropy: H(X) = -∑ P(x_i) * log₂(P(x_i))
+        entropy = -sum(p * math.log2(p) for p in probabilities if p > 0)
+        
+        return entropy
+    
+    def get_theoretical_compression_limit(self, text: str, unit: str = 'character') -> Dict[str, float]:
+        """
+        Calculate theoretical compression limit using Shannon Entropy.
+        
+        This provides the theoretical minimum size achievable through entropy coding.
+        
+        Args:
+            text: Input text to analyze
+            unit: Unit to analyze ('character' or 'byte')
+            
+        Returns:
+            Dictionary with theoretical limits:
+            - entropy_bits_per_unit: Shannon entropy in bits
+            - theoretical_min_bits: Minimum total bits needed
+            - theoretical_min_bytes: Minimum bytes needed (theoretical limit)
+            - original_size_bytes: Original size in bytes
+            - theoretical_compression_ratio: Theoretical best compression ratio
+            
+        Example:
+            >>> compressor = PromptCompressor()
+            >>> limits = compressor.get_theoretical_compression_limit("Your prompt")
+            >>> print(f"Theoretical minimum: {limits['theoretical_min_bytes']:.2f} bytes")
+        """
+        if not text:
+            return {
+                'entropy_bits_per_unit': 0.0,
+                'theoretical_min_bits': 0.0,
+                'theoretical_min_bytes': 0.0,
+                'original_size_bytes': 0.0,
+                'theoretical_compression_ratio': 0.0
+            }
+        
+        # Calculate Shannon Entropy
+        entropy = self.calculate_shannon_entropy(text, unit)
+        
+        # Calculate theoretical minimums
+        if unit == 'byte':
+            num_units = len(text.encode('utf-8'))
+            original_size_bytes = len(text.encode('utf-8'))
+        else:  # character
+            num_units = len(text)
+            original_size_bytes = len(text.encode('utf-8'))
+        
+        theoretical_min_bits = entropy * num_units
+        theoretical_min_bytes = theoretical_min_bits / 8.0
+        
+        # Theoretical compression ratio
+        theoretical_compression_ratio = (
+            theoretical_min_bytes / original_size_bytes 
+            if original_size_bytes > 0 else 0.0
+        )
+        
+        return {
+            'entropy_bits_per_unit': entropy,
+            'theoretical_min_bits': theoretical_min_bits,
+            'theoretical_min_bytes': theoretical_min_bytes,
+            'original_size_bytes': original_size_bytes,
+            'theoretical_compression_ratio': theoretical_compression_ratio,
+            'theoretical_space_savings_percent': (1 - theoretical_compression_ratio) * 100
+        }

lopace-0.1.0.dist-info/METADATA

@@ -0,0 +1,402 @@
+Metadata-Version: 2.4
+Name: lopace
+Version: 0.1.0
+Summary: Lossless Optimized Prompt Accurate Compression Engine
+Home-page: https://github.com/connectaman/LoPace
+Author: Aman Ulla
+License: MIT
+Project-URL: Homepage, https://github.com/amanulla/lopace
+Project-URL: Repository, https://github.com/amanulla/lopace
+Project-URL: Issues, https://github.com/amanulla/lopace/issues
+Keywords: prompt,compression,tokenization,zstd,bpe,nlp
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Text Processing :: Linguistic
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: zstandard>=0.22.0
+Requires-Dist: tiktoken>=0.5.0
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: requires-python
+
+# LoPace
+
+**Lossless Optimized Prompt Accurate Compression Engine**
+
+A professional, open-source Python package for compressing and decompressing prompts using multiple techniques: Zstd, Token-based (BPE), and Hybrid methods. Achieve up to 80% space reduction while maintaining perfect lossless reconstruction.
+
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Python 3.8+](https://img.shields.io/badge/python-3.8+-blue.svg)](https://www.python.org/downloads/)
+
+## Features
+
+- 🚀 **Three Compression Methods**:
+  - **Zstd**: Dictionary-based compression using Zstandard algorithm
+  - **Token**: Byte-Pair Encoding (BPE) tokenization with binary packing
+  - **Hybrid**: Combination of tokenization and Zstd (best compression ratio)
+
+- ✅ **Lossless**: Perfect reconstruction of original prompts
+- 📊 **Compression Statistics**: Analyze compression ratios and space savings
+- 🔧 **Simple API**: Easy-to-use interface for all compression methods
+- 🎯 **Database-Ready**: Optimized for storing prompts in databases
+
+## Installation
+
+```bash
+pip install lopace
+```
+
+### Dependencies
+
+- `zstandard>=0.22.0` - For Zstd compression
+- `tiktoken>=0.5.0` - For BPE tokenization
+
+## Quick Start
+
+```python
+from lopace import PromptCompressor, CompressionMethod
+
+# Initialize compressor
+compressor = PromptCompressor(model="cl100k_base", zstd_level=15)
+
+# Your prompt
+prompt = "You are a helpful AI assistant..."
+
+# Compress using hybrid method (recommended)
+compressed = compressor.compress(prompt, CompressionMethod.HYBRID)
+
+# Decompress back to original
+original = compressor.decompress(compressed, CompressionMethod.HYBRID)
+
+# Verify losslessness
+assert original == prompt  # ✓ True
+```
+
+## Usage Examples
+
+### Basic Compression/Decompression
+
+```python
+from lopace import PromptCompressor, CompressionMethod
+
+compressor = PromptCompressor()
+
+# Compress and return both original and compressed
+original, compressed = compressor.compress_and_return_both(
+    "Your prompt here",
+    CompressionMethod.HYBRID
+)
+
+# Decompress
+recovered = compressor.decompress(compressed, CompressionMethod.HYBRID)
+```
+
+### Using Different Methods
+
+```python
+compressor = PromptCompressor()
+
+prompt = "Your system prompt here..."
+
+# Method 1: Zstd only
+zstd_compressed = compressor.compress_zstd(prompt)
+zstd_decompressed = compressor.decompress_zstd(zstd_compressed)
+
+# Method 2: Token-based (BPE)
+token_compressed = compressor.compress_token(prompt)
+token_decompressed = compressor.decompress_token(token_compressed)
+
+# Method 3: Hybrid (recommended - best compression)
+hybrid_compressed = compressor.compress_hybrid(prompt)
+hybrid_decompressed = compressor.decompress_hybrid(hybrid_compressed)
+```
+
+### Get Compression Statistics
+
+```python
+compressor = PromptCompressor()
+prompt = "Your long system prompt..."
+
+# Get stats for all methods
+stats = compressor.get_compression_stats(prompt)
+
+print(f"Original Size: {stats['original_size_bytes']} bytes")
+print(f"Original Tokens: {stats['original_size_tokens']}")
+
+for method, method_stats in stats['methods'].items():
+    print(f"\n{method}:")
+    print(f"  Compressed: {method_stats['compressed_size_bytes']} bytes")
+    print(f"  Space Saved: {method_stats['space_saved_percent']:.2f}%")
+```
+
+## Compression Methods Explained
+
+### 1. Zstd Compression
+
+Uses Zstandard's dictionary-based algorithm to find repeated patterns and replace them with shorter references.
+
+**Best for**: General text compression, when tokenization overhead is not needed.
+
+```python
+compressed = compressor.compress_zstd(prompt)
+original = compressor.decompress_zstd(compressed)
+```
+
+### 2. Token-Based Compression
+
+Uses Byte-Pair Encoding (BPE) to convert text to token IDs, then packs them as binary data.
+
+**Best for**: When you need token IDs anyway, or when working with LLM tokenizers.
+
+```python
+compressed = compressor.compress_token(prompt)
+original = compressor.decompress_token(compressed)
+```
+
+### 3. Hybrid Compression (Recommended)
+
+Combines tokenization and Zstd compression for maximum efficiency:
+
+1. Tokenizes text to reduce redundancy
+2. Packs tokens as binary (2 bytes per token)
+3. Applies Zstd compression on the binary data
+
+**Best for**: Database storage where maximum compression is needed.
+
+```python
+compressed = compressor.compress_hybrid(prompt)
+original = compressor.decompress_hybrid(compressed)
+```
+
+## API Reference
+
+### `PromptCompressor`
+
+Main compressor class.
+
+#### Constructor
+
+```python
+PromptCompressor(
+    model: str = "cl100k_base",
+    zstd_level: int = 15
+)
+```
+
+**Parameters:**
+- `model`: Tokenizer model name (default: `"cl100k_base"`)
+  - Options: `"cl100k_base"`, `"p50k_base"`, `"r50k_base"`, `"gpt2"`, etc.
+- `zstd_level`: Zstd compression level 1-22 (default: `15`)
+  - Higher = better compression but slower
+
+#### Methods
+
+##### `compress(text: str, method: CompressionMethod) -> bytes`
+
+Compress a prompt using the specified method.
+
+##### `decompress(compressed_data: bytes, method: CompressionMethod) -> str`
+
+Decompress a compressed prompt.
+
+##### `compress_and_return_both(text: str, method: CompressionMethod) -> Tuple[str, bytes]`
+
+Compress and return both original and compressed versions.
+
+##### `get_compression_stats(text: str, method: Optional[CompressionMethod]) -> dict`
+
+Get detailed compression statistics for analysis.
+
+### `CompressionMethod`
+
+Enumeration of available compression methods:
+
+- `CompressionMethod.ZSTD` - Zstandard compression
+- `CompressionMethod.TOKEN` - Token-based compression
+- `CompressionMethod.HYBRID` - Hybrid compression (recommended)
+
+## How It Works
+
+### Compression Pipeline (Hybrid Method)
+
+```
+Input: Raw System Prompt String (100%)
+  ↓
+Tokenization: Convert to Tiktoken IDs (~70% reduced)
+  ↓
+Binary Packing: Convert IDs to uint16 (~50% of above)
+  ↓
+Zstd: Final compression (~30% further reduction)
+  ↓
+Output: Compressed Binary Blob
+```
+
+### Why Hybrid is Best for Databases
+
+1. **Searchability**: Token IDs can be searched without full decompression
+2. **Consistency**: Fixed tokenizer ensures stable compression ratios
+3. **Efficiency**: Maximum space savings for millions of prompts
+
+## Example Output
+
+```python
+# Original prompt: 500 bytes
+# After compression:
+#   Zstd: 180 bytes (64% space saved)
+#   Token: 240 bytes (52% space saved)
+#   Hybrid: 120 bytes (76% space saved) ← Best!
+```
+
+## Running the Example
+
+```bash
+python example.py
+```
+
+This will demonstrate all compression methods and show statistics.
+
+## Interactive Web App (Streamlit)
+
+LoPace includes an interactive Streamlit web application with comprehensive evaluation metrics:
+
+### Features
+
+- **Interactive Interface**: Enter prompts and see real-time compression results
+- **Comprehensive Metrics**: All four industry-standard metrics:
+  - Compression Ratio (CR): $CR = \frac{S_{original}}{S_{compressed}}$
+  - Space Savings (SS): $SS = 1 - \frac{S_{compressed}}{S_{original}}$
+  - Bits Per Character (BPC): $BPC = \frac{Total Bits}{Total Characters}$
+  - Throughput (MB/s): $T = \frac{Data Size}{Time}$
+- **Lossless Verification**:
+  - SHA-256 Hash Verification
+  - Exact Match (Character-by-Character)
+  - Reconstruction Error: $E = \frac{1}{N} \sum_{i=1}^{N} \mathbb{1}(x_i \neq \hat{x}_i) = 0$
+- **Side-by-Side Comparison**: Compare all three compression methods
+- **Real-time Configuration**: Adjust tokenizer model and Zstd level
+
+### Running the Streamlit App
+
+```bash
+streamlit run streamlit_app.py
+```
+
+The app will open in your default web browser at `http://localhost:8501`
+
+### Screenshot Preview
+
+The app features:
+- **Left Panel**: Text input area for entering prompts
+- **Right Panel**: Results with tabs for each compression method
+- **Metrics Dashboard**: Real-time calculation of all evaluation metrics
+- **Verification Section**: Hash matching and exact match verification
+- **Comparison Table**: Side-by-side comparison of all methods
+
+## Development
+
+### Setup Development Environment
+
+```bash
+git clone https://github.com/amanulla/lopace.git
+cd lopace
+pip install -r requirements-dev.txt
+```
+
+### Running Tests
+
+```bash
+pytest
+```
+
+### CI/CD Pipeline
+
+This project uses GitHub Actions for automated testing and publishing:
+
+- **Tests run automatically** on every push and pull request
+- **Publishing to PyPI** happens automatically when:
+  - All tests pass ✅
+  - Push is to `main`/`master` branch or a version tag (e.g., `v0.1.0`)
+
+See [.github/workflows/README.md](.github/workflows/README.md) for detailed setup instructions.
+
+## Mathematical Background
+
+### Compression Techniques Used
+
+LoPace uses the following compression techniques:
+
+1. **LZ77 (Sliding Window)**: Used **indirectly** through Zstandard
+   - Zstandard internally uses LZ77-style algorithms to find repeated patterns
+   - Instead of storing "assistant" again, it stores a tuple: (distance_back, length)
+   - We use this by calling `zstandard.compress()` - the LZ77 is handled internally
+
+2. **Huffman Coding / FSE (Finite State Entropy)**: Used **indirectly** through Zstandard
+   - Zstandard uses FSE, a variant of Huffman coding
+   - Assigns shorter binary codes to characters/patterns that appear most frequently
+   - Again, handled internally by the zstandard library
+
+3. **BPE Tokenization**: Used **directly** via tiktoken
+   - Byte-Pair Encoding converts text to token IDs
+   - Reduces vocabulary size before compression
+   - Implemented by OpenAI's tiktoken library
+
+### Shannon Entropy
+
+The theoretical compression limit is determined by Shannon Entropy:
+
+$H(X) = -\sum_{i=1}^{n} P(x_i) \log_2 P(x_i)$
+
+Where:
+- $H(X)$ is the entropy of the source
+- $P(x_i)$ is the probability of character/pattern $x_i$
+
+LoPace **calculates** Shannon Entropy to show theoretical compression limits:
+
+```python
+compressor = PromptCompressor()
+entropy = compressor.calculate_shannon_entropy("Your prompt")
+limits = compressor.get_theoretical_compression_limit("Your prompt")
+print(f"Theoretical minimum: {limits['theoretical_min_bytes']:.2f} bytes")
+```
+
+This allows you to compare actual compression against the theoretical limit.
+
+## License
+
+MIT License - see [LICENSE](LICENSE) file for details.
+
+## Contributing
+
+Contributions are welcome! We appreciate your help in making LoPace better.
+
+Please read our [Contributing Guidelines](CONTRIBUTING.md) and [Code of Conduct](CODE_OF_CONDUCT.md) before contributing.
+
+### Quick Start for Contributors
+
+1. Fork the repository
+2. Create a feature branch (`git checkout -b feature/amazing-feature`)
+3. Make your changes
+4. Run tests (`pytest tests/ -v`)
+5. Commit your changes (`git commit -m 'Add amazing feature'`)
+6. Push to the branch (`git push origin feature/amazing-feature`)
+7. Open a Pull Request
+
+For more details, see [CONTRIBUTING.md](CONTRIBUTING.md).
+
+## Author
+
+Aman Ulla
+
+## Acknowledgments
+
+- Built on top of [zstandard](https://github.com/facebook/zstd) and [tiktoken](https://github.com/openai/tiktoken)
+- Inspired by the need for efficient prompt storage in LLM applications

lopace-0.1.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+lopace/__init__.py,sha256=PYjZWZHhSITNgag9sF0qZ_yXgZaMa3R8_3FuasiH0Nc,351
+lopace/compressor.py,sha256=nUTWDcAPYvQaeSFKx_lne-D2xIQ02IMVGE4yLODo8qE,19060
+lopace-0.1.0.dist-info/licenses/LICENSE,sha256=uFUrlsfsOwx_8Nzhq2pUgNaJghcJxXBMML3l7T39Tm0,1067
+tests/__init__.py,sha256=yXNVJE20E2iHo0qbit5SgRE35eXWq89F1kkhNHy7VJA,31
+tests/test_compressor.py,sha256=-vMztSzY89n5dpShcACrFboEQOlfJ6FxF7eQOEU3swM,8273
+lopace-0.1.0.dist-info/METADATA,sha256=yXy0jt23uvVWkGlEeCb8KEUSx1_o3N02wZZEFj5weEI,12199
+lopace-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+lopace-0.1.0.dist-info/top_level.txt,sha256=8CLB5czxmmAfR7ayh3TO5qyB1-xJoYNxabufJ37Xh5o,13
+lopace-0.1.0.dist-info/RECORD,,

lopace-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

lopace-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Aman Ulla 
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

lopace-0.1.0.dist-info/top_level.txt

@@ -0,0 +1,2 @@
+lopace
+tests

tests/__init__.py

@@ -0,0 +1 @@
+"""Tests for LoPace package."""

tests/test_compressor.py

@@ -0,0 +1,198 @@
+"""Tests for PromptCompressor class."""
+
+import pytest
+from lopace import PromptCompressor, CompressionMethod
+
+
+@pytest.fixture
+def compressor():
+    """Create a PromptCompressor instance for testing."""
+    return PromptCompressor(model="cl100k_base", zstd_level=15)
+
+
+@pytest.fixture
+def sample_prompt():
+    """Sample prompt for testing."""
+    return """You are a helpful AI assistant designed to provide accurate, 
+    detailed, and helpful responses to user queries. Your goal is to assist users 
+    by understanding their questions and providing relevant information."""
+
+
+class TestZstdCompression:
+    """Test Zstd compression/decompression."""
+    
+    def test_compress_decompress_zstd(self, compressor, sample_prompt):
+        """Test that Zstd compression is lossless."""
+        compressed = compressor.compress_zstd(sample_prompt)
+        decompressed = compressor.decompress_zstd(compressed)
+        assert decompressed == sample_prompt
+    
+    def test_zstd_compression_ratio(self, compressor, sample_prompt):
+        """Test that Zstd actually compresses."""
+        compressed = compressor.compress_zstd(sample_prompt)
+        original_size = len(sample_prompt.encode('utf-8'))
+        compressed_size = len(compressed)
+        assert compressed_size < original_size
+
+
+class TestTokenCompression:
+    """Test Token-based compression/decompression."""
+    
+    def test_compress_decompress_token(self, compressor, sample_prompt):
+        """Test that Token compression is lossless."""
+        compressed = compressor.compress_token(sample_prompt)
+        decompressed = compressor.decompress_token(compressed)
+        assert decompressed == sample_prompt
+    
+    def test_token_binary_format(self, compressor, sample_prompt):
+        """Test that token compression produces binary data with format byte."""
+        compressed = compressor.compress_token(sample_prompt)
+        assert isinstance(compressed, bytes)
+        # Should have at least format byte (1 byte)
+        assert len(compressed) >= 1
+        # Format byte should be 0 (uint16) or 1 (uint32)
+        import struct
+        format_byte = struct.unpack('B', compressed[0:1])[0]
+        assert format_byte in [0, 1]
+
+
+class TestHybridCompression:
+    """Test Hybrid compression/decompression."""
+    
+    def test_compress_decompress_hybrid(self, compressor, sample_prompt):
+        """Test that Hybrid compression is lossless."""
+        compressed = compressor.compress_hybrid(sample_prompt)
+        decompressed = compressor.decompress_hybrid(compressed)
+        assert decompressed == sample_prompt
+    
+    def test_hybrid_compression(self, compressor, sample_prompt):
+        """Test that Hybrid compression works and compares with other methods."""
+        zstd_compressed = compressor.compress_zstd(sample_prompt)
+        token_compressed = compressor.compress_token(sample_prompt)
+        hybrid_compressed = compressor.compress_hybrid(sample_prompt)
+        
+        # All methods should compress (smaller than original for longer prompts)
+        original_size = len(sample_prompt.encode('utf-8'))
+        
+        # Verify all compression methods produce valid output
+        assert len(zstd_compressed) > 0
+        assert len(token_compressed) > 0
+        assert len(hybrid_compressed) > 0
+        
+        # For very long prompts (>500 chars), hybrid should typically be better than token alone
+        # But for short prompts, Zstd overhead can make hybrid larger
+        if len(sample_prompt) > 500:
+            # On very long prompts, hybrid should generally compress well
+            assert len(hybrid_compressed) < original_size
+        
+        # Zstd alone should compress for longer prompts
+        if len(sample_prompt) > 100:
+            assert len(zstd_compressed) < original_size
+
+
+class TestGenericMethods:
+    """Test generic compress/decompress methods."""
+    
+    def test_compress_with_method(self, compressor, sample_prompt):
+        """Test generic compress method with all methods."""
+        for method in CompressionMethod:
+            compressed = compressor.compress(sample_prompt, method)
+            assert isinstance(compressed, bytes)
+            assert len(compressed) > 0
+    
+    def test_decompress_with_method(self, compressor, sample_prompt):
+        """Test generic decompress method with all methods."""
+        for method in CompressionMethod:
+            compressed = compressor.compress(sample_prompt, method)
+            decompressed = compressor.decompress(compressed, method)
+            assert decompressed == sample_prompt
+    
+    def test_compress_and_return_both(self, compressor, sample_prompt):
+        """Test compress_and_return_both method."""
+        original, compressed = compressor.compress_and_return_both(
+            sample_prompt, 
+            CompressionMethod.HYBRID
+        )
+        assert original == sample_prompt
+        assert isinstance(compressed, bytes)
+        assert len(compressed) > 0
+
+
+class TestCompressionStats:
+    """Test compression statistics."""
+    
+    def test_get_compression_stats_all_methods(self, compressor, sample_prompt):
+        """Test getting stats for all methods."""
+        stats = compressor.get_compression_stats(sample_prompt)
+        
+        assert 'original_size_bytes' in stats
+        assert 'original_size_tokens' in stats
+        assert 'methods' in stats
+        
+        assert len(stats['methods']) == 3  # ZSTD, TOKEN, HYBRID
+        
+        for method_name, method_stats in stats['methods'].items():
+            assert 'compressed_size_bytes' in method_stats
+            assert 'compression_ratio' in method_stats
+            assert 'space_saved_percent' in method_stats
+            assert 'bytes_saved' in method_stats
+    
+    def test_get_compression_stats_single_method(self, compressor, sample_prompt):
+        """Test getting stats for a single method."""
+        stats = compressor.get_compression_stats(
+            sample_prompt, 
+            CompressionMethod.HYBRID
+        )
+        
+        assert len(stats['methods']) == 1
+        assert CompressionMethod.HYBRID.value in stats['methods']
+
+
+class TestEdgeCases:
+    """Test edge cases and error handling."""
+    
+    def test_empty_string(self, compressor):
+        """Test compression of empty string."""
+        for method in CompressionMethod:
+            compressed = compressor.compress("", method)
+            decompressed = compressor.decompress(compressed, method)
+            assert decompressed == ""
+    
+    def test_large_token_ids(self, compressor):
+        """Test compression with token IDs that exceed uint16 range."""
+        # Create a prompt that might trigger large token IDs
+        # Using various special characters and unicode
+        large_prompt = "Hello " * 1000 + "世界 🌍 مرحبا " * 100
+        
+        # This should work without error, handling uint32 if needed
+        compressed = compressor.compress_token(large_prompt)
+        decompressed = compressor.decompress_token(compressed)
+        assert decompressed == large_prompt
+        
+        # Test hybrid method too
+        compressed_hybrid = compressor.compress_hybrid(large_prompt)
+        decompressed_hybrid = compressor.decompress_hybrid(compressed_hybrid)
+        assert decompressed_hybrid == large_prompt
+    
+    def test_single_character(self, compressor):
+        """Test compression of single character."""
+        for method in CompressionMethod:
+            compressed = compressor.compress("a", method)
+            decompressed = compressor.decompress(compressed, method)
+            assert decompressed == "a"
+    
+    def test_unicode_characters(self, compressor):
+        """Test compression of unicode characters."""
+        unicode_prompt = "你好世界 🌍 مرحبا"
+        for method in CompressionMethod:
+            compressed = compressor.compress(unicode_prompt, method)
+            decompressed = compressor.decompress(compressed, method)
+            assert decompressed == unicode_prompt
+    
+    def test_invalid_zstd_level(self):
+        """Test that invalid zstd_level raises error."""
+        with pytest.raises(ValueError):
+            PromptCompressor(zstd_level=0)
+        
+        with pytest.raises(ValueError):
+            PromptCompressor(zstd_level=23)