abstractcore 2.5.0__py3-none-any.whl → 2.5.3__py3-none-any.whl

abstractcore/compression/__init__.py

@@ -0,0 +1,29 @@
+"""
+Glyph visual-text compression system for AbstractCore.
+
+This module provides visual-text compression capabilities that transform long textual
+sequences into optimized images for processing by Vision-Language Models (VLMs),
+achieving 3-4x token compression without accuracy loss.
+
+Based on the Glyph framework by Z.ai/THU-COAI with AbstractCore-specific enhancements.
+"""
+
+from .glyph_processor import GlyphProcessor
+from .orchestrator import CompressionOrchestrator
+from .config import GlyphConfig, RenderingConfig
+from .quality import QualityValidator, CompressionStats
+from .cache import CompressionCache
+from .exceptions import CompressionError, CompressionQualityError
+
+__all__ = [
+    'GlyphProcessor',
+    'CompressionOrchestrator', 
+    'GlyphConfig',
+    'RenderingConfig',
+    'QualityValidator',
+    'CompressionStats',
+    'CompressionCache',
+    'CompressionError',
+    'CompressionQualityError'
+]
+

abstractcore/compression/analytics.py

@@ -0,0 +1,420 @@
+"""
+Compression analytics system for tracking and analyzing compression performance.
+"""
+
+import time
+import json
+from pathlib import Path
+from typing import Dict, Any, List, Optional
+from dataclasses import dataclass, field
+from datetime import datetime
+import statistics
+
+from ..utils.structured_logging import get_logger
+
+
+@dataclass
+class CompressionMetrics:
+    """Metrics for a single compression operation."""
+    timestamp: float
+    provider: str
+    model: str
+    original_tokens: int
+    compressed_tokens: int
+    compression_ratio: float
+    quality_score: float
+    processing_time: float
+    images_created: int
+    method: str  # "glyph", "hybrid", etc.
+    success: bool
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary."""
+        return {
+            "timestamp": self.timestamp,
+            "provider": self.provider,
+            "model": self.model,
+            "original_tokens": self.original_tokens,
+            "compressed_tokens": self.compressed_tokens,
+            "compression_ratio": self.compression_ratio,
+            "quality_score": self.quality_score,
+            "processing_time": self.processing_time,
+            "images_created": self.images_created,
+            "method": self.method,
+            "success": self.success,
+            "metadata": self.metadata
+        }
+
+
+class CompressionAnalytics:
+    """
+    Analytics system for tracking compression performance.
+
+    Collects metrics, analyzes trends, and provides insights for optimization.
+    """
+
+    def __init__(self, storage_path: Optional[Path] = None):
+        """
+        Initialize analytics system.
+
+        Args:
+            storage_path: Path to store analytics data
+        """
+        self.logger = get_logger(self.__class__.__name__)
+        self.metrics: List[CompressionMetrics] = []
+        self.storage_path = storage_path or Path.home() / ".abstractcore" / "analytics" / "compression.json"
+        self.storage_path.parent.mkdir(parents=True, exist_ok=True)
+
+        # Load existing metrics
+        self._load_metrics()
+
+    def _load_metrics(self):
+        """Load existing metrics from storage."""
+        if self.storage_path.exists():
+            try:
+                with open(self.storage_path, 'r') as f:
+                    data = json.load(f)
+                    for item in data:
+                        self.metrics.append(CompressionMetrics(**item))
+                self.logger.debug(f"Loaded {len(self.metrics)} historical metrics")
+            except Exception as e:
+                self.logger.warning(f"Failed to load metrics: {e}")
+
+    def _save_metrics(self):
+        """Save metrics to storage."""
+        try:
+            data = [m.to_dict() for m in self.metrics]
+            with open(self.storage_path, 'w') as f:
+                json.dump(data, f, indent=2)
+            self.logger.debug(f"Saved {len(self.metrics)} metrics")
+        except Exception as e:
+            self.logger.error(f"Failed to save metrics: {e}")
+
+    def record_compression(
+        self,
+        provider: str,
+        model: str,
+        original_tokens: int,
+        compressed_tokens: int,
+        quality_score: float,
+        processing_time: float,
+        images_created: int = 0,
+        method: str = "glyph",
+        success: bool = True,
+        metadata: Dict[str, Any] = None
+    ) -> CompressionMetrics:
+        """
+        Record a compression operation.
+
+        Args:
+            provider: Provider used
+            model: Model used
+            original_tokens: Original token count
+            compressed_tokens: Compressed token count
+            quality_score: Quality score (0-1)
+            processing_time: Processing time in seconds
+            images_created: Number of images created
+            method: Compression method used
+            success: Whether compression succeeded
+            metadata: Additional metadata
+
+        Returns:
+            Created CompressionMetrics object
+        """
+        compression_ratio = original_tokens / compressed_tokens if compressed_tokens > 0 else 1.0
+
+        metric = CompressionMetrics(
+            timestamp=time.time(),
+            provider=provider,
+            model=model,
+            original_tokens=original_tokens,
+            compressed_tokens=compressed_tokens,
+            compression_ratio=compression_ratio,
+            quality_score=quality_score,
+            processing_time=processing_time,
+            images_created=images_created,
+            method=method,
+            success=success,
+            metadata=metadata or {}
+        )
+
+        self.metrics.append(metric)
+        self._save_metrics()
+
+        self.logger.info(
+            f"Recorded compression: {provider}/{model}, "
+            f"{compression_ratio:.1f}x, {quality_score:.2%} quality"
+        )
+
+        return metric
+
+    def get_provider_stats(self, provider: str) -> Dict[str, Any]:
+        """
+        Get statistics for a specific provider.
+
+        Args:
+            provider: Provider name
+
+        Returns:
+            Statistics dictionary
+        """
+        provider_metrics = [m for m in self.metrics if m.provider == provider and m.success]
+
+        if not provider_metrics:
+            return {"provider": provider, "no_data": True}
+
+        ratios = [m.compression_ratio for m in provider_metrics]
+        qualities = [m.quality_score for m in provider_metrics]
+        times = [m.processing_time for m in provider_metrics]
+
+        return {
+            "provider": provider,
+            "total_compressions": len(provider_metrics),
+            "avg_compression_ratio": statistics.mean(ratios),
+            "median_compression_ratio": statistics.median(ratios),
+            "best_compression_ratio": max(ratios),
+            "avg_quality_score": statistics.mean(qualities),
+            "avg_processing_time": statistics.mean(times),
+            "success_rate": len(provider_metrics) / len([m for m in self.metrics if m.provider == provider])
+        }
+
+    def get_model_stats(self, provider: str, model: str) -> Dict[str, Any]:
+        """
+        Get statistics for a specific model.
+
+        Args:
+            provider: Provider name
+            model: Model name
+
+        Returns:
+            Statistics dictionary
+        """
+        model_metrics = [
+            m for m in self.metrics
+            if m.provider == provider and m.model == model and m.success
+        ]
+
+        if not model_metrics:
+            return {"provider": provider, "model": model, "no_data": True}
+
+        ratios = [m.compression_ratio for m in model_metrics]
+        qualities = [m.quality_score for m in model_metrics]
+        times = [m.processing_time for m in model_metrics]
+        images = [m.images_created for m in model_metrics]
+
+        return {
+            "provider": provider,
+            "model": model,
+            "total_compressions": len(model_metrics),
+            "avg_compression_ratio": statistics.mean(ratios),
+            "std_compression_ratio": statistics.stdev(ratios) if len(ratios) > 1 else 0,
+            "avg_quality_score": statistics.mean(qualities),
+            "avg_processing_time": statistics.mean(times),
+            "avg_images_created": statistics.mean(images),
+            "percentiles": {
+                "p25": statistics.quantiles(ratios, n=4)[0] if len(ratios) > 1 else ratios[0],
+                "p50": statistics.median(ratios),
+                "p75": statistics.quantiles(ratios, n=4)[2] if len(ratios) > 1 else ratios[0],
+            } if ratios else {}
+        }
+
+    def get_method_comparison(self) -> Dict[str, Any]:
+        """
+        Compare different compression methods.
+
+        Returns:
+            Comparison statistics
+        """
+        methods = {}
+
+        for method in set(m.method for m in self.metrics):
+            method_metrics = [m for m in self.metrics if m.method == method and m.success]
+
+            if method_metrics:
+                ratios = [m.compression_ratio for m in method_metrics]
+                qualities = [m.quality_score for m in method_metrics]
+                times = [m.processing_time for m in method_metrics]
+
+                methods[method] = {
+                    "count": len(method_metrics),
+                    "avg_compression": statistics.mean(ratios),
+                    "avg_quality": statistics.mean(qualities),
+                    "avg_time": statistics.mean(times),
+                    "efficiency": statistics.mean(ratios) / statistics.mean(times) if times else 0
+                }
+
+        return methods
+
+    def get_trends(self, hours: int = 24) -> Dict[str, Any]:
+        """
+        Get compression trends over time.
+
+        Args:
+            hours: Number of hours to analyze
+
+        Returns:
+            Trend analysis
+        """
+        cutoff_time = time.time() - (hours * 3600)
+        recent_metrics = [m for m in self.metrics if m.timestamp > cutoff_time and m.success]
+
+        if not recent_metrics:
+            return {"no_recent_data": True}
+
+        # Sort by timestamp
+        recent_metrics.sort(key=lambda m: m.timestamp)
+
+        # Calculate rolling averages
+        window_size = max(1, len(recent_metrics) // 10)
+        rolling_ratios = []
+        rolling_qualities = []
+
+        for i in range(len(recent_metrics) - window_size + 1):
+            window = recent_metrics[i:i + window_size]
+            rolling_ratios.append(statistics.mean([m.compression_ratio for m in window]))
+            rolling_qualities.append(statistics.mean([m.quality_score for m in window]))
+
+        # Detect trends
+        if len(rolling_ratios) > 1:
+            ratio_trend = "improving" if rolling_ratios[-1] > rolling_ratios[0] else "declining"
+            quality_trend = "improving" if rolling_qualities[-1] > rolling_qualities[0] else "declining"
+        else:
+            ratio_trend = "stable"
+            quality_trend = "stable"
+
+        return {
+            "period_hours": hours,
+            "total_compressions": len(recent_metrics),
+            "ratio_trend": ratio_trend,
+            "quality_trend": quality_trend,
+            "current_avg_ratio": statistics.mean([m.compression_ratio for m in recent_metrics[-window_size:]]) if recent_metrics else 0,
+            "current_avg_quality": statistics.mean([m.quality_score for m in recent_metrics[-window_size:]]) if recent_metrics else 0
+        }
+
+    def get_optimization_suggestions(self) -> List[str]:
+        """
+        Generate optimization suggestions based on analytics.
+
+        Returns:
+            List of suggestions
+        """
+        suggestions = []
+
+        # Analyze recent performance
+        recent_metrics = [m for m in self.metrics[-100:] if m.success]  # Last 100 compressions
+
+        if recent_metrics:
+            avg_ratio = statistics.mean([m.compression_ratio for m in recent_metrics])
+            avg_quality = statistics.mean([m.quality_score for m in recent_metrics])
+
+            # Compression ratio suggestions
+            if avg_ratio < 3.0:
+                suggestions.append("Consider using more aggressive compression settings (smaller fonts, more columns)")
+            elif avg_ratio > 5.0 and avg_quality < 0.90:
+                suggestions.append("High compression may be affecting quality - consider balancing settings")
+
+            # Quality suggestions
+            if avg_quality < 0.90:
+                suggestions.append("Quality scores are low - increase DPI or font size")
+            elif avg_quality > 0.95 and avg_ratio < 4.0:
+                suggestions.append("Quality is very high - room for more aggressive compression")
+
+            # Provider-specific suggestions
+            provider_stats = {}
+            for provider in set(m.provider for m in recent_metrics):
+                stats = self.get_provider_stats(provider)
+                if not stats.get("no_data"):
+                    provider_stats[provider] = stats
+
+            if provider_stats:
+                best_provider = max(provider_stats.items(), key=lambda x: x[1].get("avg_compression_ratio", 0))
+                if best_provider[1]["avg_compression_ratio"] > avg_ratio * 1.2:
+                    suggestions.append(f"Provider '{best_provider[0]}' shows better compression - consider using it more")
+
+            # Method suggestions
+            method_comparison = self.get_method_comparison()
+            if "hybrid" in method_comparison and "glyph" in method_comparison:
+                if method_comparison["hybrid"]["avg_compression"] > method_comparison["glyph"]["avg_compression"] * 2:
+                    suggestions.append("Hybrid compression shows significant improvement - use it for large documents")
+
+        if not suggestions:
+            suggestions.append("Performance is optimal - no specific improvements recommended")
+
+        return suggestions
+
+    def generate_report(self) -> str:
+        """
+        Generate a comprehensive analytics report.
+
+        Returns:
+            Formatted report string
+        """
+        report = ["=" * 80]
+        report.append("COMPRESSION ANALYTICS REPORT")
+        report.append("=" * 80)
+        report.append(f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
+        report.append(f"Total compressions: {len(self.metrics)}")
+
+        # Overall statistics
+        if self.metrics:
+            successful = [m for m in self.metrics if m.success]
+            if successful:
+                report.append(f"\nOVERALL STATISTICS:")
+                report.append(f"  Success rate: {len(successful)/len(self.metrics):.1%}")
+                report.append(f"  Avg compression: {statistics.mean([m.compression_ratio for m in successful]):.1f}x")
+                report.append(f"  Avg quality: {statistics.mean([m.quality_score for m in successful]):.1%}")
+                report.append(f"  Avg time: {statistics.mean([m.processing_time for m in successful]):.2f}s")
+
+        # Provider breakdown
+        providers = set(m.provider for m in self.metrics)
+        if providers:
+            report.append(f"\nPROVIDER PERFORMANCE:")
+            for provider in sorted(providers):
+                stats = self.get_provider_stats(provider)
+                if not stats.get("no_data"):
+                    report.append(f"  {provider}:")
+                    report.append(f"    Compressions: {stats['total_compressions']}")
+                    report.append(f"    Avg ratio: {stats['avg_compression_ratio']:.1f}x")
+                    report.append(f"    Avg quality: {stats['avg_quality_score']:.1%}")
+
+        # Method comparison
+        methods = self.get_method_comparison()
+        if methods:
+            report.append(f"\nMETHOD COMPARISON:")
+            for method, stats in methods.items():
+                report.append(f"  {method}:")
+                report.append(f"    Avg compression: {stats['avg_compression']:.1f}x")
+                report.append(f"    Avg quality: {stats['avg_quality']:.1%}")
+                report.append(f"    Efficiency: {stats['efficiency']:.2f}")
+
+        # Trends
+        trends = self.get_trends(24)
+        if not trends.get("no_recent_data"):
+            report.append(f"\nRECENT TRENDS (24h):")
+            report.append(f"  Compression trend: {trends['ratio_trend']}")
+            report.append(f"  Quality trend: {trends['quality_trend']}")
+            report.append(f"  Current avg ratio: {trends['current_avg_ratio']:.1f}x")
+
+        # Suggestions
+        suggestions = self.get_optimization_suggestions()
+        if suggestions:
+            report.append(f"\nOPTIMIZATION SUGGESTIONS:")
+            for i, suggestion in enumerate(suggestions, 1):
+                report.append(f"  {i}. {suggestion}")
+
+        report.append("=" * 80)
+
+        return "\n".join(report)
+
+
+# Global analytics instance
+_analytics_instance = None
+
+
+def get_analytics() -> CompressionAnalytics:
+    """Get global analytics instance."""
+    global _analytics_instance
+    if _analytics_instance is None:
+        _analytics_instance = CompressionAnalytics()
+    return _analytics_instance

abstractcore/compression/cache.py

@@ -0,0 +1,250 @@
+"""
+Compression cache system for Glyph.
+"""
+
+import os
+import json
+import time
+import hashlib
+import shutil
+from pathlib import Path
+from typing import List, Optional, Dict, Any
+from dataclasses import asdict
+
+from .exceptions import CompressionCacheError
+from ..utils.structured_logging import get_logger
+
+
+class CompressionCache:
+    """Cache system for compressed content."""
+    
+    def __init__(self, cache_dir: str = None, max_size_gb: float = 1.0, ttl_days: int = 7):
+        """
+        Initialize compression cache.
+        
+        Args:
+            cache_dir: Cache directory path
+            max_size_gb: Maximum cache size in GB
+            ttl_days: Time-to-live in days
+        """
+        self.logger = get_logger(self.__class__.__name__)
+        
+        if cache_dir is None:
+            cache_dir = str(Path.home() / ".abstractcore" / "glyph_cache")
+        
+        self.cache_dir = Path(cache_dir)
+        self.max_size_bytes = int(max_size_gb * 1024 * 1024 * 1024)
+        self.ttl_seconds = ttl_days * 24 * 3600
+        
+        # Create cache directory (thread-safe for parallel executions)
+        try:
+            self.cache_dir.mkdir(parents=True, exist_ok=True)
+            self.logger.debug(f"Cache directory created/verified: {self.cache_dir}")
+        except Exception as e:
+            self.logger.error(f"Failed to create cache directory {self.cache_dir}: {e}")
+            raise CompressionCacheError(f"Cannot create cache directory: {e}")
+
+        # Initialize metadata
+        self.metadata_file = self.cache_dir / "cache_metadata.json"
+        self.metadata = self._load_metadata()
+        
+        self.logger.debug(f"Initialized compression cache at {self.cache_dir}")
+    
+    def _load_metadata(self) -> Dict[str, Any]:
+        """Load cache metadata."""
+        if self.metadata_file.exists():
+            try:
+                with open(self.metadata_file, 'r') as f:
+                    return json.load(f)
+            except Exception as e:
+                self.logger.warning(f"Failed to load cache metadata: {e}")
+        
+        return {
+            'entries': {},
+            'created': time.time(),
+            'last_cleanup': time.time()
+        }
+    
+    def _save_metadata(self):
+        """Save cache metadata."""
+        try:
+            with open(self.metadata_file, 'w') as f:
+                json.dump(self.metadata, f, indent=2)
+        except Exception as e:
+            self.logger.error(f"Failed to save cache metadata: {e}")
+    
+    def _generate_cache_key(self, content: str, config: Dict[str, Any]) -> str:
+        """Generate cache key from content and configuration."""
+        # Create a hash of content + configuration
+        content_hash = hashlib.sha256(content.encode('utf-8')).hexdigest()[:16]
+        config_str = json.dumps(config, sort_keys=True)
+        config_hash = hashlib.sha256(config_str.encode('utf-8')).hexdigest()[:8]
+        
+        return f"{content_hash}_{config_hash}"
+    
+    def get(self, cache_key: str) -> Optional[List[Path]]:
+        """
+        Get cached compression result.
+        
+        Args:
+            cache_key: Cache key
+            
+        Returns:
+            List of image paths if cached, None otherwise
+        """
+        if cache_key not in self.metadata['entries']:
+            return None
+        
+        entry = self.metadata['entries'][cache_key]
+        
+        # Check TTL
+        if time.time() - entry['created'] > self.ttl_seconds:
+            self._remove_entry(cache_key)
+            return None
+        
+        # Check if files still exist
+        image_paths = [Path(path) for path in entry['image_paths']]
+        if not all(path.exists() for path in image_paths):
+            self._remove_entry(cache_key)
+            return None
+        
+        # Update access time
+        entry['last_accessed'] = time.time()
+        self._save_metadata()
+        
+        self.logger.debug(f"Cache hit for key {cache_key}")
+        return image_paths
+    
+    def set(self, cache_key: str, image_paths: List[Path], 
+            compression_stats: Dict[str, Any] = None):
+        """
+        Store compression result in cache.
+        
+        Args:
+            cache_key: Cache key
+            image_paths: List of rendered image paths
+            compression_stats: Optional compression statistics
+        """
+        try:
+            # Create cache entry directory
+            entry_dir = self.cache_dir / cache_key
+            entry_dir.mkdir(exist_ok=True)
+            
+            # Copy images to cache
+            cached_paths = []
+            for i, image_path in enumerate(image_paths):
+                if image_path.exists():
+                    cached_path = entry_dir / f"image_{i}{image_path.suffix}"
+                    shutil.copy2(image_path, cached_path)
+                    cached_paths.append(str(cached_path))
+            
+            # Store metadata
+            entry = {
+                'created': time.time(),
+                'last_accessed': time.time(),
+                'image_paths': cached_paths,
+                'compression_stats': compression_stats or {},
+                'size_bytes': sum(Path(path).stat().st_size for path in cached_paths)
+            }
+            
+            self.metadata['entries'][cache_key] = entry
+            self._save_metadata()
+            
+            # Cleanup if needed
+            self._cleanup_if_needed()
+            
+            self.logger.debug(f"Cached compression result for key {cache_key}")
+            
+        except Exception as e:
+            self.logger.error(f"Failed to cache compression result: {e}")
+            raise CompressionCacheError(f"Cache storage failed: {e}")
+    
+    def _remove_entry(self, cache_key: str):
+        """Remove cache entry."""
+        if cache_key in self.metadata['entries']:
+            # Remove files
+            entry_dir = self.cache_dir / cache_key
+            if entry_dir.exists():
+                shutil.rmtree(entry_dir, ignore_errors=True)
+            
+            # Remove metadata
+            del self.metadata['entries'][cache_key]
+            self._save_metadata()
+    
+    def _cleanup_if_needed(self):
+        """Cleanup cache if size or TTL limits exceeded."""
+        now = time.time()
+        
+        # Skip if cleaned up recently
+        if now - self.metadata.get('last_cleanup', 0) < 3600:  # 1 hour
+            return
+        
+        self.logger.debug("Starting cache cleanup")
+        
+        # Remove expired entries
+        expired_keys = []
+        for key, entry in self.metadata['entries'].items():
+            if now - entry['created'] > self.ttl_seconds:
+                expired_keys.append(key)
+        
+        for key in expired_keys:
+            self._remove_entry(key)
+        
+        # Check size limit
+        total_size = sum(entry.get('size_bytes', 0) for entry in self.metadata['entries'].values())
+        
+        if total_size > self.max_size_bytes:
+            # Remove least recently accessed entries
+            entries_by_access = sorted(
+                self.metadata['entries'].items(),
+                key=lambda x: x[1].get('last_accessed', 0)
+            )
+            
+            while total_size > self.max_size_bytes and entries_by_access:
+                key, entry = entries_by_access.pop(0)
+                total_size -= entry.get('size_bytes', 0)
+                self._remove_entry(key)
+        
+        self.metadata['last_cleanup'] = now
+        self._save_metadata()
+        
+        self.logger.debug(f"Cache cleanup completed. Entries: {len(self.metadata['entries'])}")
+    
+    def clear(self):
+        """Clear all cache entries."""
+        try:
+            # Remove all entry directories
+            for entry_dir in self.cache_dir.iterdir():
+                if entry_dir.is_dir() and entry_dir.name != "cache_metadata.json":
+                    shutil.rmtree(entry_dir, ignore_errors=True)
+            
+            # Reset metadata
+            self.metadata = {
+                'entries': {},
+                'created': time.time(),
+                'last_cleanup': time.time()
+            }
+            self._save_metadata()
+            
+            self.logger.info("Cache cleared")
+            
+        except Exception as e:
+            self.logger.error(f"Failed to clear cache: {e}")
+            raise CompressionCacheError(f"Cache clear failed: {e}")
+    
+    def get_stats(self) -> Dict[str, Any]:
+        """Get cache statistics."""
+        entries = self.metadata['entries']
+        total_size = sum(entry.get('size_bytes', 0) for entry in entries.values())
+        
+        return {
+            'total_entries': len(entries),
+            'total_size_mb': total_size / (1024 * 1024),
+            'max_size_mb': self.max_size_bytes / (1024 * 1024),
+            'utilization': total_size / self.max_size_bytes if self.max_size_bytes > 0 else 0,
+            'cache_dir': str(self.cache_dir),
+            'ttl_days': self.ttl_seconds / (24 * 3600),
+            'oldest_entry': min((entry['created'] for entry in entries.values()), default=time.time()),
+            'newest_entry': max((entry['created'] for entry in entries.values()), default=time.time())
+        }
+