agmem 0.1.6__py3-none-any.whl → 0.2.1__py3-none-any.whl

memvcs/core/compression_metrics.py

@@ -0,0 +1,248 @@
+"""
+Delta compression metrics and observability.
+
+Tracks compression effectiveness across object types to enable future
+optimization and auto-tuning of delta encoding parameters.
+
+Provides:
+- DeltaCompressionMetrics: Tracks compression ratio, object types, benefits
+- CompressionHeatmap: Visualizes which types compress best
+- Statistics reporting for gc --repack operations
+"""
+
+from dataclasses import dataclass, field
+from typing import Dict, List, Any, Optional, Tuple
+from collections import defaultdict
+
+
+@dataclass
+class ObjectCompressionStats:
+    """Statistics for a single object's compression."""
+
+    object_id: str
+    object_type: str  # "semantic", "episodic", "procedural"
+    original_size: int  # bytes
+    compressed_size: int  # bytes after delta encoding
+    compression_ratio: float  # compressed_size / original_size (0.0 = 100% compression)
+    delta_used: bool  # Whether delta encoding was applied
+    compression_benefit: float  # original_size - compressed_size
+
+
+@dataclass
+class TypeCompressionStats:
+    """Aggregated statistics for an object type."""
+
+    object_type: str
+    count: int = 0
+    total_original_size: int = 0
+    total_compressed_size: int = 0
+    avg_compression_ratio: float = 0.0
+    total_benefit: int = 0  # Total bytes saved
+    objects_with_delta: int = 0  # How many used delta encoding
+    min_ratio: float = 1.0
+    max_ratio: float = 0.0
+
+    def update_from_object(self, obj_stats: ObjectCompressionStats) -> None:
+        """Update type stats with a single object's stats."""
+        self.count += 1
+        self.total_original_size += obj_stats.original_size
+        self.total_compressed_size += obj_stats.compressed_size
+        self.total_benefit += int(obj_stats.compression_benefit)
+        if obj_stats.delta_used:
+            self.objects_with_delta += 1
+        self.min_ratio = min(self.min_ratio, obj_stats.compression_ratio)
+        self.max_ratio = max(self.max_ratio, obj_stats.compression_ratio)
+
+        # Recalculate average
+        if self.total_original_size > 0:
+            self.avg_compression_ratio = self.total_compressed_size / self.total_original_size
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dict for reporting."""
+        savings_pct = 0.0
+        if self.total_original_size > 0:
+            savings_pct = (self.total_benefit / self.total_original_size) * 100
+
+        return {
+            "object_type": self.object_type,
+            "count": self.count,
+            "total_original_bytes": self.total_original_size,
+            "total_compressed_bytes": self.total_compressed_size,
+            "avg_compression_ratio": round(self.avg_compression_ratio, 3),
+            "compression_range": f"{self.min_ratio:.1%} - {self.max_ratio:.1%}",
+            "total_bytes_saved": self.total_benefit,
+            "savings_percentage": round(savings_pct, 1),
+            "objects_using_delta": self.objects_with_delta,
+            "delta_adoption_rate": (
+                round((self.objects_with_delta / self.count * 100), 1) if self.count > 0 else 0
+            ),
+        }
+
+
+class DeltaCompressionMetrics:
+    """Tracks delta compression statistics across all objects.
+
+    Usage:
+        metrics = DeltaCompressionMetrics()
+        # ... during packing ...
+        metrics.record_object(ObjectCompressionStats(...))
+        # ... after packing ...
+        report = metrics.get_report()
+    """
+
+    def __init__(self):
+        self.objects: List[ObjectCompressionStats] = []
+        self.type_stats: Dict[str, TypeCompressionStats] = {}
+        self.total_original_size: int = 0
+        self.total_compressed_size: int = 0
+
+    def record_object(self, obj_stats: ObjectCompressionStats) -> None:
+        """Record compression stats for a single object."""
+        self.objects.append(obj_stats)
+        self.total_original_size += obj_stats.original_size
+        self.total_compressed_size += obj_stats.compressed_size
+
+        # Update type-specific stats
+        if obj_stats.object_type not in self.type_stats:
+            self.type_stats[obj_stats.object_type] = TypeCompressionStats(
+                object_type=obj_stats.object_type
+            )
+        self.type_stats[obj_stats.object_type].update_from_object(obj_stats)
+
+    def get_type_stats(self, object_type: str) -> Optional[TypeCompressionStats]:
+        """Get stats for a specific object type."""
+        return self.type_stats.get(object_type)
+
+    def get_overall_ratio(self) -> float:
+        """Get overall compression ratio across all objects."""
+        if self.total_original_size == 0:
+            return 0.0
+        return self.total_compressed_size / self.total_original_size
+
+    def get_overall_savings(self) -> int:
+        """Get total bytes saved across all objects."""
+        return self.total_original_size - self.total_compressed_size
+
+    def get_report(self) -> Dict[str, Any]:
+        """Generate a comprehensive compression report."""
+        overall_ratio = self.get_overall_ratio()
+        overall_savings = self.get_overall_savings()
+        savings_pct = (
+            (overall_savings / self.total_original_size * 100)
+            if self.total_original_size > 0
+            else 0
+        )
+
+        return {
+            "timestamp": None,  # Set by caller if needed
+            "total_objects": len(self.objects),
+            "total_original_bytes": self.total_original_size,
+            "total_compressed_bytes": self.total_compressed_size,
+            "overall_compression_ratio": round(overall_ratio, 3),
+            "total_bytes_saved": overall_savings,
+            "compression_percentage": round(savings_pct, 1),
+            "type_statistics": {otype: stats.to_dict() for otype, stats in self.type_stats.items()},
+            "recommendations": self._generate_recommendations(),
+        }
+
+    def _generate_recommendations(self) -> List[str]:
+        """Generate optimization recommendations based on compression stats."""
+        recommendations = []
+
+        # Check if delta encoding is worth it
+        objects_with_delta = sum(s.objects_with_delta for s in self.type_stats.values())
+        if objects_with_delta == 0:
+            recommendations.append("No objects used delta encoding. Check similarity thresholds.")
+
+        # Check for types with poor compression
+        for otype, stats in self.type_stats.items():
+            if stats.count > 0 and stats.avg_compression_ratio > 0.9:
+                recommendations.append(
+                    f"Type '{otype}' compresses poorly (ratio: {stats.avg_compression_ratio:.1%}). "
+                    f"Consider increasing similarity threshold or reducing delta cost."
+                )
+
+        # Check for types with excellent compression
+        for otype, stats in self.type_stats.items():
+            if stats.count > 0 and stats.avg_compression_ratio < 0.5:
+                recommendations.append(
+                    f"Type '{otype}' compresses very well (ratio: {stats.avg_compression_ratio:.1%}). "
+                    f"Consider aggressive delta encoding or reduced threshold."
+                )
+
+        if not recommendations:
+            recommendations.append("Compression is operating normally.")
+
+        return recommendations
+
+    def get_heatmap(self) -> str:
+        """Generate a text-based compression heatmap."""
+        lines = ["Delta Compression Heatmap", "=" * 50]
+
+        if not self.type_stats:
+            lines.append("No compression data available")
+            return "\n".join(lines)
+
+        # Sort by compression ratio
+        sorted_types = sorted(
+            self.type_stats.values(),
+            key=lambda s: s.avg_compression_ratio,
+        )
+
+        for stats in sorted_types:
+            if stats.count == 0:
+                continue
+            ratio = stats.avg_compression_ratio
+            # Create a simple bar chart
+            bar_width = 30
+            filled = int(bar_width * ratio)
+            bar = "█" * filled + "░" * (bar_width - filled)
+            saved_pct = (
+                (stats.total_benefit / stats.total_original_size * 100)
+                if stats.total_original_size > 0
+                else 0
+            )
+            lines.append(
+                f"{stats.object_type:12} {bar} {saved_pct:5.1f}% saved ({stats.objects_with_delta}/{stats.count} using delta)"
+            )
+
+        return "\n".join(lines)
+
+    def log_report(self, logger: Any = None) -> None:
+        """Log the compression report."""
+        report = self.get_report()
+        heatmap = self.get_heatmap()
+
+        output = [
+            "=" * 70,
+            "Delta Compression Report",
+            "=" * 70,
+            f"Total Objects: {report['total_objects']}",
+            f"Total Original: {report['total_original_bytes']:,} bytes",
+            f"Total Compressed: {report['total_compressed_bytes']:,} bytes",
+            f"Overall Ratio: {report['overall_compression_ratio']:.1%}",
+            f"Bytes Saved: {report['total_bytes_saved']:,} ({report['compression_percentage']:.1f}%)",
+            "",
+            heatmap,
+            "",
+            "Type Breakdown:",
+        ]
+
+        for otype, stats in sorted(report["type_statistics"].items()):
+            output.append(f"  {otype}:")
+            output.append(f"    Count: {stats['count']}")
+            output.append(f"    Compression: {stats['avg_compression_ratio']:.1%}")
+            output.append(f"    Saved: {stats['total_bytes_saved']:,} bytes")
+            output.append(f"    Delta adoption: {stats['delta_adoption_rate']:.0f}%")
+
+        output.extend(["", "Recommendations:"])
+        for rec in report["recommendations"]:
+            output.append(f"  - {rec}")
+
+        output.append("=" * 70)
+
+        full_output = "\n".join(output)
+        if logger:
+            logger.info(full_output)
+        else:
+            print(full_output)

memvcs/core/delta.py

@@ -0,0 +1,258 @@
+"""
+Delta encoding for pack files.
+
+Compress similar objects using delta encoding. For objects with similar content,
+store the first in full and subsequent ones as deltas (differences).
+
+This can achieve 5-10x compression improvement for highly similar content
+(common in agent episodic logs, semantic consolidations, etc).
+"""
+
+import hashlib
+from typing import List, Tuple, Dict, Optional
+
+
+def levenshtein_distance(s1: bytes, s2: bytes) -> int:
+    """
+    Compute Levenshtein distance between two byte sequences.
+    Returns edit distance (insertions, deletions, substitutions).
+    """
+    if len(s1) < len(s2):
+        s1, s2 = s2, s1
+
+    if len(s2) == 0:
+        return len(s1)
+
+    prev = list(range(len(s2) + 1))
+    for i, c1 in enumerate(s1):
+        curr = [i + 1]
+        for j, c2 in enumerate(s2):
+            insertions = prev[j + 1] + 1
+            deletions = curr[j] + 1
+            substitutions = prev[j] + (c1 != c2)
+            curr.append(min(insertions, deletions, substitutions))
+        prev = curr
+
+    return prev[-1]
+
+
+def content_similarity(data1: bytes, data2: bytes) -> float:
+    """
+    Calculate similarity between two byte sequences (0.0 to 1.0).
+    Based on Levenshtein distance normalized by max length.
+    """
+    if not data1 or not data2:
+        return 0.0
+
+    distance = levenshtein_distance(data1, data2)
+    max_len = max(len(data1), len(data2))
+
+    if max_len == 0:
+        return 1.0
+
+    return 1.0 - (distance / max_len)
+
+
+def find_similar_objects(
+    objects: Dict[str, bytes],
+    similarity_threshold: float = 0.7,
+    min_size: int = 100,
+) -> List[List[str]]:
+    """
+    Group objects by similarity.
+
+    Returns list of groups, where each group is a list of object hashes
+    sorted by size (smallest first - best compression base).
+    Only includes objects >= min_size.
+
+    Args:
+        objects: dict of hash_id -> content
+        similarity_threshold: minimum similarity (0.0-1.0) to group
+        min_size: minimum object size to consider for delta
+
+    Returns:
+        List of similarity groups, each sorted by size ascending
+    """
+    candidates = {h: content for h, content in objects.items() if len(content) >= min_size}
+
+    if not candidates:
+        return []
+
+    grouped = {}
+    used = set()
+
+    for hash_id, content in candidates.items():
+        if hash_id in used:
+            continue
+
+        group = [hash_id]
+        used.add(hash_id)
+
+        for other_id, other_content in candidates.items():
+            if other_id in used:
+                continue
+
+            similarity = content_similarity(content, other_content)
+            if similarity >= similarity_threshold:
+                group.append(other_id)
+                used.add(other_id)
+
+        if len(group) > 1:
+            # Sort by size ascending (smallest first = best base)
+            group.sort(key=lambda h: len(candidates[h]))
+            grouped[group[0]] = group
+
+    return list(grouped.values())
+
+
+def compute_delta(base: bytes, target: bytes) -> bytes:
+    """
+    Compute delta from base to target using simple run-length + offset encoding.
+
+    Format:
+    - 0x00: Copy op   - next 4 bytes = offset in base, next 4 bytes = length
+    - 0x01: Insert op - next 4 bytes = length, then <length> bytes of data
+    - 0x02: End marker
+
+    This is NOT the most efficient delta algorithm but simple and effective
+    for similar objects. Production code could use bsdiff, xdelta3, etc.
+    """
+    from difflib import SequenceMatcher
+
+    matcher = SequenceMatcher(None, base, target)
+    matching_blocks = matcher.get_matching_blocks()
+
+    delta = bytearray()
+    target_pos = 0
+
+    for block in matching_blocks:
+        base_start, target_start, size = block.a, block.b, block.size
+
+        # Insert any unmapped target bytes before this block
+        if target_start > target_pos:
+            insert_len = target_start - target_pos
+            insert_data = target[target_pos:target_start]
+            delta.append(0x01)  # Insert op
+            delta.extend(insert_len.to_bytes(4, "big"))
+            delta.extend(insert_data)
+
+        # Copy block from base
+        if size > 0:
+            delta.append(0x00)  # Copy op
+            delta.extend(base_start.to_bytes(4, "big"))
+            delta.extend(size.to_bytes(4, "big"))
+
+        target_pos = target_start + size
+
+    # Insert any remaining target bytes
+    if target_pos < len(target):
+        insert_len = len(target) - target_pos
+        insert_data = target[target_pos:]
+        delta.append(0x01)  # Insert op
+        delta.extend(insert_len.to_bytes(4, "big"))
+        delta.extend(insert_data)
+
+    delta.append(0x02)  # End marker
+
+    return bytes(delta)
+
+
+def apply_delta(base: bytes, delta: bytes) -> bytes:
+    """Apply delta to base to reconstruct target."""
+    result = bytearray()
+    pos = 0
+
+    while pos < len(delta):
+        op = delta[pos]
+        pos += 1
+
+        if op == 0x00:  # Copy op
+            if pos + 8 > len(delta):
+                break
+            offset = int.from_bytes(delta[pos : pos + 4], "big")
+            length = int.from_bytes(delta[pos + 4 : pos + 8], "big")
+            pos += 8
+            result.extend(base[offset : offset + length])
+
+        elif op == 0x01:  # Insert op
+            if pos + 4 > len(delta):
+                break
+            length = int.from_bytes(delta[pos : pos + 4], "big")
+            pos += 4
+            if pos + length > len(delta):
+                break
+            result.extend(delta[pos : pos + length])
+            pos += length
+
+        elif op == 0x02:  # End marker
+            break
+
+    return bytes(result)
+
+
+def estimate_delta_compression(base: bytes, target: bytes, delta: bytes) -> Tuple[int, float]:
+    """
+    Estimate compression achieved by delta.
+
+    Returns (original_size, ratio) where ratio = 1.0 is no compression,
+    ratio = 0.5 means delta is 50% of original target size.
+    """
+    original_size = len(target)
+    delta_size = len(delta)
+
+    if original_size == 0:
+        return (0, 0.0)
+
+    ratio = delta_size / original_size
+    return (original_size, ratio)
+
+
+class DeltaCache:
+    """
+    Cache deltas between similar objects.
+
+    Tracks base->target relationships and stores pre-computed deltas
+    to avoid recomputation.
+    """
+
+    def __init__(self):
+        self.deltas: Dict[Tuple[str, str], bytes] = {}  # (base_hash, target_hash) -> delta
+        self.bases: Dict[str, bytes] = {}  # target_hash -> base_hash (reconstruction path)
+
+    def add_delta(self, base_hash: str, target_hash: str, delta: bytes):
+        """Register a delta relationship."""
+        self.deltas[(base_hash, target_hash)] = delta
+        self.bases[target_hash] = base_hash
+
+    def get_delta(self, base_hash: str, target_hash: str) -> Optional[bytes]:
+        """Retrieve cached delta."""
+        return self.deltas.get((base_hash, target_hash))
+
+    def get_base(self, target_hash: str) -> Optional[str]:
+        """Get the base hash for a target."""
+        return self.bases.get(target_hash)
+
+    def estimate_total_savings(self, objects: Dict[str, int]) -> Tuple[int, int]:
+        """
+        Estimate total size savings from all deltas.
+
+        Returns (original_total, compressed_total).
+
+        Args:
+            objects: dict of hash_id -> original_size
+        """
+        original_total = sum(objects.values())
+        compressed_total = 0
+
+        for (base_hash, target_hash), delta in self.deltas.items():
+            # Target stored as delta instead of full copy
+            compressed_total += len(delta)
+
+        # Add all non-delta objects
+        all_objects = set(objects.keys())
+        delta_targets = set(self.bases.keys())
+        non_delta = all_objects - delta_targets
+        for obj_hash in non_delta:
+            compressed_total += objects.get(obj_hash, 0)
+
+        return (original_total, compressed_total)