superlocalmemory 2.8.2 → 2.8.3

package/src/compression/cold_storage.py

@@ -0,0 +1,217 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""
+Cold storage management for archived memories.
+Handles compression and archival to gzipped JSON files.
+"""
+
+import sqlite3
+import json
+import gzip
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import List, Dict, Optional, Any
+
+from compression.config import CompressionConfig
+
+
+MEMORY_DIR = Path.home() / ".claude-memory"
+DB_PATH = MEMORY_DIR / "memory.db"
+COLD_STORAGE_PATH = MEMORY_DIR / "cold-storage"
+
+
+class ColdStorageManager:
+    """Manage cold storage archives for very old memories."""
+
+    def __init__(self, db_path: Path = DB_PATH, storage_path: Path = COLD_STORAGE_PATH):
+        self.db_path = db_path
+        self.storage_path = storage_path
+        self.storage_path.mkdir(exist_ok=True)
+        self.config = CompressionConfig()
+
+    def move_to_cold_storage(self, memory_ids: List[int]) -> int:
+        """
+        Move archived memories to gzipped JSON file.
+
+        Args:
+            memory_ids: List of memory IDs to archive
+
+        Returns:
+            Number of memories archived
+        """
+        if not memory_ids:
+            return 0
+
+        conn = sqlite3.connect(self.db_path)
+        try:
+            cursor = conn.cursor()
+
+            # Build placeholders for SQL query
+            placeholders = ','.join('?' * len(memory_ids))
+
+            # Get memories from archive table
+            cursor.execute(f'''
+                SELECT m.id, m.content, m.summary, m.tags, m.project_name,
+                       m.created_at, a.full_content
+                FROM memories m
+                LEFT JOIN memory_archive a ON m.id = a.memory_id
+                WHERE m.id IN ({placeholders})
+            ''', memory_ids)
+
+            memories = cursor.fetchall()
+
+            if not memories:
+                return 0
+
+            # Build JSON export
+            export_data = []
+
+            for memory in memories:
+                mem_id, content, summary, tags, project_name, created_at, full_content = memory
+
+                export_data.append({
+                    'id': mem_id,
+                    'tier3_content': self._safe_json_load(content),
+                    'summary': summary,
+                    'tags': self._safe_json_load(tags) if tags else [],
+                    'project': project_name,
+                    'created_at': created_at,
+                    'full_content': full_content  # May be None if not archived
+                })
+
+            # Write to gzipped file
+            filename = f"archive-{datetime.now().strftime('%Y-%m')}.json.gz"
+            filepath = self.storage_path / filename
+
+            # If file exists, append to it
+            existing_data = []
+            if filepath.exists():
+                try:
+                    with gzip.open(filepath, 'rt', encoding='utf-8') as f:
+                        existing_data = json.load(f)
+                except Exception:
+                    pass  # File might be corrupted, start fresh
+
+            # Merge with existing data (avoid duplicates)
+            existing_ids = {item['id'] for item in existing_data}
+            for item in export_data:
+                if item['id'] not in existing_ids:
+                    existing_data.append(item)
+
+            # Write combined data
+            with gzip.open(filepath, 'wt', encoding='utf-8') as f:
+                json.dump(existing_data, f, indent=2)
+
+            # Delete from archive table (keep Tier 3 version in main table)
+            cursor.executemany('DELETE FROM memory_archive WHERE memory_id = ?',
+                              [(mid,) for mid in memory_ids])
+
+            conn.commit()
+        finally:
+            conn.close()
+
+        return len(export_data)
+
+    def _safe_json_load(self, data: str) -> Any:
+        """Safely load JSON data."""
+        try:
+            return json.loads(data)
+        except (json.JSONDecodeError, TypeError):
+            return data
+
+    def restore_from_cold_storage(self, memory_id: int) -> Optional[str]:
+        """
+        Restore full content from cold storage archive.
+
+        Args:
+            memory_id: ID of memory to restore
+
+        Returns:
+            Full content if found, None otherwise
+        """
+        # Search all archive files
+        for archive_file in self.storage_path.glob('archive-*.json.gz'):
+            try:
+                with gzip.open(archive_file, 'rt', encoding='utf-8') as f:
+                    data = json.load(f)
+
+                    for memory in data:
+                        if memory['id'] == memory_id:
+                            full_content = memory.get('full_content')
+
+                            if full_content:
+                                # Restore to archive table
+                                conn = sqlite3.connect(self.db_path)
+                                try:
+                                    cursor = conn.cursor()
+
+                                    cursor.execute('''
+                                        INSERT OR REPLACE INTO memory_archive
+                                        (memory_id, full_content, archived_at)
+                                        VALUES (?, ?, CURRENT_TIMESTAMP)
+                                    ''', (memory_id, full_content))
+
+                                    conn.commit()
+                                finally:
+                                    conn.close()
+
+                                return full_content
+            except Exception as e:
+                print(f"Error reading archive {archive_file}: {e}")
+                continue
+
+        return None
+
+    def get_cold_storage_candidates(self) -> List[int]:
+        """Get memory IDs that are candidates for cold storage."""
+        threshold_date = datetime.now() - timedelta(days=self.config.cold_storage_threshold_days)
+
+        conn = sqlite3.connect(self.db_path)
+        try:
+            cursor = conn.cursor()
+
+            cursor.execute('''
+                SELECT id FROM memories
+                WHERE tier = 3
+                AND created_at < ?
+                AND importance < 8
+            ''', (threshold_date.isoformat(),))
+
+            memory_ids = [row[0] for row in cursor.fetchall()]
+        finally:
+            conn.close()
+
+        return memory_ids
+
+    def get_cold_storage_stats(self) -> Dict[str, Any]:
+        """Get statistics about cold storage."""
+        stats = {
+            'archive_count': 0,
+            'total_memories': 0,
+            'total_size_bytes': 0,
+            'archives': []
+        }
+
+        for archive_file in self.storage_path.glob('archive-*.json.gz'):
+            try:
+                size = archive_file.stat().st_size
+
+                with gzip.open(archive_file, 'rt', encoding='utf-8') as f:
+                    data = json.load(f)
+                    memory_count = len(data)
+
+                stats['archive_count'] += 1
+                stats['total_memories'] += memory_count
+                stats['total_size_bytes'] += size
+
+                stats['archives'].append({
+                    'filename': archive_file.name,
+                    'memory_count': memory_count,
+                    'size_bytes': size,
+                    'size_mb': round(size / 1024 / 1024, 2)
+                })
+            except Exception:
+                continue
+
+        return stats

package/src/compression/config.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""
+Compression configuration management.
+Handles loading and saving compression settings from config.json.
+"""
+
+import json
+from pathlib import Path
+from typing import Dict, Any
+
+
+MEMORY_DIR = Path.home() / ".claude-memory"
+CONFIG_PATH = MEMORY_DIR / "config.json"
+
+
+class CompressionConfig:
+    """Configuration for compression behavior."""
+
+    def __init__(self):
+        self.config = self._load_config()
+        self.compression_settings = self.config.get('compression', {})
+
+    def _load_config(self) -> Dict[str, Any]:
+        """Load configuration from config.json."""
+        if CONFIG_PATH.exists():
+            with open(CONFIG_PATH, 'r') as f:
+                return json.load(f)
+        return {}
+
+    def save(self):
+        """Save configuration back to config.json."""
+        with open(CONFIG_PATH, 'w') as f:
+            json.dump(self.config, f, indent=2)
+
+    @property
+    def enabled(self) -> bool:
+        return self.compression_settings.get('enabled', True)
+
+    @property
+    def tier2_threshold_days(self) -> int:
+        return self.compression_settings.get('tier2_threshold_days', 30)
+
+    @property
+    def tier3_threshold_days(self) -> int:
+        return self.compression_settings.get('tier3_threshold_days', 90)
+
+    @property
+    def cold_storage_threshold_days(self) -> int:
+        return self.compression_settings.get('cold_storage_threshold_days', 365)
+
+    @property
+    def preserve_high_importance(self) -> bool:
+        return self.compression_settings.get('preserve_high_importance', True)
+
+    @property
+    def preserve_recently_accessed(self) -> bool:
+        return self.compression_settings.get('preserve_recently_accessed', True)
+
+    def initialize_defaults(self):
+        """Initialize compression settings in config if not present."""
+        if 'compression' not in self.config:
+            self.config['compression'] = {
+                'enabled': True,
+                'tier2_threshold_days': 30,
+                'tier3_threshold_days': 90,
+                'cold_storage_threshold_days': 365,
+                'preserve_high_importance': True,
+                'preserve_recently_accessed': True
+            }
+            self.save()

package/src/compression/orchestrator.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""
+Compression orchestrator.
+Coordinates classification, compression, and archival operations.
+"""
+
+import sqlite3
+from datetime import datetime
+from pathlib import Path
+from typing import Dict, Any
+
+from compression.config import CompressionConfig
+from compression.tier_classifier import TierClassifier
+from compression.tier2_compressor import Tier2Compressor
+from compression.tier3_compressor import Tier3Compressor
+from compression.cold_storage import ColdStorageManager
+
+
+MEMORY_DIR = Path.home() / ".claude-memory"
+DB_PATH = MEMORY_DIR / "memory.db"
+
+
+class CompressionOrchestrator:
+    """Main orchestrator for compression operations."""
+
+    def __init__(self, db_path: Path = DB_PATH):
+        self.db_path = db_path
+        self.config = CompressionConfig()
+        self.classifier = TierClassifier(db_path)
+        self.tier2_compressor = Tier2Compressor(db_path)
+        self.tier3_compressor = Tier3Compressor(db_path)
+        self.cold_storage = ColdStorageManager(db_path)
+
+    def run_full_compression(self) -> Dict[str, Any]:
+        """
+        Run full compression cycle: classify, compress, and archive.
+
+        Returns:
+            Statistics about compression operation
+        """
+        if not self.config.enabled:
+            return {'status': 'disabled', 'message': 'Compression is disabled in config'}
+
+        stats = {
+            'started_at': datetime.now().isoformat(),
+            'tier_updates': 0,
+            'tier2_compressed': 0,
+            'tier3_compressed': 0,
+            'cold_stored': 0,
+            'errors': []
+        }
+
+        try:
+            # Step 1: Classify memories into tiers
+            tier_updates = self.classifier.classify_memories()
+            stats['tier_updates'] = len(tier_updates)
+
+            # Step 2: Compress Tier 2 memories
+            stats['tier2_compressed'] = self.tier2_compressor.compress_all_tier2()
+
+            # Step 3: Compress Tier 3 memories
+            stats['tier3_compressed'] = self.tier3_compressor.compress_all_tier3()
+
+            # Step 4: Move old memories to cold storage
+            candidates = self.cold_storage.get_cold_storage_candidates()
+            if candidates:
+                stats['cold_stored'] = self.cold_storage.move_to_cold_storage(candidates)
+
+            # Get final tier stats
+            stats['tier_stats'] = self.classifier.get_tier_stats()
+
+            # Calculate space savings
+            stats['space_savings'] = self._calculate_space_savings()
+
+        except Exception as e:
+            stats['errors'].append(str(e))
+
+        stats['completed_at'] = datetime.now().isoformat()
+        return stats
+
+    def _calculate_space_savings(self) -> Dict[str, Any]:
+        """Calculate estimated space savings from compression."""
+        conn = sqlite3.connect(self.db_path)
+        try:
+            cursor = conn.cursor()
+
+            # Get size of compressed content
+            cursor.execute('''
+                SELECT
+                    tier,
+                    COUNT(*) as count,
+                    SUM(LENGTH(content)) as total_size
+                FROM memories
+                GROUP BY tier
+            ''')
+
+            tier_sizes = {}
+            for tier, count, total_size in cursor.fetchall():
+                tier_sizes[tier] = {
+                    'count': count,
+                    'size_bytes': total_size or 0
+                }
+
+            # Get size of archived content
+            cursor.execute('''
+                SELECT
+                    COUNT(*) as count,
+                    SUM(LENGTH(full_content)) as total_size
+                FROM memory_archive
+            ''')
+            archive_count, archive_size = cursor.fetchone()
+
+        finally:
+            conn.close()
+
+        # Estimate original size if all were Tier 1
+        tier1_avg = tier_sizes.get(1, {}).get('size_bytes', 50000) / max(tier_sizes.get(1, {}).get('count', 1), 1)
+        total_memories = sum(t.get('count', 0) for t in tier_sizes.values())
+        estimated_original = int(tier1_avg * total_memories)
+
+        current_size = sum(t.get('size_bytes', 0) for t in tier_sizes.values())
+
+        return {
+            'estimated_original_bytes': estimated_original,
+            'current_size_bytes': current_size,
+            'savings_bytes': estimated_original - current_size,
+            'savings_percent': round((1 - current_size / max(estimated_original, 1)) * 100, 1),
+            'tier_breakdown': tier_sizes,
+            'archive_count': archive_count or 0,
+            'archive_size_bytes': archive_size or 0
+        }

package/src/compression/tier2_compressor.py

@@ -0,0 +1,228 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""
+Tier 2 compression logic.
+Compresses memories to summary + key excerpts format.
+"""
+
+import sqlite3
+import json
+import re
+from datetime import datetime
+from pathlib import Path
+from typing import List
+
+
+MEMORY_DIR = Path.home() / ".claude-memory"
+DB_PATH = MEMORY_DIR / "memory.db"
+
+
+class Tier2Compressor:
+    """Compress memories to summary + key excerpts (Tier 2)."""
+
+    def __init__(self, db_path: Path = DB_PATH):
+        self.db_path = db_path
+
+    def compress_to_tier2(self, memory_id: int) -> bool:
+        """
+        Compress memory to summary + excerpts.
+
+        Args:
+            memory_id: ID of memory to compress
+
+        Returns:
+            True if compression succeeded, False otherwise
+        """
+        conn = sqlite3.connect(self.db_path)
+        try:
+            cursor = conn.cursor()
+
+            # Get full content
+            cursor.execute('''
+                SELECT content, summary, tier FROM memories WHERE id = ?
+            ''', (memory_id,))
+            result = cursor.fetchone()
+
+            if not result:
+                return False
+
+            content, existing_summary, current_tier = result
+
+            # Skip if already compressed or in wrong tier
+            if current_tier != 2:
+                return False
+
+            # Check if already archived (don't re-compress)
+            cursor.execute('''
+                SELECT full_content FROM memory_archive WHERE memory_id = ?
+            ''', (memory_id,))
+            if cursor.fetchone():
+                return True  # Already compressed
+
+            # Try to parse as JSON (might already be compressed)
+            try:
+                parsed = json.loads(content)
+                if isinstance(parsed, dict) and 'summary' in parsed:
+                    return True  # Already compressed
+            except (json.JSONDecodeError, TypeError):
+                pass  # Not compressed yet
+
+            # Generate/enhance summary if needed
+            if not existing_summary or len(existing_summary) < 100:
+                summary = self._generate_summary(content)
+            else:
+                summary = existing_summary
+
+            # Extract key excerpts (important sentences, code blocks, lists)
+            excerpts = self._extract_key_excerpts(content)
+
+            # Store compressed version
+            compressed_content = {
+                'summary': summary,
+                'excerpts': excerpts,
+                'original_length': len(content),
+                'compressed_at': datetime.now().isoformat()
+            }
+
+            # Move full content to archive table
+            cursor.execute('''
+                INSERT INTO memory_archive (memory_id, full_content, archived_at)
+                VALUES (?, ?, CURRENT_TIMESTAMP)
+            ''', (memory_id, content))
+
+            # Update memory with compressed version
+            cursor.execute('''
+                UPDATE memories
+                SET content = ?, tier = 2, updated_at = CURRENT_TIMESTAMP
+                WHERE id = ?
+            ''', (json.dumps(compressed_content), memory_id))
+
+            conn.commit()
+            return True
+        finally:
+            conn.close()
+
+    def _generate_summary(self, content: str, max_length: int = 300) -> str:
+        """
+        Generate extractive summary from content.
+        Uses sentence scoring based on heuristics (no external LLM).
+
+        Args:
+            content: Full content text
+            max_length: Maximum summary length in characters
+
+        Returns:
+            Extracted summary
+        """
+        # Split into sentences
+        sentences = re.split(r'[.!?]+', content)
+
+        # Score sentences by importance (simple heuristic)
+        scored_sentences = []
+
+        for i, sent in enumerate(sentences):
+            sent = sent.strip()
+            if len(sent) < 10:
+                continue
+
+            score = 0
+
+            # Boost if contains tech terms
+            tech_terms = ['api', 'database', 'auth', 'component', 'function',
+                         'class', 'method', 'variable', 'error', 'bug', 'fix',
+                         'implement', 'refactor', 'test', 'deploy']
+            score += sum(1 for term in tech_terms if term in sent.lower())
+
+            # Boost if at start or end (thesis/conclusion)
+            if i == 0 or i == len(sentences) - 1:
+                score += 2
+
+            # Boost if contains numbers/specifics
+            if re.search(r'\d+', sent):
+                score += 1
+
+            # Boost if contains important keywords
+            important_keywords = ['important', 'critical', 'note', 'remember',
+                                 'key', 'main', 'primary', 'must', 'should']
+            score += sum(2 for kw in important_keywords if kw in sent.lower())
+
+            scored_sentences.append((score, sent))
+
+        # Take top sentences up to max_length
+        scored_sentences.sort(reverse=True, key=lambda x: x[0])
+
+        summary_parts = []
+        current_length = 0
+
+        for score, sent in scored_sentences:
+            if current_length + len(sent) > max_length:
+                break
+
+            summary_parts.append(sent)
+            current_length += len(sent)
+
+        if not summary_parts:
+            # Fallback: take first sentence
+            return sentences[0][:max_length] if sentences else content[:max_length]
+
+        return '. '.join(summary_parts) + '.'
+
+    def _extract_key_excerpts(self, content: str, max_excerpts: int = 3) -> List[str]:
+        """
+        Extract key excerpts (code blocks, lists, important paragraphs).
+
+        Args:
+            content: Full content text
+            max_excerpts: Maximum number of excerpts to extract
+
+        Returns:
+            List of excerpt strings
+        """
+        excerpts = []
+
+        # Extract code blocks (markdown or indented)
+        code_blocks = re.findall(r'```[\s\S]*?```', content)
+        excerpts.extend(code_blocks[:2])  # Max 2 code blocks
+
+        # Extract bullet lists
+        list_pattern = r'(?:^|\n)(?:[-*•]|\d+\.)\s+.+(?:\n(?:[-*•]|\d+\.)\s+.+)*'
+        lists = re.findall(list_pattern, content, re.MULTILINE)
+        if lists and len(excerpts) < max_excerpts:
+            excerpts.extend(lists[:1])  # Max 1 list
+
+        # Extract paragraphs with important keywords if we need more
+        if len(excerpts) < max_excerpts:
+            paragraphs = content.split('\n\n')
+            important_keywords = ['important', 'critical', 'note', 'remember', 'key']
+
+            for para in paragraphs:
+                if len(excerpts) >= max_excerpts:
+                    break
+
+                if any(kw in para.lower() for kw in important_keywords):
+                    # Truncate long paragraphs
+                    if len(para) > 200:
+                        para = para[:197] + '...'
+                    excerpts.append(para)
+
+        # Truncate if too many
+        return excerpts[:max_excerpts]
+
+    def compress_all_tier2(self) -> int:
+        """Compress all memories that are in Tier 2."""
+        conn = sqlite3.connect(self.db_path)
+        try:
+            cursor = conn.cursor()
+
+            cursor.execute('SELECT id FROM memories WHERE tier = 2')
+            memory_ids = [row[0] for row in cursor.fetchall()]
+        finally:
+            conn.close()
+
+        compressed_count = 0
+        for memory_id in memory_ids:
+            if self.compress_to_tier2(memory_id):
+                compressed_count += 1
+
+        return compressed_count