superlocalmemory 2.8.5 → 3.0.0

package/src/compression/cold_storage.py

@@ -1,217 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-License-Identifier: MIT
-# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
-"""
-Cold storage management for archived memories.
-Handles compression and archival to gzipped JSON files.
-"""
-
-import sqlite3
-import json
-import gzip
-from datetime import datetime, timedelta
-from pathlib import Path
-from typing import List, Dict, Optional, Any
-
-from compression.config import CompressionConfig
-
-
-MEMORY_DIR = Path.home() / ".claude-memory"
-DB_PATH = MEMORY_DIR / "memory.db"
-COLD_STORAGE_PATH = MEMORY_DIR / "cold-storage"
-
-
-class ColdStorageManager:
-    """Manage cold storage archives for very old memories."""
-
-    def __init__(self, db_path: Path = DB_PATH, storage_path: Path = COLD_STORAGE_PATH):
-        self.db_path = db_path
-        self.storage_path = storage_path
-        self.storage_path.mkdir(exist_ok=True)
-        self.config = CompressionConfig()
-
-    def move_to_cold_storage(self, memory_ids: List[int]) -> int:
-        """
-        Move archived memories to gzipped JSON file.
-
-        Args:
-            memory_ids: List of memory IDs to archive
-
-        Returns:
-            Number of memories archived
-        """
-        if not memory_ids:
-            return 0
-
-        conn = sqlite3.connect(self.db_path)
-        try:
-            cursor = conn.cursor()
-
-            # Build placeholders for SQL query
-            placeholders = ','.join('?' * len(memory_ids))
-
-            # Get memories from archive table
-            cursor.execute(f'''
-                SELECT m.id, m.content, m.summary, m.tags, m.project_name,
-                       m.created_at, a.full_content
-                FROM memories m
-                LEFT JOIN memory_archive a ON m.id = a.memory_id
-                WHERE m.id IN ({placeholders})
-            ''', memory_ids)
-
-            memories = cursor.fetchall()
-
-            if not memories:
-                return 0
-
-            # Build JSON export
-            export_data = []
-
-            for memory in memories:
-                mem_id, content, summary, tags, project_name, created_at, full_content = memory
-
-                export_data.append({
-                    'id': mem_id,
-                    'tier3_content': self._safe_json_load(content),
-                    'summary': summary,
-                    'tags': self._safe_json_load(tags) if tags else [],
-                    'project': project_name,
-                    'created_at': created_at,
-                    'full_content': full_content  # May be None if not archived
-                })
-
-            # Write to gzipped file
-            filename = f"archive-{datetime.now().strftime('%Y-%m')}.json.gz"
-            filepath = self.storage_path / filename
-
-            # If file exists, append to it
-            existing_data = []
-            if filepath.exists():
-                try:
-                    with gzip.open(filepath, 'rt', encoding='utf-8') as f:
-                        existing_data = json.load(f)
-                except Exception:
-                    pass  # File might be corrupted, start fresh
-
-            # Merge with existing data (avoid duplicates)
-            existing_ids = {item['id'] for item in existing_data}
-            for item in export_data:
-                if item['id'] not in existing_ids:
-                    existing_data.append(item)
-
-            # Write combined data
-            with gzip.open(filepath, 'wt', encoding='utf-8') as f:
-                json.dump(existing_data, f, indent=2)
-
-            # Delete from archive table (keep Tier 3 version in main table)
-            cursor.executemany('DELETE FROM memory_archive WHERE memory_id = ?',
-                              [(mid,) for mid in memory_ids])
-
-            conn.commit()
-        finally:
-            conn.close()
-
-        return len(export_data)
-
-    def _safe_json_load(self, data: str) -> Any:
-        """Safely load JSON data."""
-        try:
-            return json.loads(data)
-        except (json.JSONDecodeError, TypeError):
-            return data
-
-    def restore_from_cold_storage(self, memory_id: int) -> Optional[str]:
-        """
-        Restore full content from cold storage archive.
-
-        Args:
-            memory_id: ID of memory to restore
-
-        Returns:
-            Full content if found, None otherwise
-        """
-        # Search all archive files
-        for archive_file in self.storage_path.glob('archive-*.json.gz'):
-            try:
-                with gzip.open(archive_file, 'rt', encoding='utf-8') as f:
-                    data = json.load(f)
-
-                    for memory in data:
-                        if memory['id'] == memory_id:
-                            full_content = memory.get('full_content')
-
-                            if full_content:
-                                # Restore to archive table
-                                conn = sqlite3.connect(self.db_path)
-                                try:
-                                    cursor = conn.cursor()
-
-                                    cursor.execute('''
-                                        INSERT OR REPLACE INTO memory_archive
-                                        (memory_id, full_content, archived_at)
-                                        VALUES (?, ?, CURRENT_TIMESTAMP)
-                                    ''', (memory_id, full_content))
-
-                                    conn.commit()
-                                finally:
-                                    conn.close()
-
-                                return full_content
-            except Exception as e:
-                print(f"Error reading archive {archive_file}: {e}")
-                continue
-
-        return None
-
-    def get_cold_storage_candidates(self) -> List[int]:
-        """Get memory IDs that are candidates for cold storage."""
-        threshold_date = datetime.now() - timedelta(days=self.config.cold_storage_threshold_days)
-
-        conn = sqlite3.connect(self.db_path)
-        try:
-            cursor = conn.cursor()
-
-            cursor.execute('''
-                SELECT id FROM memories
-                WHERE tier = 3
-                AND created_at < ?
-                AND importance < 8
-            ''', (threshold_date.isoformat(),))
-
-            memory_ids = [row[0] for row in cursor.fetchall()]
-        finally:
-            conn.close()
-
-        return memory_ids
-
-    def get_cold_storage_stats(self) -> Dict[str, Any]:
-        """Get statistics about cold storage."""
-        stats = {
-            'archive_count': 0,
-            'total_memories': 0,
-            'total_size_bytes': 0,
-            'archives': []
-        }
-
-        for archive_file in self.storage_path.glob('archive-*.json.gz'):
-            try:
-                size = archive_file.stat().st_size
-
-                with gzip.open(archive_file, 'rt', encoding='utf-8') as f:
-                    data = json.load(f)
-                    memory_count = len(data)
-
-                stats['archive_count'] += 1
-                stats['total_memories'] += memory_count
-                stats['total_size_bytes'] += size
-
-                stats['archives'].append({
-                    'filename': archive_file.name,
-                    'memory_count': memory_count,
-                    'size_bytes': size,
-                    'size_mb': round(size / 1024 / 1024, 2)
-                })
-            except Exception:
-                continue
-
-        return stats

package/src/compression/config.py

@@ -1,72 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-License-Identifier: MIT
-# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
-"""
-Compression configuration management.
-Handles loading and saving compression settings from config.json.
-"""
-
-import json
-from pathlib import Path
-from typing import Dict, Any
-
-
-MEMORY_DIR = Path.home() / ".claude-memory"
-CONFIG_PATH = MEMORY_DIR / "config.json"
-
-
-class CompressionConfig:
-    """Configuration for compression behavior."""
-
-    def __init__(self):
-        self.config = self._load_config()
-        self.compression_settings = self.config.get('compression', {})
-
-    def _load_config(self) -> Dict[str, Any]:
-        """Load configuration from config.json."""
-        if CONFIG_PATH.exists():
-            with open(CONFIG_PATH, 'r') as f:
-                return json.load(f)
-        return {}
-
-    def save(self):
-        """Save configuration back to config.json."""
-        with open(CONFIG_PATH, 'w') as f:
-            json.dump(self.config, f, indent=2)
-
-    @property
-    def enabled(self) -> bool:
-        return self.compression_settings.get('enabled', True)
-
-    @property
-    def tier2_threshold_days(self) -> int:
-        return self.compression_settings.get('tier2_threshold_days', 30)
-
-    @property
-    def tier3_threshold_days(self) -> int:
-        return self.compression_settings.get('tier3_threshold_days', 90)
-
-    @property
-    def cold_storage_threshold_days(self) -> int:
-        return self.compression_settings.get('cold_storage_threshold_days', 365)
-
-    @property
-    def preserve_high_importance(self) -> bool:
-        return self.compression_settings.get('preserve_high_importance', True)
-
-    @property
-    def preserve_recently_accessed(self) -> bool:
-        return self.compression_settings.get('preserve_recently_accessed', True)
-
-    def initialize_defaults(self):
-        """Initialize compression settings in config if not present."""
-        if 'compression' not in self.config:
-            self.config['compression'] = {
-                'enabled': True,
-                'tier2_threshold_days': 30,
-                'tier3_threshold_days': 90,
-                'cold_storage_threshold_days': 365,
-                'preserve_high_importance': True,
-                'preserve_recently_accessed': True
-            }
-            self.save()

package/src/compression/orchestrator.py

@@ -1,133 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-License-Identifier: MIT
-# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
-"""
-Compression orchestrator.
-Coordinates classification, compression, and archival operations.
-"""
-
-import sqlite3
-from datetime import datetime
-from pathlib import Path
-from typing import Dict, Any
-
-from compression.config import CompressionConfig
-from compression.tier_classifier import TierClassifier
-from compression.tier2_compressor import Tier2Compressor
-from compression.tier3_compressor import Tier3Compressor
-from compression.cold_storage import ColdStorageManager
-
-
-MEMORY_DIR = Path.home() / ".claude-memory"
-DB_PATH = MEMORY_DIR / "memory.db"
-
-
-class CompressionOrchestrator:
-    """Main orchestrator for compression operations."""
-
-    def __init__(self, db_path: Path = DB_PATH):
-        self.db_path = db_path
-        self.config = CompressionConfig()
-        self.classifier = TierClassifier(db_path)
-        self.tier2_compressor = Tier2Compressor(db_path)
-        self.tier3_compressor = Tier3Compressor(db_path)
-        self.cold_storage = ColdStorageManager(db_path)
-
-    def run_full_compression(self) -> Dict[str, Any]:
-        """
-        Run full compression cycle: classify, compress, and archive.
-
-        Returns:
-            Statistics about compression operation
-        """
-        if not self.config.enabled:
-            return {'status': 'disabled', 'message': 'Compression is disabled in config'}
-
-        stats = {
-            'started_at': datetime.now().isoformat(),
-            'tier_updates': 0,
-            'tier2_compressed': 0,
-            'tier3_compressed': 0,
-            'cold_stored': 0,
-            'errors': []
-        }
-
-        try:
-            # Step 1: Classify memories into tiers
-            tier_updates = self.classifier.classify_memories()
-            stats['tier_updates'] = len(tier_updates)
-
-            # Step 2: Compress Tier 2 memories
-            stats['tier2_compressed'] = self.tier2_compressor.compress_all_tier2()
-
-            # Step 3: Compress Tier 3 memories
-            stats['tier3_compressed'] = self.tier3_compressor.compress_all_tier3()
-
-            # Step 4: Move old memories to cold storage
-            candidates = self.cold_storage.get_cold_storage_candidates()
-            if candidates:
-                stats['cold_stored'] = self.cold_storage.move_to_cold_storage(candidates)
-
-            # Get final tier stats
-            stats['tier_stats'] = self.classifier.get_tier_stats()
-
-            # Calculate space savings
-            stats['space_savings'] = self._calculate_space_savings()
-
-        except Exception as e:
-            stats['errors'].append(str(e))
-
-        stats['completed_at'] = datetime.now().isoformat()
-        return stats
-
-    def _calculate_space_savings(self) -> Dict[str, Any]:
-        """Calculate estimated space savings from compression."""
-        conn = sqlite3.connect(self.db_path)
-        try:
-            cursor = conn.cursor()
-
-            # Get size of compressed content
-            cursor.execute('''
-                SELECT
-                    tier,
-                    COUNT(*) as count,
-                    SUM(LENGTH(content)) as total_size
-                FROM memories
-                GROUP BY tier
-            ''')
-
-            tier_sizes = {}
-            for tier, count, total_size in cursor.fetchall():
-                tier_sizes[tier] = {
-                    'count': count,
-                    'size_bytes': total_size or 0
-                }
-
-            # Get size of archived content
-            cursor.execute('''
-                SELECT
-                    COUNT(*) as count,
-                    SUM(LENGTH(full_content)) as total_size
-                FROM memory_archive
-            ''')
-            archive_count, archive_size = cursor.fetchone()
-
-        finally:
-            conn.close()
-
-        # Estimate original size if all were Tier 1
-        tier1_avg = tier_sizes.get(1, {}).get('size_bytes', 50000) / max(tier_sizes.get(1, {}).get('count', 1), 1)
-        total_memories = sum(t.get('count', 0) for t in tier_sizes.values())
-        estimated_original = int(tier1_avg * total_memories)
-
-        current_size = sum(t.get('size_bytes', 0) for t in tier_sizes.values())
-
-        return {
-            'estimated_original_bytes': estimated_original,
-            'current_size_bytes': current_size,
-            'savings_bytes': estimated_original - current_size,
-            'savings_percent': round((1 - current_size / max(estimated_original, 1)) * 100, 1),
-            'tier_breakdown': tier_sizes,
-            'archive_count': archive_count or 0,
-            'archive_size_bytes': archive_size or 0
-        }

package/src/compression/tier2_compressor.py

@@ -1,228 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-License-Identifier: MIT
-# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
-"""
-Tier 2 compression logic.
-Compresses memories to summary + key excerpts format.
-"""
-
-import sqlite3
-import json
-import re
-from datetime import datetime
-from pathlib import Path
-from typing import List
-
-
-MEMORY_DIR = Path.home() / ".claude-memory"
-DB_PATH = MEMORY_DIR / "memory.db"
-
-
-class Tier2Compressor:
-    """Compress memories to summary + key excerpts (Tier 2)."""
-
-    def __init__(self, db_path: Path = DB_PATH):
-        self.db_path = db_path
-
-    def compress_to_tier2(self, memory_id: int) -> bool:
-        """
-        Compress memory to summary + excerpts.
-
-        Args:
-            memory_id: ID of memory to compress
-
-        Returns:
-            True if compression succeeded, False otherwise
-        """
-        conn = sqlite3.connect(self.db_path)
-        try:
-            cursor = conn.cursor()
-
-            # Get full content
-            cursor.execute('''
-                SELECT content, summary, tier FROM memories WHERE id = ?
-            ''', (memory_id,))
-            result = cursor.fetchone()
-
-            if not result:
-                return False
-
-            content, existing_summary, current_tier = result
-
-            # Skip if already compressed or in wrong tier
-            if current_tier != 2:
-                return False
-
-            # Check if already archived (don't re-compress)
-            cursor.execute('''
-                SELECT full_content FROM memory_archive WHERE memory_id = ?
-            ''', (memory_id,))
-            if cursor.fetchone():
-                return True  # Already compressed
-
-            # Try to parse as JSON (might already be compressed)
-            try:
-                parsed = json.loads(content)
-                if isinstance(parsed, dict) and 'summary' in parsed:
-                    return True  # Already compressed
-            except (json.JSONDecodeError, TypeError):
-                pass  # Not compressed yet
-
-            # Generate/enhance summary if needed
-            if not existing_summary or len(existing_summary) < 100:
-                summary = self._generate_summary(content)
-            else:
-                summary = existing_summary
-
-            # Extract key excerpts (important sentences, code blocks, lists)
-            excerpts = self._extract_key_excerpts(content)
-
-            # Store compressed version
-            compressed_content = {
-                'summary': summary,
-                'excerpts': excerpts,
-                'original_length': len(content),
-                'compressed_at': datetime.now().isoformat()
-            }
-
-            # Move full content to archive table
-            cursor.execute('''
-                INSERT INTO memory_archive (memory_id, full_content, archived_at)
-                VALUES (?, ?, CURRENT_TIMESTAMP)
-            ''', (memory_id, content))
-
-            # Update memory with compressed version
-            cursor.execute('''
-                UPDATE memories
-                SET content = ?, tier = 2, updated_at = CURRENT_TIMESTAMP
-                WHERE id = ?
-            ''', (json.dumps(compressed_content), memory_id))
-
-            conn.commit()
-            return True
-        finally:
-            conn.close()
-
-    def _generate_summary(self, content: str, max_length: int = 300) -> str:
-        """
-        Generate extractive summary from content.
-        Uses sentence scoring based on heuristics (no external LLM).
-
-        Args:
-            content: Full content text
-            max_length: Maximum summary length in characters
-
-        Returns:
-            Extracted summary
-        """
-        # Split into sentences
-        sentences = re.split(r'[.!?]+', content)
-
-        # Score sentences by importance (simple heuristic)
-        scored_sentences = []
-
-        for i, sent in enumerate(sentences):
-            sent = sent.strip()
-            if len(sent) < 10:
-                continue
-
-            score = 0
-
-            # Boost if contains tech terms
-            tech_terms = ['api', 'database', 'auth', 'component', 'function',
-                         'class', 'method', 'variable', 'error', 'bug', 'fix',
-                         'implement', 'refactor', 'test', 'deploy']
-            score += sum(1 for term in tech_terms if term in sent.lower())
-
-            # Boost if at start or end (thesis/conclusion)
-            if i == 0 or i == len(sentences) - 1:
-                score += 2
-
-            # Boost if contains numbers/specifics
-            if re.search(r'\d+', sent):
-                score += 1
-
-            # Boost if contains important keywords
-            important_keywords = ['important', 'critical', 'note', 'remember',
-                                 'key', 'main', 'primary', 'must', 'should']
-            score += sum(2 for kw in important_keywords if kw in sent.lower())
-
-            scored_sentences.append((score, sent))
-
-        # Take top sentences up to max_length
-        scored_sentences.sort(reverse=True, key=lambda x: x[0])
-
-        summary_parts = []
-        current_length = 0
-
-        for score, sent in scored_sentences:
-            if current_length + len(sent) > max_length:
-                break
-
-            summary_parts.append(sent)
-            current_length += len(sent)
-
-        if not summary_parts:
-            # Fallback: take first sentence
-            return sentences[0][:max_length] if sentences else content[:max_length]
-
-        return '. '.join(summary_parts) + '.'
-
-    def _extract_key_excerpts(self, content: str, max_excerpts: int = 3) -> List[str]:
-        """
-        Extract key excerpts (code blocks, lists, important paragraphs).
-
-        Args:
-            content: Full content text
-            max_excerpts: Maximum number of excerpts to extract
-
-        Returns:
-            List of excerpt strings
-        """
-        excerpts = []
-
-        # Extract code blocks (markdown or indented)
-        code_blocks = re.findall(r'```[\s\S]*?```', content)
-        excerpts.extend(code_blocks[:2])  # Max 2 code blocks
-
-        # Extract bullet lists
-        list_pattern = r'(?:^|\n)(?:[-*•]|\d+\.)\s+.+(?:\n(?:[-*•]|\d+\.)\s+.+)*'
-        lists = re.findall(list_pattern, content, re.MULTILINE)
-        if lists and len(excerpts) < max_excerpts:
-            excerpts.extend(lists[:1])  # Max 1 list
-
-        # Extract paragraphs with important keywords if we need more
-        if len(excerpts) < max_excerpts:
-            paragraphs = content.split('\n\n')
-            important_keywords = ['important', 'critical', 'note', 'remember', 'key']
-
-            for para in paragraphs:
-                if len(excerpts) >= max_excerpts:
-                    break
-
-                if any(kw in para.lower() for kw in important_keywords):
-                    # Truncate long paragraphs
-                    if len(para) > 200:
-                        para = para[:197] + '...'
-                    excerpts.append(para)
-
-        # Truncate if too many
-        return excerpts[:max_excerpts]
-
-    def compress_all_tier2(self) -> int:
-        """Compress all memories that are in Tier 2."""
-        conn = sqlite3.connect(self.db_path)
-        try:
-            cursor = conn.cursor()
-
-            cursor.execute('SELECT id FROM memories WHERE tier = 2')
-            memory_ids = [row[0] for row in cursor.fetchall()]
-        finally:
-            conn.close()
-
-        compressed_count = 0
-        for memory_id in memory_ids:
-            if self.compress_to_tier2(memory_id):
-                compressed_count += 1
-
-        return compressed_count