superlocalmemory 2.7.6 → 2.8.0

package/src/lifecycle/__init__.py

@@ -0,0 +1,54 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""SLM v2.8 Lifecycle Engine — Memory State Machine + Bounded Growth.
+
+Manages memory states: ACTIVE → WARM → COLD → ARCHIVED → TOMBSTONED.
+Layers on top of existing tier-based compression.
+All features opt-in: absent config = v2.7 behavior.
+
+Graceful degradation: if this module fails to import,
+core memory operations continue unchanged.
+"""
+import threading
+from pathlib import Path
+from typing import Optional, Dict, Any
+
+# Feature flags
+LIFECYCLE_AVAILABLE = False
+_init_error = None
+
+try:
+    from .lifecycle_engine import LifecycleEngine
+    from .lifecycle_evaluator import LifecycleEvaluator
+    from .retention_policy import RetentionPolicyManager
+    from .bounded_growth import BoundedGrowthEnforcer
+    LIFECYCLE_AVAILABLE = True
+except ImportError as e:
+    _init_error = str(e)
+
+# Lazy singletons
+_lifecycle_engine: Optional["LifecycleEngine"] = None
+_lifecycle_lock = threading.Lock()
+
+
+def get_lifecycle_engine(db_path: Optional[Path] = None) -> Optional["LifecycleEngine"]:
+    """Get or create the lifecycle engine singleton. Returns None if unavailable."""
+    global _lifecycle_engine
+    if not LIFECYCLE_AVAILABLE:
+        return None
+    with _lifecycle_lock:
+        if _lifecycle_engine is None:
+            try:
+                _lifecycle_engine = LifecycleEngine(db_path)
+            except Exception:
+                return None
+        return _lifecycle_engine
+
+
+def get_status() -> Dict[str, Any]:
+    """Return lifecycle engine status."""
+    return {
+        "lifecycle_available": LIFECYCLE_AVAILABLE,
+        "init_error": _init_error,
+        "engine_active": _lifecycle_engine is not None,
+    }

package/src/lifecycle/bounded_growth.py

@@ -0,0 +1,239 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""Bounded growth enforcement — ensures memory counts stay within limits.
+
+When the count of memories in a given lifecycle state exceeds configurable
+bounds, the lowest-scoring memories are transitioned to the next state.
+
+Scoring formula: importance_norm * recency_factor * frequency_factor * behavioral_value
+  - importance_norm: importance / 10.0 (0.1 to 1.0)
+  - recency_factor: 1.0 / (1.0 + days_stale / 30.0) (exponential decay)
+  - frequency_factor: 0.5 + 0.5 * min(access_count / age_days, 1.0)
+  - behavioral_value: 1.0 (placeholder for Phase 2 integration)
+
+Lower score = evict first.
+"""
+import sqlite3
+import json
+from datetime import datetime
+from pathlib import Path
+from typing import Optional, Dict, Any, List
+
+from .lifecycle_engine import LifecycleEngine
+
+
+# Default bounds — generous defaults, configurable via lifecycle_config.json
+DEFAULT_BOUNDS: Dict[str, int] = {
+    "max_active": 10000,
+    "max_warm": 5000,
+}
+
+
+class BoundedGrowthEnforcer:
+    """Enforces memory count limits by transitioning lowest-scoring memories.
+
+    When active memories exceed max_active, the lowest-scoring transition to warm.
+    When warm memories exceed max_warm, the lowest-scoring transition to cold.
+    """
+
+    def __init__(
+        self,
+        db_path: Optional[str] = None,
+        config_path: Optional[str] = None,
+    ):
+        if db_path is None:
+            db_path = str(Path.home() / ".claude-memory" / "memory.db")
+        self._db_path = str(db_path)
+        self._config_path = config_path
+        self._engine = LifecycleEngine(self._db_path)
+
+    def _get_connection(self) -> sqlite3.Connection:
+        """Get a SQLite connection to memory.db."""
+        conn = sqlite3.connect(self._db_path)
+        conn.row_factory = sqlite3.Row
+        return conn
+
+    def enforce_bounds(self, profile: Optional[str] = None) -> Dict[str, Any]:
+        """Check memory counts and transition excess memories.
+
+        Args:
+            profile: Filter by profile (None = all)
+
+        Returns:
+            Dict with enforced status, counts, limits, and transitions list
+        """
+        bounds = self._load_bounds()
+        max_active = bounds.get("max_active", DEFAULT_BOUNDS["max_active"])
+        max_warm = bounds.get("max_warm", DEFAULT_BOUNDS["max_warm"])
+
+        transitions: List[Dict[str, Any]] = []
+
+        # Enforce active limit
+        active_transitions = self._enforce_state_limit(
+            state="active",
+            target_state="warm",
+            max_count=max_active,
+            profile=profile,
+        )
+        transitions.extend(active_transitions)
+
+        # Enforce warm limit
+        warm_transitions = self._enforce_state_limit(
+            state="warm",
+            target_state="cold",
+            max_count=max_warm,
+            profile=profile,
+        )
+        transitions.extend(warm_transitions)
+
+        # Build result
+        dist = self._engine.get_state_distribution()
+        return {
+            "enforced": len(transitions) > 0,
+            "active_count": dist.get("active", 0),
+            "active_limit": max_active,
+            "warm_count": dist.get("warm", 0),
+            "warm_limit": max_warm,
+            "transitions": transitions,
+        }
+
+    def _enforce_state_limit(
+        self,
+        state: str,
+        target_state: str,
+        max_count: int,
+        profile: Optional[str] = None,
+    ) -> List[Dict[str, Any]]:
+        """Enforce a single state's count limit.
+
+        Scores all memories in the given state, evicts the lowest-scoring
+        excess memories to target_state.
+        """
+        scored = self.score_all_memories(state=state, profile=profile)
+        current_count = len(scored)
+
+        if current_count <= max_count:
+            return []
+
+        excess = current_count - max_count
+        # Sort ascending by score — lowest scores evicted first
+        scored.sort(key=lambda s: s["score"])
+        to_evict = scored[:excess]
+
+        # Batch transition for performance — single connection, single commit
+        mem_ids = [e["memory_id"] for e in to_evict]
+        reasons = [f"bounded_growth_score_{e['score']:.4f}" for e in to_evict]
+        score_map = {e["memory_id"]: e["score"] for e in to_evict}
+
+        result = self._engine.batch_transition(mem_ids, target_state, reasons)
+
+        transitions = []
+        for entry in result.get("succeeded", []):
+            transitions.append({
+                "memory_id": entry["memory_id"],
+                "from_state": state,
+                "to_state": target_state,
+                "score": score_map.get(entry["memory_id"], 0.0),
+            })
+        return transitions
+
+    def score_all_memories(
+        self,
+        state: Optional[str] = None,
+        profile: Optional[str] = None,
+    ) -> List[Dict[str, Any]]:
+        """Score all memories in a given state.
+
+        Args:
+            state: Lifecycle state to filter (None = all non-terminal states)
+            profile: Filter by profile
+
+        Returns:
+            List of dicts with memory_id and score, sorted descending by score
+        """
+        conn = self._get_connection()
+        try:
+            if state:
+                query = (
+                    "SELECT id, importance, last_accessed, created_at, access_count "
+                    "FROM memories WHERE lifecycle_state = ?"
+                )
+                params: list = [state]
+            else:
+                query = (
+                    "SELECT id, importance, last_accessed, created_at, access_count "
+                    "FROM memories WHERE lifecycle_state IN ('active', 'warm', 'cold')"
+                )
+                params = []
+
+            if profile:
+                query += " AND profile = ?"
+                params.append(profile)
+
+            rows = conn.execute(query, params).fetchall()
+            now = datetime.now()
+
+            scores = []
+            for row in rows:
+                score = self._score_row(row, now)
+                scores.append({"memory_id": row["id"], "score": score})
+
+            scores.sort(key=lambda s: s["score"], reverse=True)
+            return scores
+        finally:
+            conn.close()
+
+    def _score_row(self, row: sqlite3.Row, now: datetime) -> float:
+        """Compute composite lifecycle score for a memory.
+
+        Score = importance_norm * recency_factor * frequency_factor * behavioral_value
+
+        Higher score = more valuable = keep longer.
+        """
+        # Importance: normalize to 0.1-1.0
+        importance = max(row["importance"] or 5, 1)
+        importance_norm = importance / 10.0
+
+        # Recency: exponential decay, halves every ~30 days
+        last_access_str = row["last_accessed"] or row["created_at"]
+        days_stale = 0
+        if last_access_str:
+            try:
+                last_access = datetime.fromisoformat(str(last_access_str))
+                days_stale = max((now - last_access).days, 0)
+            except (ValueError, TypeError):
+                days_stale = 0
+        recency_factor = 1.0 / (1.0 + days_stale / 30.0)
+
+        # Access frequency: normalized by age
+        access_count = row["access_count"] or 0
+        created_str = row["created_at"]
+        age_days = 1
+        if created_str:
+            try:
+                created = datetime.fromisoformat(str(created_str))
+                age_days = max((now - created).days, 1)
+            except (ValueError, TypeError):
+                age_days = 1
+        frequency_factor = 0.5 + 0.5 * min(access_count / age_days, 1.0)
+
+        # Behavioral value: placeholder for Phase 2 integration
+        behavioral_value = 1.0
+
+        return importance_norm * recency_factor * frequency_factor * behavioral_value
+
+    def _load_bounds(self) -> Dict[str, int]:
+        """Load bounds config from lifecycle_config.json. Returns defaults if missing."""
+        try:
+            if self._config_path:
+                config_path = Path(self._config_path)
+            else:
+                config_path = Path(self._db_path).parent / "lifecycle_config.json"
+            if config_path.exists():
+                with open(config_path) as f:
+                    user_config = json.load(f)
+                bounds = user_config.get("bounds", {})
+                return {**DEFAULT_BOUNDS, **bounds}
+        except Exception:
+            pass
+        return dict(DEFAULT_BOUNDS)

package/src/lifecycle/compaction_engine.py

@@ -0,0 +1,226 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""Formal memory compaction with information preservation guarantees.
+
+Archives full content before compaction, replaces with compressed summary
+containing key entities. Supports lossless restoration from archive.
+"""
+import re
+import sqlite3
+from datetime import datetime, timezone
+from typing import Dict, List, Optional
+
+
+# Common English stopwords for entity extraction (no external dependencies)
+_STOPWORDS = frozenset(
+    "a an the is are was were be been being have has had do does did will would "
+    "shall should may might can could am not no nor and but or if then else when "
+    "at by for with about against between through during before after above below "
+    "to from up down in out on off over under again further once here there all "
+    "each every both few more most other some such only own same so than too very "
+    "just don doesn didn won wouldn isn aren wasn weren hasn haven hadn it its "
+    "i me my myself we our ours ourselves you your yours yourself yourselves he "
+    "him his himself she her hers herself they them their theirs themselves what "
+    "which who whom this that these those of as into how also many use used uses "
+    "using like make makes made includes include including provides provide "
+    "widely ideal rapid many".split()
+)
+
+# Minimum word length for entity candidates
+_MIN_WORD_LEN = 3
+# Maximum number of entities to extract
+_MAX_ENTITIES = 8
+
+
+class CompactionEngine:
+    """Manages content compaction and restoration for memory lifecycle.
+
+    When a memory transitions to ARCHIVED state, this engine:
+    1. Saves the full content to memory_archive (lossless backup)
+    2. Replaces content with a compact summary + key entities
+    3. Can restore full content if memory is reactivated
+    """
+
+    def __init__(self, db_path: str) -> None:
+        self.db_path = db_path
+        self._ensure_archive_table()
+
+    def _get_conn(self) -> sqlite3.Connection:
+        """Create a new connection with row factory."""
+        conn = sqlite3.connect(self.db_path)
+        conn.row_factory = sqlite3.Row
+        return conn
+
+    def _ensure_archive_table(self) -> None:
+        """Ensure the memory_archive table exists."""
+        conn = self._get_conn()
+        try:
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS memory_archive (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    memory_id INTEGER UNIQUE NOT NULL,
+                    full_content TEXT NOT NULL,
+                    archived_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                    FOREIGN KEY (memory_id) REFERENCES memories(id) ON DELETE CASCADE
+                )
+            """)
+            conn.execute(
+                "CREATE INDEX IF NOT EXISTS idx_archive_memory "
+                "ON memory_archive(memory_id)"
+            )
+            conn.commit()
+        finally:
+            conn.close()
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def compact_memory(
+        self, memory_id: int, dry_run: bool = False
+    ) -> Dict:
+        """Compact a memory: archive full content, replace with summary.
+
+        Args:
+            memory_id: ID of the memory to compact.
+            dry_run: If True, compute result but do not modify the database.
+
+        Returns:
+            Dict with keys: success, entities, summary, original_length,
+            and optionally dry_run, error.
+        """
+        conn = self._get_conn()
+        try:
+            row = conn.execute(
+                "SELECT id, content, tags FROM memories WHERE id = ?",
+                (memory_id,),
+            ).fetchone()
+
+            if row is None:
+                return {"success": False, "error": "Memory not found"}
+
+            content = row["content"]
+            original_length = len(content)
+
+            # Extract key entities (pure-Python TF-IDF-like approach)
+            entities = self._extract_entities(content)
+            summary = self._build_summary(content, entities, original_length)
+
+            if dry_run:
+                return {
+                    "success": True,
+                    "dry_run": True,
+                    "entities": entities,
+                    "summary": summary,
+                    "original_length": original_length,
+                }
+
+            # Archive the full content (lossless backup)
+            now = datetime.now(timezone.utc).isoformat()
+            conn.execute(
+                "INSERT OR REPLACE INTO memory_archive "
+                "(memory_id, full_content, archived_at) VALUES (?, ?, ?)",
+                (memory_id, content, now),
+            )
+
+            # Replace memory content with compacted version
+            conn.execute(
+                "UPDATE memories SET content = ? WHERE id = ?",
+                (summary, memory_id),
+            )
+            conn.commit()
+
+            return {
+                "success": True,
+                "entities": entities,
+                "summary": summary,
+                "original_length": original_length,
+            }
+        finally:
+            conn.close()
+
+    def restore_memory(self, memory_id: int) -> Dict:
+        """Restore a compacted memory from its archive.
+
+        Args:
+            memory_id: ID of the memory to restore.
+
+        Returns:
+            Dict with keys: success, and optionally restored_length, error.
+        """
+        conn = self._get_conn()
+        try:
+            archive = conn.execute(
+                "SELECT full_content FROM memory_archive WHERE memory_id = ?",
+                (memory_id,),
+            ).fetchone()
+
+            if archive is None:
+                return {"success": False, "error": "No archive found for memory"}
+
+            full_content = archive["full_content"]
+
+            # Restore original content
+            conn.execute(
+                "UPDATE memories SET content = ? WHERE id = ?",
+                (full_content, memory_id),
+            )
+
+            # Remove the archive entry (content is back in main table)
+            conn.execute(
+                "DELETE FROM memory_archive WHERE memory_id = ?",
+                (memory_id,),
+            )
+            conn.commit()
+
+            return {
+                "success": True,
+                "restored_length": len(full_content),
+            }
+        finally:
+            conn.close()
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _extract_entities(text: str) -> List[str]:
+        """Extract key entities using word-frequency ranking.
+
+        Pure Python implementation — no sklearn or external NLP deps.
+        Tokenizes, removes stopwords, counts frequency, returns top terms.
+        """
+        # Tokenize: split on non-alphanumeric, lowercase
+        tokens = re.findall(r"[a-zA-Z]+", text.lower())
+
+        # Filter: remove stopwords and short tokens
+        meaningful = [
+            t for t in tokens
+            if t not in _STOPWORDS and len(t) >= _MIN_WORD_LEN
+        ]
+
+        # Count frequencies
+        freq: Dict[str, int] = {}
+        for token in meaningful:
+            freq[token] = freq.get(token, 0) + 1
+
+        # Sort by frequency descending, then alphabetically for stability
+        ranked = sorted(freq.items(), key=lambda x: (-x[1], x[0]))
+
+        # Return top N entities
+        return [word for word, _ in ranked[:_MAX_ENTITIES]]
+
+    @staticmethod
+    def _build_summary(
+        text: str, entities: List[str], original_length: int
+    ) -> str:
+        """Build the compacted content string.
+
+        Format: [COMPACTED] Key entities: e1, e2, ... Original length: N chars.
+        """
+        entity_str = ", ".join(entities) if entities else "none"
+        return (
+            f"[COMPACTED] Key entities: {entity_str}. "
+            f"Original length: {original_length} chars."
+        )