superlocalmemory 3.4.9 → 3.4.11

package/src/superlocalmemory/mcp/tools_evolution.py

@@ -0,0 +1,338 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""SLM v3.4.11 "Skill Evolution" — Evolution MCP Tools.
+
+Three evolution tools:
+  - evolve_skill: Manually trigger evolution for a specific skill
+  - skill_health: Get health metrics for a skill or all skills
+  - skill_lineage: Get evolution lineage for a skill
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import sqlite3
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Callable
+
+logger = logging.getLogger(__name__)
+
+MEMORY_DB = Path.home() / ".superlocalmemory" / "memory.db"
+
+
+def register_evolution_tools(server, get_engine: Callable) -> None:
+    """Register evolution MCP tools for skill evolution intelligence."""
+
+    @server.tool()
+    async def evolve_skill(
+        skill_name: str,
+        evolution_type: str = "fix",
+        reason: str = "",
+    ) -> dict:
+        """Manually trigger evolution for a specific skill.
+
+        Runs the full evolution pipeline: screen -> confirm -> mutate ->
+        blind verify -> persist. Requires evolution to be enabled in config.
+
+        Args:
+            skill_name: Name of the skill to evolve (e.g. "brainstorming")
+            evolution_type: One of 'fix', 'derived', 'captured'
+            reason: Optional reason / evidence for the evolution
+        """
+        try:
+            # Check if evolution is enabled in config
+            config_path = Path.home() / ".superlocalmemory" / "config.json"
+            evo_cfg = {}
+            if config_path.exists():
+                with open(config_path) as f:
+                    cfg = json.load(f)
+                evo_cfg = cfg.get("evolution", {})
+
+            if not evo_cfg.get("enabled", False):
+                return {
+                    "success": False,
+                    "error": "Evolution is disabled. Enable via: slm config set evolution.enabled true",
+                }
+
+            from superlocalmemory.evolution.skill_evolver import SkillEvolver
+            from superlocalmemory.evolution.types import (
+                EvolutionCandidate,
+                EvolutionType,
+                TriggerType,
+            )
+
+            # Map string to enum
+            type_map = {"fix": EvolutionType.FIX, "derived": EvolutionType.DERIVED, "captured": EvolutionType.CAPTURED}
+            evo_type = type_map.get(evolution_type, EvolutionType.FIX)
+
+            # Build a minimal config for the evolver
+            class _EvoCfg:
+                enabled = True
+                backend = evo_cfg.get("backend", "auto")
+                max_evolutions_per_cycle = evo_cfg.get("max_evolutions_per_cycle", 3)
+
+            class _Cfg:
+                evolution = _EvoCfg()
+
+            db_path = str(MEMORY_DB)
+            evolver = SkillEvolver(db_path, _Cfg())
+
+            # Build candidate from manual trigger
+            evidence = (reason,) if reason else ("Manual evolution trigger via MCP",)
+            candidate = EvolutionCandidate(
+                skill_name=skill_name,
+                evolution_type=evo_type,
+                trigger=TriggerType.HEALTH_CHECK,
+                evidence=evidence,
+                effective_score=0.0,
+                invocation_count=0,
+            )
+
+            # Process through the pipeline
+            engine = get_engine()
+            profile_id = engine.profile_id if engine else "default"
+
+            evolver._store.reset_cycle()
+            outcome = evolver._process_candidate(candidate, profile_id)
+
+            # Fetch the latest record for this skill to return details
+            recent = evolver._store.get_skill_history(skill_name, limit=1)
+            record_info = {}
+            if recent:
+                r = recent[0]
+                record_info = {
+                    "id": r.id,
+                    "status": r.status.value,
+                    "mutation_summary": r.mutation_summary,
+                    "blind_verified": r.blind_verified,
+                    "rejection_reason": r.rejection_reason,
+                }
+
+            return {
+                "success": outcome == "evolved",
+                "outcome": outcome,
+                "skill_name": skill_name,
+                "evolution_type": evolution_type,
+                **record_info,
+            }
+        except Exception as exc:
+            logger.debug("evolve_skill failed: %s", exc)
+            return {"success": False, "error": str(exc)}
+
+    @server.tool()
+    async def skill_health(
+        skill_name: str = "",
+        include_history: bool = False,
+    ) -> dict:
+        """Get health metrics for a skill or all skills.
+
+        Queries behavioral assertions (skill_performance category) and
+        tool_events to compute per-skill invocation counts, effective
+        rates, and status.
+
+        Args:
+            skill_name: Specific skill name (empty = all skills)
+            include_history: Include recent tool event history per skill
+        """
+        try:
+            engine = get_engine()
+            profile_id = engine.profile_id if engine else "default"
+            db_path = str(MEMORY_DB)
+
+            conn = sqlite3.connect(db_path, timeout=10)
+            conn.row_factory = sqlite3.Row
+
+            # Gather per-skill invocation stats from tool_events
+            # Skills are logged as tool_name='Skill' with actual skill name in input_summary
+            if skill_name:
+                # M-LIKE: Escape LIKE wildcards in user-provided skill_name
+                safe_name = skill_name.replace('\\', '\\\\').replace('%', r'\%').replace('_', r'\_')
+                event_query = (
+                    "SELECT input_summary, event_type, created_at, duration_ms "
+                    "FROM tool_events "
+                    "WHERE profile_id = ? AND tool_name = 'Skill' "
+                    "AND input_summary LIKE ? ESCAPE '\\' "
+                    "ORDER BY created_at DESC"
+                )
+                event_rows = conn.execute(event_query, (profile_id, f"%{safe_name}%")).fetchall()
+                # Aggregate
+                invocations = len(event_rows)
+                errors = sum(1 for r in event_rows if dict(r).get("event_type") == "error")
+                last_invoked = dict(event_rows[0]).get("created_at", "") if event_rows else ""
+                effective_rate = ((invocations - errors) / invocations) if invocations > 0 else 0.0
+                skill_entries = [{
+                    "name": skill_name,
+                    "invocations": invocations,
+                    "errors": errors,
+                    "effective_rate": round(effective_rate, 4),
+                    "last_invoked": last_invoked,
+                    "status": "healthy" if effective_rate >= 0.7 else ("degraded" if effective_rate >= 0.4 else "critical"),
+                }]
+                if include_history:
+                    skill_entries[0]["recent_events"] = [
+                        dict(r) for r in event_rows[:10]
+                    ]
+            else:
+                # Get all Skill tool events and extract skill names from input_summary
+                event_query = (
+                    "SELECT input_summary, event_type, created_at "
+                    "FROM tool_events "
+                    "WHERE profile_id = ? AND tool_name = 'Skill' "
+                    "ORDER BY created_at DESC LIMIT 500"
+                )
+                event_rows = conn.execute(event_query, (profile_id,)).fetchall()
+
+                # Parse skill names from input_summary and aggregate
+                from collections import defaultdict
+                skill_stats: dict = defaultdict(lambda: {"invocations": 0, "errors": 0, "last_invoked": ""})
+                for row in event_rows:
+                    r = dict(row)
+                    summary = r.get("input_summary", "")
+                    # Extract skill name from JSON or plain text
+                    sname = ""
+                    try:
+                        parsed = json.loads(summary)
+                        sname = parsed.get("skill", "") or parsed.get("name", "")
+                    except (json.JSONDecodeError, TypeError):
+                        if ":" in summary:
+                            sname = summary.split('"')[1] if '"' in summary else summary.strip()
+                    if not sname:
+                        continue
+                    stats = skill_stats[sname]
+                    stats["invocations"] += 1
+                    if r.get("event_type") == "error":
+                        stats["errors"] += 1
+                    if not stats["last_invoked"]:
+                        stats["last_invoked"] = r.get("created_at", "")
+
+                skill_entries = []
+                for sname, stats in sorted(skill_stats.items(), key=lambda x: x[1]["invocations"], reverse=True)[:50]:
+                    inv = stats["invocations"]
+                    errs = stats["errors"]
+                    eff = ((inv - errs) / inv) if inv > 0 else 0.0
+                    skill_entries.append({
+                        "name": sname,
+                        "invocations": inv,
+                        "errors": errs,
+                        "effective_rate": round(eff, 4),
+                        "last_invoked": stats["last_invoked"],
+                        "status": "healthy" if eff >= 0.7 else ("degraded" if eff >= 0.4 else "critical"),
+                    })
+
+            # Gather skill_performance assertions
+            assertion_query = (
+                "SELECT trigger_condition, action, confidence "
+                "FROM behavioral_assertions "
+                "WHERE profile_id = ? AND category = 'skill_performance'"
+            )
+            assertion_params = [profile_id]
+            if skill_name:
+                safe_assert_name = skill_name.replace('\\', '\\\\').replace('%', r'\%').replace('_', r'\_')
+                assertion_query += " AND trigger_condition LIKE ? ESCAPE '\\'"
+                assertion_params.append(f"%{safe_assert_name}%")
+            assertion_rows = conn.execute(assertion_query, tuple(assertion_params)).fetchall()
+
+            skills = skill_entries
+
+            # Add assertion insights
+            assertion_insights = [
+                {"trigger": dict(a)["trigger_condition"], "action": dict(a)["action"], "confidence": dict(a)["confidence"]}
+                for a in assertion_rows
+            ]
+
+            conn.close()
+
+            return {
+                "skills": skills,
+                "skill_count": len(skills),
+                "assertion_insights": assertion_insights,
+                "profile_id": profile_id,
+            }
+        except Exception as exc:
+            logger.debug("skill_health failed: %s", exc)
+            return {"skills": [], "skill_count": 0, "error": str(exc)}
+
+    @server.tool()
+    async def skill_lineage(
+        skill_name: str = "",
+    ) -> dict:
+        """Get evolution lineage for a skill.
+
+        Queries the skill_evolution_log table and builds a version tree
+        showing how skills evolved from their parents.
+
+        Args:
+            skill_name: Specific skill name (empty = all skills)
+        """
+        try:
+            db_path = str(MEMORY_DB)
+            conn = sqlite3.connect(db_path, timeout=10)
+            conn.row_factory = sqlite3.Row
+
+            if skill_name:
+                rows = conn.execute(
+                    "SELECT id, skill_name, parent_skill_id, evolution_type, "
+                    "trigger_type, generation, status, mutation_summary, "
+                    "blind_verified, created_at, completed_at "
+                    "FROM skill_evolution_log "
+                    "WHERE skill_name = ? OR parent_skill_id = ? "
+                    "ORDER BY created_at ASC",
+                    (skill_name, skill_name),
+                ).fetchall()
+            else:
+                rows = conn.execute(
+                    "SELECT id, skill_name, parent_skill_id, evolution_type, "
+                    "trigger_type, generation, status, mutation_summary, "
+                    "blind_verified, created_at, completed_at "
+                    "FROM skill_evolution_log "
+                    "ORDER BY created_at DESC LIMIT 100",
+                ).fetchall()
+
+            conn.close()
+
+            lineage = [
+                {
+                    "id": dict(r)["id"],
+                    "skill_name": dict(r)["skill_name"],
+                    "parent_skill_id": dict(r).get("parent_skill_id", ""),
+                    "evolution_type": dict(r)["evolution_type"],
+                    "trigger": dict(r)["trigger_type"],
+                    "generation": dict(r).get("generation", 0),
+                    "status": dict(r)["status"],
+                    "mutation_summary": dict(r).get("mutation_summary", ""),
+                    "blind_verified": bool(dict(r).get("blind_verified", 0)),
+                    "created_at": dict(r).get("created_at", ""),
+                    "completed_at": dict(r).get("completed_at", ""),
+                }
+                for r in rows
+            ]
+
+            # Build tree structure: group by root skill
+            tree: dict = {}
+            for entry in lineage:
+                root = entry.get("parent_skill_id") or entry["skill_name"]
+                if root not in tree:
+                    tree[root] = {"root": root, "evolutions": []}
+                tree[root]["evolutions"].append({
+                    "id": entry["id"],
+                    "skill_name": entry["skill_name"],
+                    "evolution_type": entry["evolution_type"],
+                    "status": entry["status"],
+                    "generation": entry["generation"],
+                    "created_at": entry["created_at"],
+                })
+
+            return {
+                "lineage": lineage,
+                "lineage_count": len(lineage),
+                "tree": tree,
+            }
+        except Exception as exc:
+            logger.debug("skill_lineage failed: %s", exc)
+            return {"lineage": [], "lineage_count": 0, "tree": {}, "error": str(exc)}

package/src/superlocalmemory/retrieval/engine.py

@@ -193,9 +193,6 @@ class RetrievalEngine:
         facts = self._load_facts(top, profile_id)
 
         # V3.3.21: Session diversity for aggregation queries.
-        # Cat 1 (single-hop/aggregation) needs facts from MULTIPLE sessions.
-        # Without diversity enforcement, top-20 may all come from 1-2 sessions,
-        # missing scattered mentions across 19+ sessions.
         if strat.query_type == "aggregation" and facts:
             top = self._enforce_session_diversity(top, facts, min_sessions=3, top_k=20)
 
@@ -212,8 +209,18 @@ class RetrievalEngine:
             ce_alpha = 0.5 if strat.query_type in ("multi_hop", "temporal") else 0.75
             top = self._apply_reranker(query, top, facts, alpha=ce_alpha)
 
+        # V3.4.11: Channel diversity — guarantee entity_graph results appear in
+        # the final output. Applied AFTER reranker so results can't be pushed out.
+        final_top = top[:effective_limit]
+        final_top = self._enforce_channel_diversity(
+            final_top, fused, ch_results, effective_limit,
+        )
+        # Reload facts for any newly injected results
+        if len(final_top) > len(top[:effective_limit]):
+            facts = self._load_facts(final_top, profile_id)
+
         # 6. Build response
-        results = self._build_results(top[:effective_limit], facts, strat)
+        results = self._build_results(final_top, facts, strat)
         ms = (time.monotonic() - t0) * 1000.0
         return RecallResponse(
             query=query, mode=mode, results=results,
@@ -334,6 +341,54 @@ class RetrievalEngine:
         remaining = [fr for fr in rest if fr.fact_id not in promoted_ids]
         return top + promoted + remaining
 
+    # -- Channel diversity enforcement ----------------------------------------
+
+    @staticmethod
+    def _enforce_channel_diversity(
+        top: list,
+        fused: list,
+        ch_results: dict[str, list[tuple[str, float]]],
+        effective_limit: int,
+        min_per_channel: int = 2,
+    ) -> list:
+        """Ensure structure channels (entity_graph) get representation.
+
+        V3.4.11: entity_graph finds valid results but RRF scores them low
+        because they don't overlap with semantic/bm25 results. This interleaves
+        top entity_graph facts into positions 3-4 of the final output instead
+        of appending at the end where they'd never be seen.
+        """
+        structure_channels = ["entity_graph"]
+        top_ids = {fr.fact_id for fr in top}
+
+        promoted = []
+        for ch_name in structure_channels:
+            ch_items = ch_results.get(ch_name, [])
+            if not ch_items:
+                continue
+
+            present = sum(1 for fid, _ in ch_items if fid in top_ids)
+            if present >= min_per_channel:
+                continue
+
+            needed = min_per_channel - present
+            ch_fids = {fid for fid, _ in ch_items}
+            for fr in fused:
+                if fr.fact_id in ch_fids and fr.fact_id not in top_ids:
+                    promoted.append(fr)
+                    top_ids.add(fr.fact_id)
+                    needed -= 1
+                    if needed <= 0:
+                        break
+
+        if not promoted:
+            return top
+
+        # Append as safety net — with proper RRF weights (strategy.py),
+        # entity_graph facts should already rank naturally in the top-k.
+        # This only fires when they're still missing despite weight boost.
+        return list(top) + promoted
+
     # -- Channel execution --------------------------------------------------
 
     def _embed_query(self, query: str) -> list[float] | None:
@@ -369,6 +424,11 @@ class RetrievalEngine:
         if needs_embedding:
             try:
                 q_emb = self._embed_query(query)
+                if q_emb is None:
+                    logger.warning(
+                        "Query embedding returned None — semantic, hopfield, "
+                        "spreading_activation channels will be skipped this recall"
+                    )
             except Exception as exc:
                 logger.warning("Query embedding failed: %s", exc)
 

package/src/superlocalmemory/retrieval/forgetting_filter.py

@@ -41,9 +41,21 @@ _ZONE_WEIGHTS: dict[str, float] = {
     "forgotten": 0.0,
 }
 
-# Zones where facts are excluded from results
+# V3.4.11: Deep recall weights — includes cold/archive with reduced scores
+_DEEP_ZONE_WEIGHTS: dict[str, float] = {
+    "active": 1.0,
+    "warm": 0.7,
+    "cold": 0.3,
+    "archive": 0.15,
+    "forgotten": 0.05,
+}
+
+# Zones where facts are excluded from results (default recall)
 _EXCLUDED_ZONES: frozenset[str] = frozenset({"archive", "forgotten"})
 
+# Deep recall excludes nothing — every fact is searchable
+_DEEP_EXCLUDED_ZONES: frozenset[str] = frozenset()
+
 
 class ForgettingFilter:
     """Post-retrieval filter that applies Ebbinghaus retention weighting.
@@ -51,11 +63,12 @@ class ForgettingFilter:
     Removes archived/forgotten facts and adjusts scores for other zones.
     """
 
-    __slots__ = ("_db", "_config")
+    __slots__ = ("_db", "_config", "_deep_recall")
 
-    def __init__(self, db: DatabaseManager, config: ForgettingConfig) -> None:
+    def __init__(self, db: DatabaseManager, config: ForgettingConfig, deep_recall: bool = False) -> None:
         self._db = db
         self._config = config
+        self._deep_recall = deep_recall
 
     def filter(
         self,
@@ -112,12 +125,14 @@ class ForgettingFilter:
 
                 zone = ret_data.get("lifecycle_zone", "active")
 
-                if zone in _EXCLUDED_ZONES:
-                    # Archive/forgotten: remove from results
+                # V3.4.11: Deep recall mode includes all tiers
+                excluded = _DEEP_EXCLUDED_ZONES if self._deep_recall else _EXCLUDED_ZONES
+                weights = _DEEP_ZONE_WEIGHTS if self._deep_recall else _ZONE_WEIGHTS
+
+                if zone in excluded:
                     continue
 
-                # Apply weight
-                weight = _ZONE_WEIGHTS.get(zone, 1.0)
+                weight = weights.get(zone, 1.0)
                 new_results.append((fact_id, score * weight))
 
             filtered[channel_name] = new_results

package/src/superlocalmemory/retrieval/strategy.py

@@ -17,11 +17,11 @@ from dataclasses import dataclass, field
 
 STRATEGY_PRESETS: dict[str, dict[str, float]] = {
     "temporal": {"semantic": 0.8, "bm25": 1.5, "entity_graph": 0.8, "temporal": 2.0, "spreading_activation": 0.5, "hopfield": 0.5},
-    "multi_hop": {"semantic": 1.0, "bm25": 0.8, "entity_graph": 2.0, "temporal": 0.5, "spreading_activation": 2.0, "hopfield": 0.7},
+    "multi_hop": {"semantic": 1.0, "bm25": 0.8, "entity_graph": 2.5, "temporal": 0.5, "spreading_activation": 2.0, "hopfield": 0.7},
     "aggregation": {"semantic": 1.2, "bm25": 1.5, "entity_graph": 1.0, "temporal": 0.5, "spreading_activation": 0.8, "hopfield": 0.6},
     "opinion": {"semantic": 1.8, "bm25": 0.6, "entity_graph": 0.8, "temporal": 0.3, "spreading_activation": 0.5, "hopfield": 0.5},
     "factual": {"semantic": 1.2, "bm25": 1.4, "entity_graph": 1.0, "temporal": 0.6, "spreading_activation": 0.8, "hopfield": 0.8},
-    "entity": {"semantic": 1.0, "bm25": 1.5, "entity_graph": 1.2, "temporal": 0.5, "spreading_activation": 1.0, "hopfield": 0.9},
+    "entity": {"semantic": 1.0, "bm25": 1.2, "entity_graph": 3.0, "temporal": 0.5, "spreading_activation": 1.5, "hopfield": 0.9},
     "general": {},
     "vague": {"semantic": 0.8, "bm25": 0.5, "entity_graph": 0.6, "temporal": 0.3, "spreading_activation": 1.5, "hopfield": 1.1},
 }