superlocalmemory 3.4.10 → 3.4.12

package/src/superlocalmemory/retrieval/strategy.py

@@ -17,11 +17,11 @@ from dataclasses import dataclass, field
 
 STRATEGY_PRESETS: dict[str, dict[str, float]] = {
     "temporal": {"semantic": 0.8, "bm25": 1.5, "entity_graph": 0.8, "temporal": 2.0, "spreading_activation": 0.5, "hopfield": 0.5},
-    "multi_hop": {"semantic": 1.0, "bm25": 0.8, "entity_graph": 2.0, "temporal": 0.5, "spreading_activation": 2.0, "hopfield": 0.7},
+    "multi_hop": {"semantic": 1.0, "bm25": 0.8, "entity_graph": 2.5, "temporal": 0.5, "spreading_activation": 2.0, "hopfield": 0.7},
     "aggregation": {"semantic": 1.2, "bm25": 1.5, "entity_graph": 1.0, "temporal": 0.5, "spreading_activation": 0.8, "hopfield": 0.6},
     "opinion": {"semantic": 1.8, "bm25": 0.6, "entity_graph": 0.8, "temporal": 0.3, "spreading_activation": 0.5, "hopfield": 0.5},
     "factual": {"semantic": 1.2, "bm25": 1.4, "entity_graph": 1.0, "temporal": 0.6, "spreading_activation": 0.8, "hopfield": 0.8},
-    "entity": {"semantic": 1.0, "bm25": 1.5, "entity_graph": 1.2, "temporal": 0.5, "spreading_activation": 1.0, "hopfield": 0.9},
+    "entity": {"semantic": 1.0, "bm25": 1.2, "entity_graph": 3.0, "temporal": 0.5, "spreading_activation": 1.5, "hopfield": 0.9},
     "general": {},
     "vague": {"semantic": 0.8, "bm25": 0.5, "entity_graph": 0.6, "temporal": 0.3, "spreading_activation": 1.5, "hopfield": 1.1},
 }

package/src/superlocalmemory/server/routes/behavioral.py

@@ -193,6 +193,7 @@ async def get_tool_events(tool_name: str = "", limit: int = 100):
     try:
         import sqlite3 as _sqlite3
         profile = get_active_profile()
+        limit = min(int(limit), 1000)
         conn = _sqlite3.connect(str(MEMORY_DIR / "memory.db"))
         conn.row_factory = _sqlite3.Row
 
@@ -208,11 +209,12 @@ async def get_tool_events(tool_name: str = "", limit: int = 100):
         query += " ORDER BY created_at DESC LIMIT ?"
         params.append(limit)
 
-        rows = conn.execute(query, tuple(params)).fetchall()
-        conn.close()
-
-        events = [dict(r) for r in rows]
-        return {"events": events, "count": len(events)}
+        try:
+            rows = conn.execute(query, tuple(params)).fetchall()
+            events = [dict(r) for r in rows]
+            return {"events": events, "count": len(events)}
+        finally:
+            conn.close()
     except Exception as e:
         logger.debug("get_tool_events error: %s", e)
         return {"events": [], "count": 0, "error": str(e)}
@@ -276,16 +278,18 @@ async def log_tool_event_api(data: dict):
         output_summary = str(output_summary)[:500] if output_summary else ""
 
         conn = _sqlite3.connect(str(MEMORY_DIR / "memory.db"))
-        conn.execute(
-            "INSERT INTO tool_events "
-            "(session_id, profile_id, project_path, tool_name, event_type, "
-            " input_summary, output_summary, duration_ms, metadata, created_at) "
-            "VALUES (?, ?, ?, ?, ?, ?, ?, 0, '{}', ?)",
-            (session_id, profile, project_path, tool_name, event_type,
-             input_summary, output_summary, now),
-        )
-        conn.commit()
-        conn.close()
+        try:
+            conn.execute(
+                "INSERT INTO tool_events "
+                "(session_id, profile_id, project_path, tool_name, event_type, "
+                " input_summary, output_summary, duration_ms, metadata, created_at) "
+                "VALUES (?, ?, ?, ?, ?, ?, ?, 0, '{}', ?)",
+                (session_id, profile, project_path, tool_name, event_type,
+                 input_summary, output_summary, now),
+            )
+            conn.commit()
+        finally:
+            conn.close()
         return {"ok": True}
     except Exception as e:
         return {"ok": False, "error": str(e)}

package/src/superlocalmemory/server/routes/evolution.py

@@ -0,0 +1,213 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Evolution API routes — dashboard endpoints for skill evolution engine.
+
+Routes: /api/evolution/status, /api/evolution/enable, /api/evolution/run
+"""
+
+import logging
+from pathlib import Path
+
+from fastapi import APIRouter
+
+from .helpers import get_active_profile, MEMORY_DIR
+
+logger = logging.getLogger("superlocalmemory.routes.evolution")
+router = APIRouter()
+
+
+@router.get("/api/evolution/status")
+async def evolution_status():
+    """Get evolution engine status, backend, and recent history."""
+    try:
+        import json as _json
+        from superlocalmemory.evolution.skill_evolver import detect_backend
+        from superlocalmemory.evolution.evolution_store import EvolutionStore
+
+        # Read config directly from config.json (SLMConfig.load doesn't serialize evolution)
+        config_path = MEMORY_DIR / "config.json"
+        evo_cfg = {}
+        if config_path.exists():
+            with open(config_path) as f:
+                cfg = _json.load(f)
+            evo_cfg = cfg.get("evolution", {})
+
+        enabled = evo_cfg.get("enabled", False)
+        backend = detect_backend() if enabled else "none"
+        db_path = str(MEMORY_DIR / "memory.db")
+
+        store = EvolutionStore(db_path)
+        stats = store.get_stats()
+        recent = store.get_recent(limit=10)
+
+        return {
+            "enabled": enabled,
+            "backend": backend,
+            "config": {
+                "backend_setting": evo_cfg.get("backend", "auto"),
+                "max_per_cycle": evo_cfg.get("max_evolutions_per_cycle", 3),
+            },
+            "stats": {
+                "total": stats.get("total", 0),
+                "promoted": stats.get("by_status", {}).get("promoted", 0),
+                "rejected": stats.get("by_status", {}).get("rejected", 0),
+                "failed": stats.get("by_status", {}).get("failed", 0),
+                "cycle_budget_remaining": stats.get("cycle_budget_remaining", 3),
+            },
+            "recent": [
+                {
+                    "id": r.id,
+                    "skill_name": r.skill_name,
+                    "evolution_type": r.evolution_type.value,
+                    "trigger": r.trigger.value,
+                    "status": r.status.value,
+                    "mutation_summary": r.mutation_summary,
+                    "blind_verified": r.blind_verified,
+                    "created_at": r.created_at,
+                }
+                for r in recent
+            ],
+        }
+    except Exception as e:
+        logger.debug("evolution_status error: %s", e)
+        return {"enabled": False, "backend": "none", "error": str(e)}
+
+
+@router.post("/api/evolution/enable")
+async def evolution_enable():
+    """Enable skill evolution engine. Writes directly to config.json."""
+    try:
+        import json as _json
+
+        config_path = MEMORY_DIR / "config.json"
+        cfg = {}
+        if config_path.exists():
+            with open(config_path) as f:
+                cfg = _json.load(f)
+
+        if "evolution" not in cfg:
+            cfg["evolution"] = {}
+        cfg["evolution"]["enabled"] = True
+        cfg["evolution"]["backend"] = "auto"
+
+        with open(config_path, "w") as f:
+            _json.dump(cfg, f, indent=2)
+
+        return {"ok": True, "message": "Evolution enabled. Will use auto-detected backend."}
+    except Exception as e:
+        logger.error("evolution_enable error: %s", e)
+        return {"ok": False, "error": str(e)}
+
+
+@router.post("/api/evolution/run")
+async def evolution_run():
+    """Manually trigger an evolution cycle."""
+    try:
+        import json as _json
+        from superlocalmemory.evolution.skill_evolver import SkillEvolver
+
+        config_path = MEMORY_DIR / "config.json"
+        evo_cfg = {}
+        if config_path.exists():
+            with open(config_path) as f:
+                evo_cfg = _json.load(f).get("evolution", {})
+
+        if not evo_cfg.get("enabled", False):
+            return {"ok": False, "error": "Evolution is disabled. Enable first."}
+
+        profile = get_active_profile()
+        db_path = str(MEMORY_DIR / "memory.db")
+
+        # Build a minimal config object for the evolver
+        class _EvoCfg:
+            enabled = True
+            backend = evo_cfg.get("backend", "auto")
+            max_evolutions_per_cycle = evo_cfg.get("max_evolutions_per_cycle", 3)
+        class _Cfg:
+            evolution = _EvoCfg()
+
+        evolver = SkillEvolver(db_path, _Cfg())
+        result = evolver.run_consolidation_cycle(profile)
+
+        return {"ok": True, **result}
+    except Exception as e:
+        logger.error("evolution_run error: %s", e)
+        return {"ok": False, "error": str(e)}
+
+
+@router.get("/api/evolution/lineage")
+async def evolution_lineage(skill_name: str = ""):
+    """Get evolution lineage for a skill or all skills.
+
+    Returns lineage records and a tree structure grouped by root skill.
+    """
+    try:
+        import sqlite3 as _sqlite3
+
+        db_path = str(MEMORY_DIR / "memory.db")
+        conn = _sqlite3.connect(db_path, timeout=10)
+        conn.row_factory = _sqlite3.Row
+
+        if skill_name:
+            rows = conn.execute(
+                "SELECT id, skill_name, parent_skill_id, evolution_type, "
+                "trigger_type, generation, status, mutation_summary, "
+                "blind_verified, created_at, completed_at "
+                "FROM skill_evolution_log "
+                "WHERE skill_name = ? OR parent_skill_id = ? "
+                "ORDER BY created_at ASC",
+                (skill_name, skill_name),
+            ).fetchall()
+        else:
+            rows = conn.execute(
+                "SELECT id, skill_name, parent_skill_id, evolution_type, "
+                "trigger_type, generation, status, mutation_summary, "
+                "blind_verified, created_at, completed_at "
+                "FROM skill_evolution_log "
+                "ORDER BY created_at DESC LIMIT 100",
+            ).fetchall()
+
+        conn.close()
+
+        lineage = [
+            {
+                "id": dict(r)["id"],
+                "skill_name": dict(r)["skill_name"],
+                "parent_skill_id": dict(r).get("parent_skill_id", ""),
+                "evolution_type": dict(r)["evolution_type"],
+                "trigger": dict(r)["trigger_type"],
+                "generation": dict(r).get("generation", 0),
+                "status": dict(r)["status"],
+                "mutation_summary": dict(r).get("mutation_summary", ""),
+                "blind_verified": bool(dict(r).get("blind_verified", 0)),
+                "created_at": dict(r).get("created_at", ""),
+                "completed_at": dict(r).get("completed_at", ""),
+            }
+            for r in rows
+        ]
+
+        # Build tree structure: group by root skill
+        tree: dict = {}
+        for entry in lineage:
+            root = entry.get("parent_skill_id") or entry["skill_name"]
+            if root not in tree:
+                tree[root] = {"root": root, "evolutions": []}
+            tree[root]["evolutions"].append({
+                "id": entry["id"],
+                "skill_name": entry["skill_name"],
+                "evolution_type": entry["evolution_type"],
+                "status": entry["status"],
+                "generation": entry["generation"],
+                "created_at": entry["created_at"],
+            })
+
+        return {
+            "lineage": lineage,
+            "lineage_count": len(lineage),
+            "tree": tree,
+        }
+    except Exception as e:
+        logger.debug("evolution_lineage error: %s", e)
+        return {"lineage": [], "lineage_count": 0, "tree": {}, "error": str(e)}

package/src/superlocalmemory/server/routes/tiers.py

@@ -0,0 +1,195 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+"""SuperLocalMemory V3.4.11 "Scale-Ready" - Tier Management Routes
+
+Routes: /api/tiers/stats, /api/tiers/evaluate, /api/tiers/pin, /api/tiers/unpin
+
+Uses lightweight sqlite3 directly (not MemoryEngine) for fast dashboard queries.
+All connections use WAL mode + busy_timeout for concurrency safety.
+"""
+
+import logging
+import re
+import sqlite3
+from contextlib import contextmanager
+from datetime import datetime, UTC
+
+from fastapi import APIRouter, HTTPException, Request
+from pydantic import BaseModel, Field
+
+from .helpers import DB_PATH
+
+logger = logging.getLogger("superlocalmemory.routes.tiers")
+router = APIRouter()
+
+_PROFILE_PATTERN = re.compile(r'^[a-zA-Z0-9_-]+$')
+_MAX_REASON_LENGTH = 500
+
+
+class PinRequest(BaseModel):
+    fact_id: str = Field(..., min_length=1)
+    reason: str = Field(default="", max_length=_MAX_REASON_LENGTH)
+
+
+@contextmanager
+def _db():
+    """Context-managed DB connection with WAL + busy_timeout."""
+    conn = sqlite3.connect(str(DB_PATH))
+    conn.execute("PRAGMA journal_mode=WAL")
+    conn.execute("PRAGMA busy_timeout=5000")
+    conn.row_factory = sqlite3.Row
+    try:
+        yield conn
+    finally:
+        conn.close()
+
+
+def _validate_profile(profile_id: str) -> str:
+    """Validate profile_id against allowed pattern."""
+    if not profile_id or not _PROFILE_PATTERN.match(profile_id):
+        raise HTTPException(status_code=400, detail="Invalid profile_id")
+    return profile_id
+
+
+@router.get("/api/tiers/stats")
+async def tier_stats(profile_id: str = "default"):
+    """Get tier distribution stats."""
+    profile_id = _validate_profile(profile_id)
+
+    with _db() as conn:
+        try:
+            c = conn.cursor()
+            c.execute(
+                "SELECT lifecycle, COUNT(*) as cnt FROM atomic_facts "
+                "WHERE profile_id = ? GROUP BY lifecycle", (profile_id,),
+            )
+            dist = {row["lifecycle"]: row["cnt"] for row in c.fetchall()}
+
+            pinned = 0
+            try:
+                c.execute(
+                    "SELECT COUNT(*) as c FROM pinned_facts "
+                    "WHERE profile_id = ?", (profile_id,),
+                )
+                pinned = c.fetchone()["c"]
+            except sqlite3.OperationalError:
+                pass  # pinned_facts table may not exist yet
+
+            total = sum(dist.values())
+            return {
+                "active": dist.get("active", 0),
+                "warm": dist.get("warm", 0),
+                "cold": dist.get("cold", 0),
+                "archived": dist.get("archived", 0),
+                "total": total,
+                "pinned": pinned,
+                "active_pct": round(
+                    dist.get("active", 0) / max(total, 1) * 100, 1,
+                ),
+            }
+        except Exception as exc:
+            logger.error("tier_stats failed: %s", exc, exc_info=True)
+            raise HTTPException(
+                status_code=500, detail="Internal storage error",
+            ) from None
+
+
+@router.post("/api/tiers/evaluate")
+async def evaluate_tiers_route(request: Request, profile_id: str = "default"):
+    """Manually trigger tier evaluation.
+
+    Uses the shared engine (via lazy import) instead of re-initializing
+    DatabaseManager on every request.
+    """
+    profile_id = _validate_profile(profile_id)
+
+    try:
+        from superlocalmemory.core.tier_manager import evaluate_tiers
+        from .helpers import get_engine_lazy
+
+        engine = get_engine_lazy(request.app.state)
+        if engine is None or not hasattr(engine, '_db') or engine._db is None:
+            raise HTTPException(
+                status_code=503, detail="Engine not initialized",
+            )
+        stats = evaluate_tiers(engine._db, profile_id)
+        return {"success": True, "stats": stats}
+    except HTTPException:
+        raise
+    except Exception as exc:
+        logger.error("evaluate_tiers failed: %s", exc, exc_info=True)
+        raise HTTPException(
+            status_code=500, detail="Tier evaluation failed",
+        ) from None
+
+
+@router.post("/api/tiers/pin")
+async def pin_fact_route(request: PinRequest, profile_id: str = "default"):
+    """Pin a fact to stay in active tier forever.
+
+    Validates fact exists in the specified profile before pinning.
+    """
+    profile_id = _validate_profile(profile_id)
+
+    with _db() as conn:
+        try:
+            # Verify fact exists in this profile
+            c = conn.cursor()
+            c.execute(
+                "SELECT fact_id FROM atomic_facts "
+                "WHERE fact_id = ? AND profile_id = ?",
+                (request.fact_id, profile_id),
+            )
+            if c.fetchone() is None:
+                raise HTTPException(
+                    status_code=404,
+                    detail=f"Fact {request.fact_id[:8]}... not found",
+                )
+
+            now = datetime.now(UTC).isoformat()
+            conn.execute(
+                "INSERT OR REPLACE INTO pinned_facts "
+                "(fact_id, profile_id, pinned_at, reason) "
+                "VALUES (?, ?, ?, ?)",
+                (request.fact_id, profile_id, now, request.reason),
+            )
+            conn.execute(
+                "UPDATE atomic_facts SET lifecycle = 'active' "
+                "WHERE fact_id = ? AND profile_id = ?",
+                (request.fact_id, profile_id),
+            )
+            conn.commit()
+            return {"success": True, "message": f"Fact {request.fact_id[:8]}... pinned"}
+        except HTTPException:
+            raise
+        except Exception as exc:
+            logger.error("pin_fact failed: %s", exc, exc_info=True)
+            raise HTTPException(
+                status_code=500, detail="Failed to pin fact",
+            ) from None
+
+
+@router.post("/api/tiers/unpin")
+async def unpin_fact_route(request: PinRequest, profile_id: str = "default"):
+    """Unpin a fact, allowing normal tier demotion.
+
+    Lifecycle stays 'active' until the next tier evaluation cycle demotes it
+    based on access patterns. This is intentional — immediate demotion would
+    surprise the user.
+    """
+    profile_id = _validate_profile(profile_id)
+
+    with _db() as conn:
+        try:
+            conn.execute(
+                "DELETE FROM pinned_facts WHERE fact_id = ? AND profile_id = ?",
+                (request.fact_id, profile_id),
+            )
+            conn.commit()
+            return {"success": True, "unpinned": True}
+        except Exception as exc:
+            logger.error("unpin_fact failed: %s", exc, exc_info=True)
+            raise HTTPException(
+                status_code=500, detail="Failed to unpin fact",
+            ) from None

package/src/superlocalmemory/server/unified_daemon.py

@@ -268,6 +268,20 @@ async def lifespan(application: FastAPI):
             if reranker and hasattr(reranker, 'warmup_sync'):
                 reranker.warmup_sync(timeout=120)
 
+        # V3.4.11: Pre-warm embedding worker (load ONNX model on startup)
+        # Without this, first recall takes 60-90s for model load.
+        # Same pattern as reranker warmup above.
+        import threading
+        def _warmup_embedder():
+            try:
+                embedder = getattr(retrieval_eng, '_embedder', None) if retrieval_eng else None
+                if embedder and hasattr(embedder, 'embed'):
+                    embedder.embed("warmup")
+                    logger.info("Embedding worker pre-warmed (ONNX model loaded)")
+            except Exception as exc:
+                logger.warning("Embedding warmup failed: %s", exc)
+        threading.Thread(target=_warmup_embedder, daemon=True, name="embed-warmup").start()
+
     except Exception as exc:
         logger.warning("Engine init failed: %s", exc)
         application.state.engine = None
@@ -318,6 +332,8 @@ async def lifespan(application: FastAPI):
     if enable_legacy:
         asyncio.create_task(_start_legacy_redirect(_DEFAULT_PORT, _LEGACY_PORT))
 
+    global _start_time
+    _start_time = time.monotonic()
     _last_activity = time.monotonic()
     logger.info("Unified daemon ready on port %d (24/7 mode)" if idle_timeout <= 0
                 else "Unified daemon ready on port %d (idle timeout: %ds)",
@@ -422,7 +438,7 @@ def _register_dashboard_routes(application: FastAPI) -> None:
                 return JSONResponse(
                     status_code=429,
                     content={"error": "Too many requests."},
-                    headers={"Retry-After": str(limiter.window_seconds)},
+                    headers={"Retry-After": str(getattr(limiter, 'window', 60))},
                 )
             response = await call_next(request)
             response.headers["X-RateLimit-Remaining"] = str(remaining)
@@ -472,6 +488,19 @@ def _register_dashboard_routes(application: FastAPI) -> None:
     application.include_router(profiles_router)
     application.include_router(backup_router)
     application.include_router(data_io_router)
+
+    # Optional routers — ImportError-safe so missing modules don't crash startup
+    try:
+        from superlocalmemory.server.routes.tiers import router as tiers_router
+        application.include_router(tiers_router)
+    except ImportError:
+        logger.debug("tiers_router not available")
+
+    try:
+        from superlocalmemory.server.routes.evolution import router as evolution_router
+        application.include_router(evolution_router)
+    except ImportError:
+        logger.debug("evolution_router not available")
     application.include_router(events_router)
     application.include_router(agents_router)
     application.include_router(ws_router)
@@ -542,19 +571,25 @@ def _register_daemon_routes(application: FastAPI) -> None:
         }
 
     @application.get("/recall")
-    async def recall(q: str = "", limit: int = 20):
+    async def recall(q: str = "", query: str = "", limit: int = 20):
         _update_activity()
+        search_query = q or query  # Accept both ?q= and ?query= for compatibility
         engine = application.state.engine
         if engine is None:
             raise HTTPException(503, detail="Engine not initialized")
+        if not search_query:
+            return {"results": [], "count": 0, "query_type": "none", "retrieval_time_ms": 0}
         try:
-            response = engine.recall(q, limit=limit)
+            response = engine.recall(search_query, limit=limit)
             results = [
                 {
                     "content": r.fact.content,
                     "score": round(r.score, 4),
                     "fact_type": getattr(r.fact.fact_type, 'value', str(r.fact.fact_type)),
                     "fact_id": r.fact.fact_id,
+                    "channel_scores": {
+                        k: round(v, 4) for k, v in r.channel_scores.items()
+                    } if r.channel_scores else {},
                 }
                 for r in response.results
             ]
@@ -590,7 +625,6 @@ def _register_daemon_routes(application: FastAPI) -> None:
     async def status():
         _update_activity()
         engine = application.state.engine
-        uptime = time.monotonic() - _last_activity
         fact_count = engine.fact_count if engine else 0
         mode = engine._config.mode.value if engine and hasattr(engine, '_config') else "unknown"
         return {
@@ -656,7 +690,7 @@ def _start_memory_watchdog() -> None:
     """
     import threading
 
-    MAX_WORKER_MB = 2048  # 2GB per worker — kill if exceeded
+    MAX_WORKER_MB = 4096  # 4GB per worker — ONNX full model is 1.6GB + overhead
 
     def watchdog_loop():
         while True: