superlocalmemory 3.4.9 → 3.4.11

package/src/superlocalmemory/server/routes/evolution.py

@@ -0,0 +1,213 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Evolution API routes — dashboard endpoints for skill evolution engine.
+
+Routes: /api/evolution/status, /api/evolution/enable, /api/evolution/run
+"""
+
+import logging
+from pathlib import Path
+
+from fastapi import APIRouter
+
+from .helpers import get_active_profile, MEMORY_DIR
+
+logger = logging.getLogger("superlocalmemory.routes.evolution")
+router = APIRouter()
+
+
+@router.get("/api/evolution/status")
+async def evolution_status():
+    """Get evolution engine status, backend, and recent history."""
+    try:
+        import json as _json
+        from superlocalmemory.evolution.skill_evolver import detect_backend
+        from superlocalmemory.evolution.evolution_store import EvolutionStore
+
+        # Read config directly from config.json (SLMConfig.load doesn't serialize evolution)
+        config_path = MEMORY_DIR / "config.json"
+        evo_cfg = {}
+        if config_path.exists():
+            with open(config_path) as f:
+                cfg = _json.load(f)
+            evo_cfg = cfg.get("evolution", {})
+
+        enabled = evo_cfg.get("enabled", False)
+        backend = detect_backend() if enabled else "none"
+        db_path = str(MEMORY_DIR / "memory.db")
+
+        store = EvolutionStore(db_path)
+        stats = store.get_stats()
+        recent = store.get_recent(limit=10)
+
+        return {
+            "enabled": enabled,
+            "backend": backend,
+            "config": {
+                "backend_setting": evo_cfg.get("backend", "auto"),
+                "max_per_cycle": evo_cfg.get("max_evolutions_per_cycle", 3),
+            },
+            "stats": {
+                "total": stats.get("total", 0),
+                "promoted": stats.get("by_status", {}).get("promoted", 0),
+                "rejected": stats.get("by_status", {}).get("rejected", 0),
+                "failed": stats.get("by_status", {}).get("failed", 0),
+                "cycle_budget_remaining": stats.get("cycle_budget_remaining", 3),
+            },
+            "recent": [
+                {
+                    "id": r.id,
+                    "skill_name": r.skill_name,
+                    "evolution_type": r.evolution_type.value,
+                    "trigger": r.trigger.value,
+                    "status": r.status.value,
+                    "mutation_summary": r.mutation_summary,
+                    "blind_verified": r.blind_verified,
+                    "created_at": r.created_at,
+                }
+                for r in recent
+            ],
+        }
+    except Exception as e:
+        logger.debug("evolution_status error: %s", e)
+        return {"enabled": False, "backend": "none", "error": str(e)}
+
+
+@router.post("/api/evolution/enable")
+async def evolution_enable():
+    """Enable skill evolution engine. Writes directly to config.json."""
+    try:
+        import json as _json
+
+        config_path = MEMORY_DIR / "config.json"
+        cfg = {}
+        if config_path.exists():
+            with open(config_path) as f:
+                cfg = _json.load(f)
+
+        if "evolution" not in cfg:
+            cfg["evolution"] = {}
+        cfg["evolution"]["enabled"] = True
+        cfg["evolution"]["backend"] = "auto"
+
+        with open(config_path, "w") as f:
+            _json.dump(cfg, f, indent=2)
+
+        return {"ok": True, "message": "Evolution enabled. Will use auto-detected backend."}
+    except Exception as e:
+        logger.error("evolution_enable error: %s", e)
+        return {"ok": False, "error": str(e)}
+
+
+@router.post("/api/evolution/run")
+async def evolution_run():
+    """Manually trigger an evolution cycle."""
+    try:
+        import json as _json
+        from superlocalmemory.evolution.skill_evolver import SkillEvolver
+
+        config_path = MEMORY_DIR / "config.json"
+        evo_cfg = {}
+        if config_path.exists():
+            with open(config_path) as f:
+                evo_cfg = _json.load(f).get("evolution", {})
+
+        if not evo_cfg.get("enabled", False):
+            return {"ok": False, "error": "Evolution is disabled. Enable first."}
+
+        profile = get_active_profile()
+        db_path = str(MEMORY_DIR / "memory.db")
+
+        # Build a minimal config object for the evolver
+        class _EvoCfg:
+            enabled = True
+            backend = evo_cfg.get("backend", "auto")
+            max_evolutions_per_cycle = evo_cfg.get("max_evolutions_per_cycle", 3)
+        class _Cfg:
+            evolution = _EvoCfg()
+
+        evolver = SkillEvolver(db_path, _Cfg())
+        result = evolver.run_consolidation_cycle(profile)
+
+        return {"ok": True, **result}
+    except Exception as e:
+        logger.error("evolution_run error: %s", e)
+        return {"ok": False, "error": str(e)}
+
+
+@router.get("/api/evolution/lineage")
+async def evolution_lineage(skill_name: str = ""):
+    """Get evolution lineage for a skill or all skills.
+
+    Returns lineage records and a tree structure grouped by root skill.
+    """
+    try:
+        import sqlite3 as _sqlite3
+
+        db_path = str(MEMORY_DIR / "memory.db")
+        conn = _sqlite3.connect(db_path, timeout=10)
+        conn.row_factory = _sqlite3.Row
+
+        if skill_name:
+            rows = conn.execute(
+                "SELECT id, skill_name, parent_skill_id, evolution_type, "
+                "trigger_type, generation, status, mutation_summary, "
+                "blind_verified, created_at, completed_at "
+                "FROM skill_evolution_log "
+                "WHERE skill_name = ? OR parent_skill_id = ? "
+                "ORDER BY created_at ASC",
+                (skill_name, skill_name),
+            ).fetchall()
+        else:
+            rows = conn.execute(
+                "SELECT id, skill_name, parent_skill_id, evolution_type, "
+                "trigger_type, generation, status, mutation_summary, "
+                "blind_verified, created_at, completed_at "
+                "FROM skill_evolution_log "
+                "ORDER BY created_at DESC LIMIT 100",
+            ).fetchall()
+
+        conn.close()
+
+        lineage = [
+            {
+                "id": dict(r)["id"],
+                "skill_name": dict(r)["skill_name"],
+                "parent_skill_id": dict(r).get("parent_skill_id", ""),
+                "evolution_type": dict(r)["evolution_type"],
+                "trigger": dict(r)["trigger_type"],
+                "generation": dict(r).get("generation", 0),
+                "status": dict(r)["status"],
+                "mutation_summary": dict(r).get("mutation_summary", ""),
+                "blind_verified": bool(dict(r).get("blind_verified", 0)),
+                "created_at": dict(r).get("created_at", ""),
+                "completed_at": dict(r).get("completed_at", ""),
+            }
+            for r in rows
+        ]
+
+        # Build tree structure: group by root skill
+        tree: dict = {}
+        for entry in lineage:
+            root = entry.get("parent_skill_id") or entry["skill_name"]
+            if root not in tree:
+                tree[root] = {"root": root, "evolutions": []}
+            tree[root]["evolutions"].append({
+                "id": entry["id"],
+                "skill_name": entry["skill_name"],
+                "evolution_type": entry["evolution_type"],
+                "status": entry["status"],
+                "generation": entry["generation"],
+                "created_at": entry["created_at"],
+            })
+
+        return {
+            "lineage": lineage,
+            "lineage_count": len(lineage),
+            "tree": tree,
+        }
+    except Exception as e:
+        logger.debug("evolution_lineage error: %s", e)
+        return {"lineage": [], "lineage_count": 0, "tree": {}, "error": str(e)}

package/src/superlocalmemory/server/routes/tiers.py

@@ -0,0 +1,195 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+"""SuperLocalMemory V3.4.11 "Scale-Ready" - Tier Management Routes
+
+Routes: /api/tiers/stats, /api/tiers/evaluate, /api/tiers/pin, /api/tiers/unpin
+
+Uses lightweight sqlite3 directly (not MemoryEngine) for fast dashboard queries.
+All connections use WAL mode + busy_timeout for concurrency safety.
+"""
+
+import logging
+import re
+import sqlite3
+from contextlib import contextmanager
+from datetime import datetime, UTC
+
+from fastapi import APIRouter, HTTPException, Request
+from pydantic import BaseModel, Field
+
+from .helpers import DB_PATH
+
+logger = logging.getLogger("superlocalmemory.routes.tiers")
+router = APIRouter()
+
+_PROFILE_PATTERN = re.compile(r'^[a-zA-Z0-9_-]+$')
+_MAX_REASON_LENGTH = 500
+
+
+class PinRequest(BaseModel):
+    fact_id: str = Field(..., min_length=1)
+    reason: str = Field(default="", max_length=_MAX_REASON_LENGTH)
+
+
+@contextmanager
+def _db():
+    """Context-managed DB connection with WAL + busy_timeout."""
+    conn = sqlite3.connect(str(DB_PATH))
+    conn.execute("PRAGMA journal_mode=WAL")
+    conn.execute("PRAGMA busy_timeout=5000")
+    conn.row_factory = sqlite3.Row
+    try:
+        yield conn
+    finally:
+        conn.close()
+
+
+def _validate_profile(profile_id: str) -> str:
+    """Validate profile_id against allowed pattern."""
+    if not profile_id or not _PROFILE_PATTERN.match(profile_id):
+        raise HTTPException(status_code=400, detail="Invalid profile_id")
+    return profile_id
+
+
+@router.get("/api/tiers/stats")
+async def tier_stats(profile_id: str = "default"):
+    """Get tier distribution stats."""
+    profile_id = _validate_profile(profile_id)
+
+    with _db() as conn:
+        try:
+            c = conn.cursor()
+            c.execute(
+                "SELECT lifecycle, COUNT(*) as cnt FROM atomic_facts "
+                "WHERE profile_id = ? GROUP BY lifecycle", (profile_id,),
+            )
+            dist = {row["lifecycle"]: row["cnt"] for row in c.fetchall()}
+
+            pinned = 0
+            try:
+                c.execute(
+                    "SELECT COUNT(*) as c FROM pinned_facts "
+                    "WHERE profile_id = ?", (profile_id,),
+                )
+                pinned = c.fetchone()["c"]
+            except sqlite3.OperationalError:
+                pass  # pinned_facts table may not exist yet
+
+            total = sum(dist.values())
+            return {
+                "active": dist.get("active", 0),
+                "warm": dist.get("warm", 0),
+                "cold": dist.get("cold", 0),
+                "archived": dist.get("archived", 0),
+                "total": total,
+                "pinned": pinned,
+                "active_pct": round(
+                    dist.get("active", 0) / max(total, 1) * 100, 1,
+                ),
+            }
+        except Exception as exc:
+            logger.error("tier_stats failed: %s", exc, exc_info=True)
+            raise HTTPException(
+                status_code=500, detail="Internal storage error",
+            ) from None
+
+
+@router.post("/api/tiers/evaluate")
+async def evaluate_tiers_route(request: Request, profile_id: str = "default"):
+    """Manually trigger tier evaluation.
+
+    Uses the shared engine (via lazy import) instead of re-initializing
+    DatabaseManager on every request.
+    """
+    profile_id = _validate_profile(profile_id)
+
+    try:
+        from superlocalmemory.core.tier_manager import evaluate_tiers
+        from .helpers import get_engine_lazy
+
+        engine = get_engine_lazy(request.app.state)
+        if engine is None or not hasattr(engine, '_db') or engine._db is None:
+            raise HTTPException(
+                status_code=503, detail="Engine not initialized",
+            )
+        stats = evaluate_tiers(engine._db, profile_id)
+        return {"success": True, "stats": stats}
+    except HTTPException:
+        raise
+    except Exception as exc:
+        logger.error("evaluate_tiers failed: %s", exc, exc_info=True)
+        raise HTTPException(
+            status_code=500, detail="Tier evaluation failed",
+        ) from None
+
+
+@router.post("/api/tiers/pin")
+async def pin_fact_route(request: PinRequest, profile_id: str = "default"):
+    """Pin a fact to stay in active tier forever.
+
+    Validates fact exists in the specified profile before pinning.
+    """
+    profile_id = _validate_profile(profile_id)
+
+    with _db() as conn:
+        try:
+            # Verify fact exists in this profile
+            c = conn.cursor()
+            c.execute(
+                "SELECT fact_id FROM atomic_facts "
+                "WHERE fact_id = ? AND profile_id = ?",
+                (request.fact_id, profile_id),
+            )
+            if c.fetchone() is None:
+                raise HTTPException(
+                    status_code=404,
+                    detail=f"Fact {request.fact_id[:8]}... not found",
+                )
+
+            now = datetime.now(UTC).isoformat()
+            conn.execute(
+                "INSERT OR REPLACE INTO pinned_facts "
+                "(fact_id, profile_id, pinned_at, reason) "
+                "VALUES (?, ?, ?, ?)",
+                (request.fact_id, profile_id, now, request.reason),
+            )
+            conn.execute(
+                "UPDATE atomic_facts SET lifecycle = 'active' "
+                "WHERE fact_id = ? AND profile_id = ?",
+                (request.fact_id, profile_id),
+            )
+            conn.commit()
+            return {"success": True, "message": f"Fact {request.fact_id[:8]}... pinned"}
+        except HTTPException:
+            raise
+        except Exception as exc:
+            logger.error("pin_fact failed: %s", exc, exc_info=True)
+            raise HTTPException(
+                status_code=500, detail="Failed to pin fact",
+            ) from None
+
+
+@router.post("/api/tiers/unpin")
+async def unpin_fact_route(request: PinRequest, profile_id: str = "default"):
+    """Unpin a fact, allowing normal tier demotion.
+
+    Lifecycle stays 'active' until the next tier evaluation cycle demotes it
+    based on access patterns. This is intentional — immediate demotion would
+    surprise the user.
+    """
+    profile_id = _validate_profile(profile_id)
+
+    with _db() as conn:
+        try:
+            conn.execute(
+                "DELETE FROM pinned_facts WHERE fact_id = ? AND profile_id = ?",
+                (request.fact_id, profile_id),
+            )
+            conn.commit()
+            return {"success": True, "unpinned": True}
+        except Exception as exc:
+            logger.error("unpin_fact failed: %s", exc, exc_info=True)
+            raise HTTPException(
+                status_code=500, detail="Failed to unpin fact",
+            ) from None

package/src/superlocalmemory/server/unified_daemon.py

@@ -268,6 +268,20 @@ async def lifespan(application: FastAPI):
             if reranker and hasattr(reranker, 'warmup_sync'):
                 reranker.warmup_sync(timeout=120)
 
+        # V3.4.11: Pre-warm embedding worker (load ONNX model on startup)
+        # Without this, first recall takes 60-90s for model load.
+        # Same pattern as reranker warmup above.
+        import threading
+        def _warmup_embedder():
+            try:
+                embedder = getattr(retrieval_eng, '_embedder', None) if retrieval_eng else None
+                if embedder and hasattr(embedder, 'embed'):
+                    embedder.embed("warmup")
+                    logger.info("Embedding worker pre-warmed (ONNX model loaded)")
+            except Exception as exc:
+                logger.warning("Embedding warmup failed: %s", exc)
+        threading.Thread(target=_warmup_embedder, daemon=True, name="embed-warmup").start()
+
     except Exception as exc:
         logger.warning("Engine init failed: %s", exc)
         application.state.engine = None
@@ -318,6 +332,8 @@ async def lifespan(application: FastAPI):
     if enable_legacy:
         asyncio.create_task(_start_legacy_redirect(_DEFAULT_PORT, _LEGACY_PORT))
 
+    global _start_time
+    _start_time = time.monotonic()
     _last_activity = time.monotonic()
     logger.info("Unified daemon ready on port %d (24/7 mode)" if idle_timeout <= 0
                 else "Unified daemon ready on port %d (idle timeout: %ds)",
@@ -422,7 +438,7 @@ def _register_dashboard_routes(application: FastAPI) -> None:
                 return JSONResponse(
                     status_code=429,
                     content={"error": "Too many requests."},
-                    headers={"Retry-After": str(limiter.window_seconds)},
+                    headers={"Retry-After": str(getattr(limiter, 'window', 60))},
                 )
             response = await call_next(request)
             response.headers["X-RateLimit-Remaining"] = str(remaining)
@@ -472,6 +488,19 @@ def _register_dashboard_routes(application: FastAPI) -> None:
     application.include_router(profiles_router)
     application.include_router(backup_router)
     application.include_router(data_io_router)
+
+    # Optional routers — ImportError-safe so missing modules don't crash startup
+    try:
+        from superlocalmemory.server.routes.tiers import router as tiers_router
+        application.include_router(tiers_router)
+    except ImportError:
+        logger.debug("tiers_router not available")
+
+    try:
+        from superlocalmemory.server.routes.evolution import router as evolution_router
+        application.include_router(evolution_router)
+    except ImportError:
+        logger.debug("evolution_router not available")
     application.include_router(events_router)
     application.include_router(agents_router)
     application.include_router(ws_router)
@@ -542,13 +571,16 @@ def _register_daemon_routes(application: FastAPI) -> None:
         }
 
     @application.get("/recall")
-    async def recall(q: str = "", limit: int = 20):
+    async def recall(q: str = "", query: str = "", limit: int = 20):
         _update_activity()
+        search_query = q or query  # Accept both ?q= and ?query= for compatibility
         engine = application.state.engine
         if engine is None:
             raise HTTPException(503, detail="Engine not initialized")
+        if not search_query:
+            return {"results": [], "count": 0, "query_type": "none", "retrieval_time_ms": 0}
         try:
-            response = engine.recall(q, limit=limit)
+            response = engine.recall(search_query, limit=limit)
             results = [
                 {
                     "content": r.fact.content,
@@ -590,7 +622,6 @@ def _register_daemon_routes(application: FastAPI) -> None:
     async def status():
         _update_activity()
         engine = application.state.engine
-        uptime = time.monotonic() - _last_activity
         fact_count = engine.fact_count if engine else 0
         mode = engine._config.mode.value if engine and hasattr(engine, '_config') else "unknown"
         return {
@@ -656,7 +687,7 @@ def _start_memory_watchdog() -> None:
     """
     import threading
 
-    MAX_WORKER_MB = 2048  # 2GB per worker — kill if exceeded
+    MAX_WORKER_MB = 4096  # 4GB per worker — ONNX full model is 1.6GB + overhead
 
     def watchdog_loop():
         while True:

package/src/superlocalmemory/storage/schema_v3410.py

@@ -0,0 +1,159 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""SuperLocalMemory V3.4.10 "Fortress" — Schema Extensions.
+
+New tables for cloud backup + entity quality:
+  - backup_destinations: Cloud backup targets (Google Drive, GitHub, local)
+  - entity_blacklist: Stop words and known garbage entity names
+
+Design rules (inherited):
+  - CREATE IF NOT EXISTS for idempotency
+  - profile_id where applicable
+  - Never ALTER existing column types
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+"""
+
+from __future__ import annotations
+
+import logging
+import sqlite3
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# DDL — Backup Destinations
+# ---------------------------------------------------------------------------
+
+_BACKUP_DESTINATIONS_DDL = """
+CREATE TABLE IF NOT EXISTS backup_destinations (
+    id TEXT PRIMARY KEY,
+    profile_id TEXT DEFAULT 'default',
+    destination_type TEXT NOT NULL,
+    display_name TEXT DEFAULT '',
+    credentials_ref TEXT DEFAULT '',
+    config TEXT DEFAULT '{}',
+    last_sync_at TEXT,
+    last_sync_status TEXT DEFAULT 'never',
+    last_sync_error TEXT DEFAULT '',
+    enabled INTEGER DEFAULT 1,
+    created_at TEXT NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS idx_backup_dest_type
+    ON backup_destinations(destination_type);
+CREATE INDEX IF NOT EXISTS idx_backup_dest_profile
+    ON backup_destinations(profile_id);
+"""
+
+# ---------------------------------------------------------------------------
+# DDL — Entity Blacklist
+# ---------------------------------------------------------------------------
+
+_ENTITY_BLACKLIST_DDL = """
+CREATE TABLE IF NOT EXISTS entity_blacklist (
+    term TEXT PRIMARY KEY,
+    reason TEXT DEFAULT 'stop_word',
+    added_at TEXT NOT NULL
+);
+"""
+
+# ---------------------------------------------------------------------------
+# Default blacklist entries (seeded on first migration)
+# ---------------------------------------------------------------------------
+
+_DEFAULT_BLACKLIST = [
+    # English stop words that frequently become garbage entities
+    "a", "an", "the", "all", "not", "no", "yes", "and", "or", "but",
+    "if", "is", "are", "was", "were", "be", "been", "being",
+    "have", "has", "had", "do", "does", "did", "will", "would",
+    "shall", "should", "can", "could", "just", "also", "only",
+    "very", "too", "so", "then", "than", "that", "this",
+    "each", "every", "both", "few", "more", "most", "other",
+    "some", "such", "any", "many", "much", "own", "same",
+    "new", "old", "first", "last", "next", "now",
+    # Months (biggest historical source of garbage)
+    "january", "february", "march", "april", "may", "june",
+    "july", "august", "september", "october", "november", "december",
+    # Technical terms commonly misclassified
+    "test", "fix", "build", "check", "run", "start", "stop",
+    "error", "status", "version", "query", "data", "file",
+    "ready", "done", "complete", "pending", "active", "failed",
+    "total", "count", "key", "value", "true", "false",
+    # Abstract nouns misclassified as people
+    "completeness", "correctness", "limitations", "requirements",
+    "performance", "security", "quality", "coverage", "progress",
+    "analysis", "research", "implementation", "verification",
+]
+
+
+# ---------------------------------------------------------------------------
+# Migration runner
+# ---------------------------------------------------------------------------
+
+def apply_v3410_schema(db_path: str | sqlite3.Connection) -> dict:
+    """Apply all v3.4.10 schema changes. Idempotent."""
+    result = {"applied": [], "errors": []}
+
+    if isinstance(db_path, sqlite3.Connection):
+        conn = db_path
+        own_connection = False
+    else:
+        conn = sqlite3.connect(str(db_path))
+        own_connection = True
+
+    try:
+        # Backup destinations table
+        try:
+            conn.executescript(_BACKUP_DESTINATIONS_DDL)
+            result["applied"].append("backup_destinations table + indexes")
+        except sqlite3.OperationalError as e:
+            result["errors"].append(f"backup_destinations: {e}")
+
+        # Entity blacklist table
+        try:
+            conn.executescript(_ENTITY_BLACKLIST_DDL)
+            result["applied"].append("entity_blacklist table")
+        except sqlite3.OperationalError as e:
+            result["errors"].append(f"entity_blacklist: {e}")
+
+        # Seed default blacklist (only inserts missing entries)
+        now = __import__("datetime").datetime.now().isoformat()
+        seeded = 0
+        for term in _DEFAULT_BLACKLIST:
+            try:
+                conn.execute(
+                    "INSERT OR IGNORE INTO entity_blacklist (term, reason, added_at) "
+                    "VALUES (?, 'stop_word', ?)",
+                    (term, now),
+                )
+                seeded += 1
+            except sqlite3.OperationalError:
+                pass
+        if seeded:
+            result["applied"].append(f"entity_blacklist seeded ({len(_DEFAULT_BLACKLIST)} terms)")
+
+        # Mark version
+        try:
+            conn.execute(
+                "INSERT OR IGNORE INTO schema_version (version, applied_at) VALUES (?, ?)",
+                ("3.4.10", now),
+            )
+        except sqlite3.OperationalError:
+            pass
+
+        conn.commit()
+
+        if result["applied"]:
+            logger.info("Schema v3.4.10 applied: %s", ", ".join(result["applied"]))
+
+    except Exception as e:
+        result["errors"].append(f"fatal: {e}")
+        logger.error("Schema v3.4.10 migration failed: %s", e)
+    finally:
+        if own_connection:
+            conn.close()
+
+    return result