superlocalmemory 3.0.16 → 3.0.18

package/src/superlocalmemory/server/routes/helpers.py

@@ -23,6 +23,27 @@ UI_DIR = Path(__file__).parent.parent / "ui"
 PROFILES_DIR = MEMORY_DIR / "profiles"
 
 
+def get_engine_lazy(app_state):
+    """Get or lazily initialize the V3 engine. Returns engine or None."""
+    engine = getattr(app_state, "engine", None)
+    if engine is not None:
+        return engine
+    if getattr(app_state, "_engine_init_attempted", False):
+        return None
+    try:
+        from superlocalmemory.core.config import SLMConfig
+        from superlocalmemory.core.engine import MemoryEngine
+        config = SLMConfig.load()
+        engine = MemoryEngine(config)
+        engine.initialize()
+        app_state.engine = engine
+        app_state._engine_init_attempted = True
+        return engine
+    except Exception:
+        app_state._engine_init_attempted = True
+        return None
+
+
 def get_db_connection() -> sqlite3.Connection:
     """Get database connection."""
     if not DB_PATH.exists():

package/src/superlocalmemory/server/routes/memories.py

@@ -12,7 +12,7 @@ from typing import Optional
 from fastapi import APIRouter, HTTPException, Query, Request
 
 from .helpers import (
-    get_db_connection, dict_factory, get_active_profile,
+    get_db_connection, dict_factory, get_active_profile, get_engine_lazy,
     SearchRequest, DB_PATH, MEMORY_DIR,
 )
 
@@ -21,8 +21,8 @@ router = APIRouter()
 
 
 def _get_engine(request: Request):
-    """Get V3 engine from app state, or None."""
-    return getattr(request.app.state, "engine", None)
+    """Get V3 engine from app state, initializing lazily on first call."""
+    return get_engine_lazy(request.app.state)
 
 
 def _preview(content: str | None) -> str:
@@ -46,19 +46,61 @@ def _fetch_graph_data(
 ) -> tuple[list, list, list]:
     """Fetch graph nodes, links, clusters from V3 or V2 schema."""
     if use_v3:
+        # Graph-first: fetch edges, then get connected nodes, then fill slots
         cursor.execute("""
-            SELECT f.fact_id as id, f.content, f.fact_type as category,
-                   f.confidence as importance, f.session_id as project_name,
-                   f.created_at
-            FROM atomic_facts f WHERE f.profile_id = ? AND f.confidence >= ?
-            ORDER BY f.confidence DESC, f.created_at DESC LIMIT ?
-        """, (profile, min_importance / 10.0, max_nodes))
-        nodes = cursor.fetchall()
+            SELECT source_id as source, target_id as target,
+                   weight, edge_type as relationship_type
+            FROM graph_edges WHERE profile_id = ?
+            ORDER BY weight DESC
+        """, (profile,))
+        all_links = cursor.fetchall()
+
+        connected_ids = set()
+        for lk in all_links:
+            connected_ids.add(lk['source'])
+            connected_ids.add(lk['target'])
+
+        # Fetch connected nodes first (these have edges to display)
+        connected_nodes: list = []
+        if connected_ids:
+            ph = ','.join('?' * len(connected_ids))
+            cursor.execute(f"""
+                SELECT fact_id as id, content, fact_type as category,
+                       confidence as importance, session_id as project_name,
+                       created_at
+                FROM atomic_facts
+                WHERE profile_id = ? AND fact_id IN ({ph})
+            """, [profile] + list(connected_ids))
+            connected_nodes = cursor.fetchall()
+
+        # Fill remaining slots with top-confidence unconnected nodes
+        remaining = max_nodes - len(connected_nodes)
+        if remaining > 0:
+            existing = {n['id'] for n in connected_nodes}
+            cursor.execute("""
+                SELECT fact_id as id, content, fact_type as category,
+                       confidence as importance, session_id as project_name,
+                       created_at
+                FROM atomic_facts
+                WHERE profile_id = ? AND confidence >= ?
+                ORDER BY confidence DESC, created_at DESC
+                LIMIT ?
+            """, (profile, min_importance / 10.0, remaining + len(existing)))
+            for n in cursor.fetchall():
+                if n['id'] not in existing:
+                    connected_nodes.append(n)
+                    if len(connected_nodes) >= max_nodes:
+                        break
+
+        nodes = connected_nodes[:max_nodes]
         for n in nodes:
             n['entities'] = []
             n['content_preview'] = _preview(n.get('content'))
-        ids = [n['id'] for n in nodes]
-        links = _fetch_edges_v3(cursor, profile, ids)
+
+        # Filter edges to only those between displayed nodes
+        node_ids = {n['id'] for n in nodes}
+        links = [lk for lk in all_links
+                 if lk['source'] in node_ids and lk['target'] in node_ids]
         return nodes, links, []
 
     # V2 fallback
@@ -160,7 +202,7 @@ async def get_memories(
 
         if use_v3:
             query = """
-                SELECT fact_id as id, content, fact_type as category,
+                SELECT fact_id as id, memory_id, content, fact_type as category,
                        confidence as importance, access_count,
                        created_at, created_at as updated_at,
                        session_id as project_name
@@ -271,39 +313,22 @@ async def get_graph(
 
 @router.post("/api/search")
 async def search_memories(request: Request, body: SearchRequest):
-    """Semantic search using V3 engine recall or fallback."""
+    """Semantic search via subprocess worker pool (memory-isolated)."""
     try:
-        engine = _get_engine(request)
-
-        if engine:
-            response = engine.recall(body.query, limit=body.limit)
-            results = []
-            for r in response.results:
-                score = r.score
-                if score < body.min_score:
-                    continue
-                if body.category and getattr(r.fact, 'fact_type', None) != body.category:
-                    continue
-                results.append({
-                    "id": r.fact.fact_id,
-                    "content": r.fact.content,
-                    "score": round(score, 4),
-                    "confidence": round(r.confidence, 4),
-                    "trust_score": round(r.trust_score, 4) if r.trust_score else None,
-                    "channel_scores": r.channel_scores,
-                    "fact_type": getattr(r.fact, 'fact_type', None),
-                    "created_at": getattr(r.fact, 'created_at', None),
-                })
-                if len(results) >= body.limit:
-                    break
+        from superlocalmemory.core.worker_pool import WorkerPool
+        pool = WorkerPool.shared()
+        result = pool.recall(body.query, limit=body.limit)
 
+        if result.get("ok"):
             return {
-                "query": body.query, "results": results, "total": len(results),
-                "query_type": response.query_type,
-                "retrieval_time_ms": response.retrieval_time_ms,
+                "query": body.query,
+                "results": result.get("results", []),
+                "total": result.get("result_count", 0),
+                "query_type": result.get("query_type", "unknown"),
+                "retrieval_time_ms": result.get("retrieval_time_ms", 0),
             }
 
-        # Fallback: direct DB search (no V3 engine)
+        # Fallback: direct DB text search (no engine needed)
         conn = get_db_connection()
         conn.row_factory = dict_factory
         cursor = conn.cursor()
@@ -392,8 +417,41 @@ async def get_cluster_detail(request: Request, cluster_id: int, limit: int = Que
         conn.close()
         if not members:
             raise HTTPException(status_code=404, detail="Cluster not found")
-        return {"cluster_info": {"cluster_id": cluster_id, "total_members": len(members)}, "members": members, "connections": []}
+        # Generate cluster summary
+        summary = ""
+        try:
+            from superlocalmemory.core.worker_pool import WorkerPool
+            pool = WorkerPool.shared()
+            texts = [m.get("content", "")[:200] for m in members[:10] if m.get("content")]
+            if texts:
+                result = pool.summarize(texts)
+                summary = result.get("summary", "") if result.get("ok") else ""
+        except Exception:
+            pass
+
+        return {
+            "cluster_info": {"cluster_id": cluster_id, "total_members": len(members)},
+            "summary": summary,
+            "members": members,
+            "connections": [],
+        }
     except HTTPException:
         raise
     except Exception as e:
         raise HTTPException(status_code=500, detail=f"Cluster detail error: {str(e)}")
+
+
+@router.get("/api/memories/{memory_id}/facts")
+async def get_memory_facts(request: Request, memory_id: str):
+    """Get original memory text with all its child atomic facts."""
+    try:
+        from superlocalmemory.core.worker_pool import WorkerPool
+        pool = WorkerPool.shared()
+        result = pool.get_memory_facts(memory_id)
+        if result.get("ok"):
+            return result
+        raise HTTPException(status_code=404, detail="Memory not found")
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error: {str(e)}")

package/src/superlocalmemory/server/routes/stats.py

@@ -72,6 +72,17 @@ async def get_stats():
                 total_clusters = cursor.fetchone()['total']
             except Exception:
                 pass
+            # Fallback: V2-migrated clusters stored as cluster_id on memories
+            if total_clusters == 0:
+                try:
+                    cursor.execute(
+                        "SELECT COUNT(DISTINCT cluster_id) as total FROM memories "
+                        "WHERE cluster_id IS NOT NULL AND profile = ?",
+                        (active_profile,),
+                    )
+                    total_clusters = cursor.fetchone()['total']
+                except Exception:
+                    pass
 
             # Fact type breakdown (replaces category in V3)
             cursor.execute("""

package/src/superlocalmemory/server/routes/v3_api.py

@@ -8,6 +8,7 @@ from __future__ import annotations
 
 import json
 import logging
+import os
 from pathlib import Path
 from fastapi import APIRouter, Request
 from fastapi.responses import JSONResponse
@@ -63,17 +64,22 @@ async def dashboard(request: Request):
 
 @router.get("/mode")
 async def get_mode():
-    """Get current operating mode."""
+    """Get current mode, provider, model — single source of truth for UI."""
     try:
         from superlocalmemory.core.config import SLMConfig
         config = SLMConfig.load()
-        modes = {
-            "a": {"name": "Local Guardian", "description": "Zero cloud. Your data never leaves your machine.", "llm": False, "eu_compliant": True},
-            "b": {"name": "Smart Local", "description": "Local LLM via Ollama. Still fully private.", "llm": "local", "eu_compliant": True},
-            "c": {"name": "Full Power", "description": "Cloud LLM for maximum accuracy.", "llm": "cloud", "eu_compliant": False},
-        }
         current = config.mode.value
-        return {"current": current, "details": modes.get(current, {}), "all_modes": modes}
+        return {
+            "mode": current,
+            "provider": config.llm.provider or "none",
+            "model": config.llm.model or "",
+            "has_key": bool(config.llm.api_key),
+            "endpoint": config.llm.api_base or "",
+            "capabilities": {
+                "llm_available": bool(config.llm.provider),
+                "cross_encoder": config.retrieval.use_cross_encoder if hasattr(config, 'retrieval') else False,
+            },
+        }
     except Exception as e:
         return JSONResponse({"error": str(e)}, status_code=500)
 
@@ -109,6 +115,127 @@ async def set_mode(request: Request):
         return JSONResponse({"error": str(e)}, status_code=500)
 
 
+@router.post("/mode/set")
+async def set_full_config(request: Request):
+    """Save mode + provider + model + API key together."""
+    try:
+        body = await request.json()
+        new_mode = body.get("mode", "a").lower()
+        provider = body.get("provider", "none")
+        model = body.get("model", "")
+        api_key = body.get("api_key", "")
+
+        if new_mode not in ("a", "b", "c"):
+            return JSONResponse({"error": "Invalid mode"}, status_code=400)
+
+        from superlocalmemory.core.config import SLMConfig
+        from superlocalmemory.storage.models import Mode
+        config = SLMConfig.for_mode(
+            Mode(new_mode),
+            llm_provider=provider if provider != "none" else "",
+            llm_model=model,
+            llm_api_key=api_key,
+            llm_api_base="http://localhost:11434" if provider == "ollama" else "",
+        )
+        old = SLMConfig.load()
+        config.active_profile = old.active_profile
+        config.save()
+
+        # Kill existing worker so next request uses new config
+        try:
+            from superlocalmemory.core.worker_pool import WorkerPool
+            WorkerPool.shared().shutdown()
+        except Exception:
+            pass
+
+        if hasattr(request.app.state, "engine"):
+            request.app.state.engine = None
+
+        return {
+            "success": True,
+            "mode": new_mode,
+            "provider": provider,
+            "model": model,
+        }
+    except Exception as e:
+        return JSONResponse({"error": str(e)}, status_code=500)
+
+
+@router.post("/provider/test")
+async def test_provider(request: Request):
+    """Test connectivity to an LLM provider."""
+    try:
+        import httpx
+        body = await request.json()
+        provider = body.get("provider", "")
+        model = body.get("model", "")
+        api_key = body.get("api_key", "")
+
+        if provider == "ollama":
+            endpoint = body.get("endpoint", "http://localhost:11434")
+            with httpx.Client(timeout=httpx.Timeout(5.0)) as c:
+                resp = c.get(f"{endpoint}/api/tags")
+                resp.raise_for_status()
+                models = [m["name"] for m in resp.json().get("models", [])]
+                found = model in models if model else len(models) > 0
+                return {
+                    "success": found,
+                    "message": f"Ollama OK, {len(models)} models" + (f", '{model}' available" if found and model else ""),
+                }
+
+        if provider == "openrouter":
+            if not api_key:
+                api_key = os.environ.get("OPENROUTER_API_KEY", "")
+            if not api_key:
+                return {"success": False, "error": "API key required"}
+            with httpx.Client(timeout=httpx.Timeout(10.0)) as c:
+                resp = c.get("https://openrouter.ai/api/v1/models", headers={"Authorization": f"Bearer {api_key}"})
+                resp.raise_for_status()
+                return {"success": True, "message": "OpenRouter connected, key valid"}
+
+        if provider == "openai":
+            if not api_key:
+                return {"success": False, "error": "API key required"}
+            with httpx.Client(timeout=httpx.Timeout(10.0)) as c:
+                resp = c.get("https://api.openai.com/v1/models", headers={"Authorization": f"Bearer {api_key}"})
+                resp.raise_for_status()
+                return {"success": True, "message": "OpenAI connected, key valid"}
+
+        if provider == "anthropic":
+            if not api_key:
+                return {"success": False, "error": "API key required"}
+            # Anthropic doesn't have a models list endpoint, just verify key format
+            if api_key.startswith("sk-ant-"):
+                return {"success": True, "message": "Anthropic key format valid"}
+            return {"success": False, "error": "Key should start with sk-ant-"}
+
+        return {"success": False, "error": f"Unknown provider: {provider}"}
+    except httpx.ConnectError:
+        return {"success": False, "error": "Cannot connect — is the service running?"}
+    except httpx.HTTPStatusError as e:
+        return {"success": False, "error": f"HTTP {e.response.status_code}: Invalid key or endpoint"}
+    except Exception as e:
+        return {"success": False, "error": str(e)}
+
+
+@router.get("/ollama/status")
+async def ollama_status():
+    """Check if Ollama is running and list available models."""
+    try:
+        import httpx
+        with httpx.Client(timeout=httpx.Timeout(5.0)) as client:
+            resp = client.get("http://localhost:11434/api/tags")
+            resp.raise_for_status()
+            data = resp.json()
+            models = [
+                {"name": m["name"], "size": m.get("size", 0)}
+                for m in data.get("models", [])
+            ]
+            return {"running": True, "models": models, "count": len(models)}
+    except Exception:
+        return {"running": False, "models": [], "count": 0}
+
+
 # ── Provider ─────────────────────────────────────────────────
 
 @router.get("/providers")
@@ -187,28 +314,32 @@ async def recall_trace(request: Request):
         query = body.get("query", "")
         limit = body.get("limit", 10)
 
-        engine = getattr(request.app.state, "engine", None)
-        if not engine:
-            return JSONResponse({"error": "Engine not initialized"}, status_code=503)
-
-        response = engine.recall(query, limit=limit)
-        results = []
-        for r in response.results[:limit]:
-            results.append({
-                "fact_id": r.fact.fact_id,
-                "content": r.fact.content[:300],
-                "score": round(r.score, 4),
-                "confidence": round(r.confidence, 4),
-                "trust_score": round(r.trust_score, 4),
-                "channel_scores": {k: round(v, 4) for k, v in (r.channel_scores or {}).items()},
-            })
+        from superlocalmemory.core.worker_pool import WorkerPool
+        pool = WorkerPool.shared()
+        result = pool.recall(query, limit=limit)
+
+        if not result.get("ok"):
+            return JSONResponse(
+                {"error": result.get("error", "Recall failed")},
+                status_code=503,
+            )
+
+        # Optional: synthesize answer from results (Mode B/C only)
+        synthesis = ""
+        if body.get("synthesize") and result.get("results"):
+            try:
+                syn_result = pool.synthesize(query, result["results"][:5])
+                synthesis = syn_result.get("synthesis", "") if syn_result.get("ok") else ""
+            except Exception:
+                pass
 
         return {
             "query": query,
-            "query_type": response.query_type,
-            "result_count": len(results),
-            "retrieval_time_ms": round(response.retrieval_time_ms, 1),
-            "results": results,
+            "query_type": result.get("query_type", "unknown"),
+            "result_count": result.get("result_count", 0),
+            "retrieval_time_ms": result.get("retrieval_time_ms", 0),
+            "results": result.get("results", []),
+            "synthesis": synthesis,
         }
     except Exception as e:
         return JSONResponse({"error": str(e)}, status_code=500)
@@ -218,29 +349,50 @@ async def recall_trace(request: Request):
 
 @router.get("/trust/dashboard")
 async def trust_dashboard(request: Request):
-    """Trust overview: per-agent scores, alerts."""
+    """Trust overview: per-agent scores, alerts. Queries DB directly."""
     try:
-        engine = getattr(request.app.state, "engine", None)
-        if not engine or not engine._trust_scorer:
-            return {"agents": [], "alerts": [], "message": "Trust scorer not available"}
-
         from superlocalmemory.core.config import SLMConfig
+        from superlocalmemory.storage.database import DatabaseManager
+        from superlocalmemory.storage import schema as _schema
         config = SLMConfig.load()
-        scores = engine._trust_scorer.get_all_scores(config.active_profile)
+        pid = config.active_profile
+
+        db_path = config.db_path
+        db = DatabaseManager(db_path)
+        db.initialize(_schema)
 
+        # Query trust scores from DB
         agents = []
-        for s in scores:
-            if isinstance(s, dict):
-                agents.append(s)
-            else:
+        try:
+            rows = db.execute(
+                "SELECT target_id, target_type, trust_score, evidence_count, "
+                "last_updated FROM trust_scores WHERE profile_id = ? "
+                "ORDER BY trust_score DESC",
+                (pid,),
+            )
+            for r in rows:
+                d = dict(r)
                 agents.append({
-                    "target_id": s.target_id,
-                    "target_type": s.target_type,
-                    "trust_score": round(s.trust_score, 3),
-                    "evidence_count": s.evidence_count,
+                    "target_id": d.get("target_id", ""),
+                    "target_type": d.get("target_type", ""),
+                    "trust_score": round(float(d.get("trust_score", 0.5)), 3),
+                    "evidence_count": d.get("evidence_count", 0),
+                    "last_updated": d.get("last_updated", ""),
                 })
+        except Exception:
+            pass
+
+        # Aggregate stats
+        avg = round(sum(a["trust_score"] for a in agents) / len(agents), 3) if agents else 0.5
+        alerts = [a for a in agents if a["trust_score"] < 0.3]
 
-        return {"agents": agents, "alerts": [], "profile": config.active_profile}
+        return {
+            "agents": agents,
+            "avg_trust": avg,
+            "alerts": alerts,
+            "total": len(agents),
+            "profile": pid,
+        }
     except Exception as e:
         return JSONResponse({"error": str(e)}, status_code=500)
 
@@ -249,9 +401,9 @@ async def trust_dashboard(request: Request):
 
 @router.get("/math/health")
 async def math_health(request: Request):
-    """Mathematical layer health: Fisher, sheaf, Langevin status."""
+    """Mathematical layer health: Fisher, sheaf, Langevin status. Queries DB directly."""
     try:
-        engine = getattr(request.app.state, "engine", None)
+        engine = None  # Engine runs in subprocess; query DB directly below
 
         health = {
             "fisher": {"status": "active", "description": "Fisher-Rao information geometry for similarity"},

package/src/superlocalmemory/server/ui.py

@@ -196,22 +196,23 @@ def create_app() -> FastAPI:
 
     @application.on_event("startup")
     async def startup_event():
-        """Initialize V3 engine and event bus on startup."""
-        # Initialize V3 engine for dashboard API routes
-        try:
-            from superlocalmemory.core.config import SLMConfig
-            from superlocalmemory.core.engine import MemoryEngine
-            config = SLMConfig.load()
-            engine = MemoryEngine(config)
-            engine.initialize()
-            application.state.engine = engine
-            logger.info("V3 engine initialized for dashboard")
-        except Exception as exc:
-            logger.warning("V3 engine init failed: %s (V3 API routes will be unavailable)", exc)
-            application.state.engine = None
-
+        """Initialize event bus. Engine runs in subprocess worker (never in this process)."""
+        # Engine is NEVER loaded in the dashboard process.
+        # All recall/search operations go through WorkerPool subprocess.
+        # This keeps the dashboard permanently at ~60 MB.
+        application.state.engine = None
+        logger.info("Dashboard started (~60 MB, engine runs in subprocess worker)")
         register_event_listener()
 
+    @application.on_event("shutdown")
+    async def shutdown_event():
+        """Kill worker subprocess on dashboard shutdown."""
+        try:
+            from superlocalmemory.core.worker_pool import WorkerPool
+            WorkerPool.shared().shutdown()
+        except Exception:
+            pass
+
     return application
 
 

package/src/superlocalmemory/storage/database.py

@@ -300,6 +300,29 @@ class DatabaseManager:
             for r in rows
         ]
 
+    def get_memory_content_batch(self, memory_ids: list[str]) -> dict[str, str]:
+        """Batch-fetch original memory text. Returns {memory_id: content}."""
+        if not memory_ids:
+            return {}
+        unique_ids = list(set(memory_ids))
+        ph = ','.join('?' * len(unique_ids))
+        rows = self.execute(
+            f"SELECT memory_id, content FROM memories WHERE memory_id IN ({ph})",
+            tuple(unique_ids),
+        )
+        return {dict(r)["memory_id"]: dict(r)["content"] for r in rows}
+
+    def get_facts_by_memory_id(
+        self, memory_id: str, profile_id: str,
+    ) -> list[AtomicFact]:
+        """Get all atomic facts for a given memory_id."""
+        rows = self.execute(
+            "SELECT * FROM atomic_facts WHERE memory_id = ? AND profile_id = ? "
+            "ORDER BY confidence DESC",
+            (memory_id, profile_id),
+        )
+        return [self._row_to_fact(r) for r in rows]
+
     def store_edge(self, edge: GraphEdge) -> str:
         """Persist a graph edge. Returns edge_id."""
         self.execute(

package/src/superlocalmemory.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: superlocalmemory
-Version: 3.0.16
+Version: 3.0.18
 Summary: Information-geometric agent memory with mathematical guarantees
 Author-email: Varun Pratap Bhardwaj <admin@superlocalmemory.com>
 License: MIT

package/src/superlocalmemory.egg-info/SOURCES.txt

@@ -30,13 +30,17 @@ src/superlocalmemory/compliance/retention.py
 src/superlocalmemory/compliance/scheduler.py
 src/superlocalmemory/core/__init__.py
 src/superlocalmemory/core/config.py
+src/superlocalmemory/core/embedding_worker.py
 src/superlocalmemory/core/embeddings.py
 src/superlocalmemory/core/engine.py
 src/superlocalmemory/core/hooks.py
 src/superlocalmemory/core/maintenance.py
 src/superlocalmemory/core/modes.py
 src/superlocalmemory/core/profiles.py
+src/superlocalmemory/core/recall_worker.py
 src/superlocalmemory/core/registry.py
+src/superlocalmemory/core/summarizer.py
+src/superlocalmemory/core/worker_pool.py
 src/superlocalmemory/dynamics/__init__.py
 src/superlocalmemory/dynamics/fisher_langevin_coupling.py
 src/superlocalmemory/encoding/__init__.py