superlocalmemory 3.3.29 → 3.4.1

package/src/superlocalmemory/server/routes/insights.py

@@ -0,0 +1,368 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later — see LICENSE file
+# Part of SuperLocalMemory v3.4.1 | https://qualixar.com
+
+"""Quick Insight Actions — 5 one-click intelligence endpoints.
+
+Actions: changed_this_week, opinions, contradictions, health, cross_project.
+All queries use direct sqlite3 (Rule 06), parameterized SQL (Rule 11),
+profile-scoped (Rule 01).
+"""
+
+from __future__ import annotations
+
+import logging
+import sqlite3
+from typing import Any, Callable
+
+from fastapi import APIRouter, Query
+from fastapi.responses import JSONResponse
+from superlocalmemory.server.routes.helpers import DB_PATH, get_active_profile
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/api/v3/insights", tags=["insights"])
+
+
+# ── Helper ────────────────────────────────────────────────────────
+
+def _get_conn(profile: str = "") -> tuple[sqlite3.Connection | None, str]:
+    """Open sqlite3 connection and resolve profile_id."""
+    pid = profile or get_active_profile()
+    if not DB_PATH.exists():
+        return None, pid
+    conn = sqlite3.connect(str(DB_PATH))
+    conn.row_factory = sqlite3.Row
+    return conn, pid
+
+
+# ── Action Handlers ───────────────────────────────────────────────
+
+def _action_changed_this_week(
+    conn: sqlite3.Connection, pid: str, limit: int = 50, days: int = 7,
+) -> dict[str, Any]:
+    """Facts created or modified in last N days."""
+    items = []
+    modified = []
+
+    # New facts
+    try:
+        rows = conn.execute(
+            "SELECT fact_id, content, fact_type, created_at, session_id, confidence "
+            "FROM atomic_facts "
+            "WHERE profile_id = ? AND created_at >= datetime('now', ?) "
+            "ORDER BY created_at DESC LIMIT ?",
+            (pid, f"-{days} days", limit),
+        ).fetchall()
+        items = [
+            {
+                "fact_id": dict(r)["fact_id"],
+                "content": (dict(r).get("content") or "")[:200],
+                "fact_type": dict(r).get("fact_type", ""),
+                "created_at": dict(r).get("created_at", ""),
+                "session_id": dict(r).get("session_id", ""),
+                "confidence": round(float(dict(r).get("confidence", 0)), 3),
+            }
+            for r in rows
+        ]
+    except Exception as exc:
+        logger.debug("changed_this_week new facts failed: %s", exc)
+
+    # Modified facts (consolidation updates)
+    try:
+        rows = conn.execute(
+            "SELECT cl.action_type, cl.new_fact_id, cl.existing_fact_id, "
+            "cl.reason, cl.timestamp, af.content AS new_content "
+            "FROM consolidation_log cl "
+            "LEFT JOIN atomic_facts af ON af.fact_id = cl.new_fact_id "
+            "WHERE cl.profile_id = ? AND cl.timestamp >= datetime('now', ?) "
+            "AND cl.action_type IN ('update', 'supersede') "
+            "ORDER BY cl.timestamp DESC LIMIT ?",
+            (pid, f"-{days} days", limit),
+        ).fetchall()
+        modified = [
+            {
+                "action_type": dict(r).get("action_type", ""),
+                "new_fact_id": dict(r).get("new_fact_id", ""),
+                "existing_fact_id": dict(r).get("existing_fact_id", ""),
+                "reason": (dict(r).get("reason") or "")[:200],
+                "timestamp": dict(r).get("timestamp", ""),
+                "new_content": (dict(r).get("new_content") or "")[:200],
+            }
+            for r in rows
+        ]
+    except Exception as exc:
+        logger.debug("changed_this_week modifications failed: %s", exc)
+
+    return {
+        "action": "changed_this_week",
+        "count": len(items),
+        "items": items,
+        "metadata": {
+            "days": days,
+            "modified_count": len(modified),
+            "modifications": modified,
+        },
+    }
+
+
+def _action_opinions(
+    conn: sqlite3.Connection, pid: str, limit: int = 50, **_: Any,
+) -> dict[str, Any]:
+    """Opinion-type facts (user preferences, decisions, rationale)."""
+    items = []
+    try:
+        rows = conn.execute(
+            "SELECT fact_id, content, created_at, session_id, confidence "
+            "FROM atomic_facts "
+            "WHERE profile_id = ? AND fact_type = 'opinion' AND lifecycle = 'active' "
+            "ORDER BY created_at DESC LIMIT ?",
+            (pid, limit),
+        ).fetchall()
+        items = [
+            {
+                "fact_id": dict(r)["fact_id"],
+                "content": (dict(r).get("content") or "")[:200],
+                "created_at": dict(r).get("created_at", ""),
+                "session_id": dict(r).get("session_id", ""),
+                "confidence": round(float(dict(r).get("confidence", 0)), 3),
+            }
+            for r in rows
+        ]
+    except Exception as exc:
+        logger.debug("opinions query failed: %s", exc)
+
+    return {"action": "opinions", "count": len(items), "items": items}
+
+
+def _action_contradictions(
+    conn: sqlite3.Connection, pid: str, limit: int = 50, **_: Any,
+) -> dict[str, Any]:
+    """Contradiction edges from sheaf cohomology."""
+    items = []
+    try:
+        rows = conn.execute(
+            "SELECT ge.edge_id, ge.source_id, ge.target_id, ge.weight, ge.created_at, "
+            "sf.content AS source_content, tf.content AS target_content "
+            "FROM graph_edges ge "
+            "LEFT JOIN atomic_facts sf ON sf.fact_id = ge.source_id "
+            "LEFT JOIN atomic_facts tf ON tf.fact_id = ge.target_id "
+            "WHERE ge.profile_id = ? AND ge.edge_type = 'contradiction' "
+            "ORDER BY ge.weight DESC, ge.created_at DESC LIMIT ?",
+            (pid, limit),
+        ).fetchall()
+        items = [
+            {
+                "edge_id": dict(r).get("edge_id", ""),
+                "source_id": dict(r).get("source_id", ""),
+                "target_id": dict(r).get("target_id", ""),
+                "severity": round(float(dict(r).get("weight", 0)), 3),
+                "source_content": (dict(r).get("source_content") or "")[:200],
+                "target_content": (dict(r).get("target_content") or "")[:200],
+                "created_at": dict(r).get("created_at", ""),
+            }
+            for r in rows
+        ]
+    except Exception as exc:
+        logger.debug("contradictions query failed: %s", exc)
+
+    return {"action": "contradictions", "count": len(items), "items": items}
+
+
+def _action_health(
+    conn: sqlite3.Connection, pid: str, **_: Any,
+) -> dict[str, Any]:
+    """Aggregate memory health: trust, retention, coverage, counts."""
+    trust = {"high": 0, "medium": 0, "low": 0, "total": 0, "avg": 0.5}
+    retention_zones: dict[str, Any] | None = None
+    coverage: dict[str, int] = {}
+    totals = {"facts": 0, "entities": 0, "edges": 0}
+    community_count = 0
+
+    # Trust distribution
+    try:
+        row = conn.execute(
+            "SELECT "
+            "COUNT(CASE WHEN trust_score >= 0.7 THEN 1 END) AS high_trust, "
+            "COUNT(CASE WHEN trust_score >= 0.4 AND trust_score < 0.7 THEN 1 END) AS med_trust, "
+            "COUNT(CASE WHEN trust_score < 0.4 THEN 1 END) AS low_trust, "
+            "COUNT(*) AS total, ROUND(AVG(trust_score), 3) AS avg_trust "
+            "FROM trust_scores WHERE profile_id = ?",
+            (pid,),
+        ).fetchone()
+        if row:
+            d = dict(row)
+            trust = {
+                "high": d.get("high_trust", 0),
+                "medium": d.get("med_trust", 0),
+                "low": d.get("low_trust", 0),
+                "total": d.get("total", 0),
+                "avg": float(d.get("avg_trust", 0) or 0.5),
+            }
+    except Exception:
+        pass
+
+    # Retention zones (v3.2+)
+    try:
+        rows = conn.execute(
+            "SELECT lifecycle_zone, COUNT(*) AS cnt, "
+            "ROUND(AVG(retention_score), 3) AS avg_retention "
+            "FROM fact_retention WHERE profile_id = ? "
+            "GROUP BY lifecycle_zone",
+            (pid,),
+        ).fetchall()
+        retention_zones = {
+            dict(r)["lifecycle_zone"]: {
+                "count": dict(r)["cnt"],
+                "avg_retention": float(dict(r).get("avg_retention", 0) or 0),
+            }
+            for r in rows
+        }
+    except Exception:
+        pass  # Table may not exist in older DBs
+
+    # Coverage by fact_type
+    try:
+        rows = conn.execute(
+            "SELECT fact_type, COUNT(*) AS cnt "
+            "FROM atomic_facts WHERE profile_id = ? AND lifecycle = 'active' "
+            "GROUP BY fact_type",
+            (pid,),
+        ).fetchall()
+        coverage = {dict(r)["fact_type"]: dict(r)["cnt"] for r in rows}
+    except Exception:
+        pass
+
+    # Totals
+    try:
+        row = conn.execute(
+            "SELECT "
+            "(SELECT COUNT(*) FROM atomic_facts WHERE profile_id = ?) AS total_facts, "
+            "(SELECT COUNT(*) FROM canonical_entities WHERE profile_id = ?) AS total_entities, "
+            "(SELECT COUNT(*) FROM graph_edges WHERE profile_id = ?) AS total_edges",
+            (pid, pid, pid),
+        ).fetchone()
+        if row:
+            d = dict(row)
+            totals = {
+                "facts": d.get("total_facts", 0),
+                "entities": d.get("total_entities", 0),
+                "edges": d.get("total_edges", 0),
+            }
+    except Exception:
+        pass
+
+    # Community count
+    try:
+        row = conn.execute(
+            "SELECT COUNT(DISTINCT community_id) AS cnt "
+            "FROM fact_importance WHERE profile_id = ? AND community_id IS NOT NULL",
+            (pid,),
+        ).fetchone()
+        if row:
+            community_count = dict(row).get("cnt", 0)
+    except Exception:
+        pass
+
+    return {
+        "action": "health",
+        "count": 1,
+        "items": [{
+            "trust": trust,
+            "retention_zones": retention_zones,
+            "coverage": coverage,
+            "totals": totals,
+            "community_count": community_count,
+        }],
+    }
+
+
+def _action_cross_project(
+    conn: sqlite3.Connection, pid: str, limit: int = 50, **_: Any,
+) -> dict[str, Any]:
+    """Entities spanning multiple sessions."""
+    items = []
+    try:
+        rows = conn.execute(
+            "SELECT ce.entity_id, ce.canonical_name, ce.entity_type, ce.fact_count, "
+            "COUNT(DISTINCT af.session_id) AS session_count, "
+            "GROUP_CONCAT(DISTINCT af.session_id) AS session_ids "
+            "FROM canonical_entities ce "
+            "JOIN atomic_facts af ON af.profile_id = ? AND EXISTS ("
+            "  SELECT 1 FROM json_each(af.canonical_entities_json) je "
+            "  WHERE je.value = ce.entity_id"
+            ") "
+            "WHERE ce.profile_id = ? "
+            "GROUP BY ce.entity_id "
+            "HAVING COUNT(DISTINCT af.session_id) > 1 "
+            "ORDER BY session_count DESC, ce.fact_count DESC LIMIT ?",
+            (pid, pid, limit),
+        ).fetchall()
+        items = [
+            {
+                "entity_id": dict(r).get("entity_id", ""),
+                "canonical_name": dict(r).get("canonical_name", ""),
+                "entity_type": dict(r).get("entity_type", ""),
+                "fact_count": dict(r).get("fact_count", 0),
+                "session_count": dict(r).get("session_count", 0),
+                "session_ids": (dict(r).get("session_ids") or "").split(",")[:10],
+            }
+            for r in rows
+        ]
+    except Exception as exc:
+        logger.debug("cross_project query failed: %s", exc)
+
+    return {"action": "cross_project", "count": len(items), "items": items}
+
+
+# ── Dispatch Map ──────────────────────────────────────────────────
+
+ALLOWED_ACTIONS: dict[str, Callable] = {
+    "changed_this_week": _action_changed_this_week,
+    "opinions": _action_opinions,
+    "contradictions": _action_contradictions,
+    "health": _action_health,
+    "cross_project": _action_cross_project,
+}
+
+
+# ── Endpoint ──────────────────────────────────────────────────────
+
+@router.get("/{action_name}")
+async def insight_action(
+    action_name: str,
+    profile: str = "",
+    limit: int = Query(default=50, ge=1, le=200),
+    days: int = Query(default=7, ge=1, le=90),
+):
+    """Run a quick insight action against the memory database."""
+    handler = ALLOWED_ACTIONS.get(action_name)
+    if not handler:
+        valid = ", ".join(sorted(ALLOWED_ACTIONS.keys()))
+        return JSONResponse(
+            {"error": f"Unknown action: '{action_name}'. Valid: {valid}"},
+            status_code=400,
+        )
+
+    conn, pid = _get_conn(profile)
+    if conn is None:
+        return {
+            "action": action_name,
+            "profile": pid,
+            "count": 0,
+            "items": [],
+            "metadata": {"error": "Database not found"},
+        }
+
+    try:
+        result = handler(conn, pid, limit=limit, days=days)
+        result["profile"] = pid
+        return result
+    except Exception as exc:
+        logger.warning("Insight action %s failed: %s", action_name, exc)
+        return JSONResponse(
+            {"error": f"Query failed: {exc}"},
+            status_code=500,
+        )
+    finally:
+        conn.close()

package/src/superlocalmemory/server/routes/learning.py

@@ -25,17 +25,25 @@ LEARNING_DB = MEMORY_DIR / "learning.db"
 
 # Feature detection
 LEARNING_AVAILABLE = False
+BEHAVIORAL_AVAILABLE = False
 try:
     from superlocalmemory.learning.feedback import FeedbackCollector
     from superlocalmemory.learning.engagement import EngagementTracker
     from superlocalmemory.learning.ranker import AdaptiveRanker
     LEARNING_AVAILABLE = True
-except ImportError:
+except ImportError as e:
+    logger.warning("V3 learning primary import failed: %s", e)
     try:
         from superlocalmemory.learning.adaptive import AdaptiveLearner
         LEARNING_AVAILABLE = True
-    except ImportError:
-        logger.info("V3 learning system not available")
+    except ImportError as e2:
+        logger.warning("V3 learning fallback import failed: %s", e2)
+
+try:
+    from superlocalmemory.learning.behavioral import BehavioralPatternStore
+    BEHAVIORAL_AVAILABLE = True
+except ImportError as e:
+    logger.warning("V3 behavioral import failed: %s", e)
 
 # Lazy singletons
 _feedback: FeedbackCollector | None = None
@@ -86,7 +94,7 @@ async def learning_status():
             from superlocalmemory.learning.feedback import FeedbackCollector
             from pathlib import Path
             import sqlite3 as _sqlite3
-            learning_db = Path.home() / ".superlocalmemory" / "learning.db"
+            learning_db = LEARNING_DB
             if learning_db.exists():
                 collector = FeedbackCollector(learning_db)
                 signal_count = collector.get_feedback_count(active_profile)
@@ -149,7 +157,7 @@ async def learning_status():
         try:
             from superlocalmemory.learning.behavioral import BehavioralPatternStore
             from pathlib import Path
-            learning_db = Path.home() / ".superlocalmemory" / "learning.db"
+            learning_db = LEARNING_DB
             if learning_db.exists():
                 store = BehavioralPatternStore(str(learning_db))
                 all_patterns = store.get_patterns(profile_id=active_profile)
@@ -179,9 +187,11 @@ async def learning_status():
             else:
                 result["tech_preferences"] = []
                 result["workflow_patterns"] = []
-        except Exception:
+        except Exception as exc:
+            logger.error("Error fetching behavioral patterns: %s", exc, exc_info=True)
             result["tech_preferences"] = []
             result["workflow_patterns"] = []
+            result["pattern_error"] = str(exc)
         result["source_scores"] = {}
 
     except Exception as e:
@@ -310,6 +320,96 @@ async def feedback_stats():
         return {"total_signals": 0, "ranking_phase": "baseline", "progress": 0, "error": str(e)}
 
 
+# ============================================================================
+# PATTERNS ENDPOINT (v3.4.1 — CRITICAL FIX: frontend calls /api/patterns)
+# ============================================================================
+
+@router.get("/api/patterns")
+async def get_patterns():
+    """Get learned behavioral patterns for the Patterns dashboard tab.
+
+    v3.4.1: This endpoint was MISSING — patterns.js calls /api/patterns
+    but no backend route existed. The frontend always showed 'No patterns'.
+    Now queries BehavioralPatternStore + learning signals.
+    """
+    active_profile = get_active_profile()
+
+    patterns: dict = {
+        "preference": [],
+        "style": [],
+        "terminology": [],
+        "workflow": [],
+    }
+    result: dict = {
+        "available": BEHAVIORAL_AVAILABLE or LEARNING_AVAILABLE,
+        "patterns": patterns,
+        "signal_stats": {},
+    }
+
+    # Behavioral patterns from BehavioralPatternStore
+    if BEHAVIORAL_AVAILABLE:
+        try:
+            store = BehavioralPatternStore(str(LEARNING_DB))
+            all_patterns = store.get_patterns(profile_id=active_profile)
+            for p in all_patterns:
+                ptype = p.get("pattern_type", "")
+                entry = {
+                    "key": p.get("pattern_key", ""),
+                    "value": p.get("metadata", {}).get("value", ""),
+                    "confidence": round(float(p.get("confidence", 0)), 3),
+                    "evidence_count": p.get("evidence_count", 0),
+                    "created_at": p.get("created_at", ""),
+                    "updated_at": p.get("updated_at", ""),
+                }
+                if ptype == "tech_preference":
+                    patterns["preference"].append(entry)
+                elif ptype == "style":
+                    patterns["style"].append(entry)
+                elif ptype == "terminology":
+                    patterns["terminology"].append(entry)
+                elif ptype in ("temporal", "interest", "workflow"):
+                    patterns["workflow"].append(entry)
+                else:
+                    patterns["preference"].append(entry)
+        except Exception as exc:
+            logger.error("Error loading patterns from behavioral store: %s", exc)
+            result["pattern_error"] = str(exc)
+
+    # Learning signal stats (feedback count, co-retrieval, channel credits)
+    try:
+        from superlocalmemory.learning.signals import LearningSignals
+        signals = LearningSignals(str(LEARNING_DB))
+        result["signal_stats"] = signals.get_signal_stats(active_profile)
+    except Exception as exc:
+        logger.debug("Signal stats unavailable: %s", exc)
+
+    # Graph intelligence contribution to learning (v3.4.1)
+    try:
+        import sqlite3 as _sqlite3
+        from superlocalmemory.server.routes.helpers import DB_PATH
+        if DB_PATH.exists():
+            conn = _sqlite3.connect(str(DB_PATH))
+            conn.row_factory = _sqlite3.Row
+            row = conn.execute(
+                "SELECT COUNT(*) AS cnt, COUNT(DISTINCT community_id) AS communities, "
+                "ROUND(AVG(pagerank_score), 4) AS avg_pagerank "
+                "FROM fact_importance WHERE profile_id = ?",
+                (active_profile,),
+            ).fetchone()
+            if row:
+                d = dict(row)
+                result["graph_intelligence"] = {
+                    "facts_analyzed": d.get("cnt", 0),
+                    "communities_detected": d.get("communities", 0),
+                    "avg_pagerank": float(d.get("avg_pagerank", 0) or 0),
+                }
+            conn.close()
+    except Exception:
+        pass
+
+    return result
+
+
 @router.post("/api/learning/backup")
 async def learning_backup():
     """Backup learning.db to a timestamped file."""

package/src/superlocalmemory/server/routes/memories.py

@@ -46,16 +46,20 @@ def _fetch_graph_data(
 ) -> tuple[list, list, list]:
     """Fetch graph nodes, links, clusters from V3 or V2 schema."""
     if use_v3:
-        # Recency-first: get the most recent nodes, then find their edges
+        # Recency-first: get the most recent nodes, then find their edges.
+        # LEFT JOIN fact_importance for graph metrics (v3.4.1 — additive only).
         cursor.execute("""
-            SELECT fact_id as id, content, fact_type as category,
-                   confidence as importance, session_id as project_name,
-                   created_at
-            FROM atomic_facts
-            WHERE profile_id = ? AND confidence >= ?
-            ORDER BY created_at DESC
+            SELECT af.fact_id as id, af.content, af.fact_type as category,
+                   af.confidence as importance, af.session_id as project_name,
+                   af.created_at,
+                   fi.pagerank_score, fi.community_id, fi.degree_centrality
+            FROM atomic_facts af
+            LEFT JOIN fact_importance fi
+                ON af.fact_id = fi.fact_id AND fi.profile_id = ?
+            WHERE af.profile_id = ? AND af.confidence >= ?
+            ORDER BY af.created_at DESC
             LIMIT ?
-        """, (profile, min_importance / 10.0, max_nodes))
+        """, (profile, profile, min_importance / 10.0, max_nodes))
         nodes = cursor.fetchall()
 
         node_ids = {n['id'] for n in nodes}
@@ -80,6 +84,13 @@ def _fetch_graph_data(
         for n in nodes:
             n['entities'] = []
             n['content_preview'] = _preview(n.get('content'))
+            # v3.4.1: Default graph metrics when fact_importance has no data
+            if n.get('pagerank_score') is None:
+                n['pagerank_score'] = 0.0
+            if n.get('community_id') is None:
+                n['community_id'] = 0
+            if n.get('degree_centrality') is None:
+                n['degree_centrality'] = 0.0
 
         # Filter edges to only those between displayed nodes
         node_ids = {n['id'] for n in nodes}
@@ -290,7 +301,7 @@ async def get_memories(
 @router.get("/api/graph")
 async def get_graph(
     request: Request,
-    max_nodes: int = Query(100, ge=10, le=500),
+    max_nodes: int = Query(100, ge=10, le=10000),
     min_importance: int = Query(1, ge=1, le=10),
 ):
     """Get knowledge graph data for D3.js force-directed visualization."""

package/src/superlocalmemory/server/routes/stats.py

@@ -64,14 +64,26 @@ async def get_stats():
                 pass
 
             total_clusters = 0
+            # v3.4.1: Use community_id from fact_importance (graph intelligence)
             try:
                 cursor.execute(
-                    "SELECT COUNT(DISTINCT scene_id) as total FROM scenes WHERE profile_id = ?",
+                    "SELECT COUNT(DISTINCT community_id) as total FROM fact_importance "
+                    "WHERE profile_id = ? AND community_id IS NOT NULL",
                     (active_profile,),
                 )
                 total_clusters = cursor.fetchone()['total']
             except Exception:
                 pass
+            # Fallback: V2 scenes table
+            if total_clusters == 0:
+                try:
+                    cursor.execute(
+                        "SELECT COUNT(DISTINCT scene_id) as total FROM scenes WHERE profile_id = ?",
+                        (active_profile,),
+                    )
+                    total_clusters = cursor.fetchone()['total']
+                except Exception:
+                    pass
             # Fallback: V2-migrated clusters stored as cluster_id on memories
             if total_clusters == 0:
                 try:
@@ -311,11 +323,21 @@ async def get_patterns():
                 from superlocalmemory.learning.behavioral import BehavioralPatternStore
                 store = BehavioralPatternStore(str(MEMORY_DIR / "learning.db"))
                 raw = store.get_patterns(profile_id=active_profile)
+                # v3.4.1: Map pattern_type to frontend-expected keys
+                type_map = {
+                    "tech_preference": "preference",
+                    "style": "style",
+                    "terminology": "terminology",
+                    "temporal": "workflow",
+                    "interest": "workflow",
+                    "workflow": "workflow",
+                }
                 grouped = defaultdict(list)
                 for p in raw:
                     meta = p.get("metadata", {})
-                    grouped[p["pattern_type"]].append({
-                        "pattern_type": p["pattern_type"],
+                    frontend_key = type_map.get(p.get("pattern_type", ""), "preference")
+                    grouped[frontend_key].append({
+                        "pattern_type": p.get("pattern_type", ""),
                         "key": meta.get("key", p.get("pattern_key", "")),
                         "value": meta.get("value", p.get("pattern_key", "")),
                         "confidence": p.get("confidence", 0),