superlocalmemory 3.3.29 → 3.4.1

package/src/superlocalmemory/server/routes/timeline.py

@@ -0,0 +1,252 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later — see LICENSE file
+# Part of SuperLocalMemory v3.4.1 | https://qualixar.com
+
+"""Memory Timeline API — unified event stream from 3 sources.
+
+Merges atomic_facts, temporal_events, and consolidation_log
+into a single time-ordered event list for D3 timeline visualization.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+import sqlite3
+from typing import Any
+
+from fastapi import APIRouter, Query
+from fastapi.responses import JSONResponse
+from superlocalmemory.server.routes.helpers import DB_PATH, get_active_profile
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/api/v3/timeline", tags=["timeline"])
+
+VALID_RANGES = {"1d", "7d", "30d", "90d", "365d"}
+VALID_GROUP_BY = {"category", "community"}
+INTERNAL_CEILING = 5000
+
+
+def _parse_range(range_str: str) -> str | None:
+    """Parse range like '7d' into SQLite modifier '-7 days'."""
+    m = re.match(r"^(\d+)d$", range_str)
+    if not m or range_str not in VALID_RANGES:
+        return None
+    return f"-{m.group(1)} days"
+
+
+@router.get("/")
+async def get_timeline(
+    range: str = "7d",
+    group_by: str = "category",
+    limit: int = Query(default=1000, ge=1, le=2000),
+    offset: int = Query(default=0, ge=0),
+    profile: str = "",
+):
+    """Get unified timeline events from all memory sources."""
+    modifier = _parse_range(range)
+    if modifier is None:
+        return JSONResponse(
+            {"error": f"Invalid range: '{range}'. Valid: {', '.join(sorted(VALID_RANGES))}"},
+            status_code=400,
+        )
+    if group_by not in VALID_GROUP_BY:
+        return JSONResponse(
+            {"error": f"Invalid group_by: '{group_by}'. Valid: {', '.join(sorted(VALID_GROUP_BY))}"},
+            status_code=400,
+        )
+
+    pid = profile or get_active_profile()
+    if not DB_PATH.exists():
+        return {"range": range, "group_by": group_by, "count": 0, "events": [], "total_available": 0, "offset": 0}
+
+    conn = sqlite3.connect(str(DB_PATH))
+    conn.row_factory = sqlite3.Row
+
+    try:
+        start_date = conn.execute("SELECT datetime('now', ?)", (modifier,)).fetchone()[0]
+    except Exception:
+        conn.close()
+        return {"range": range, "group_by": group_by, "count": 0, "events": [], "total_available": 0, "offset": 0}
+
+    events: list[dict] = []
+
+    # 1. Atomic facts
+    try:
+        rows = conn.execute(
+            "SELECT fact_id, content, fact_type, "
+            "REPLACE(created_at, ' ', 'T') || 'Z' AS created_at, "
+            "confidence, session_id "
+            "FROM atomic_facts "
+            "WHERE profile_id = ? AND created_at >= ? AND lifecycle = 'active' "
+            "ORDER BY created_at DESC LIMIT ?",
+            (pid, start_date, INTERNAL_CEILING),
+        ).fetchall()
+        for r in rows:
+            d = dict(r)
+            events.append({
+                "id": d["fact_id"],
+                "type": "fact_created",
+                "timestamp": d["created_at"],
+                "category": d.get("fact_type", "semantic"),
+                "community_id": None,
+                "content_preview": (d.get("content") or "")[:100],
+                "trust_score": None,
+                "lifecycle_zone": None,
+                "retention_score": None,
+                "session_id": d.get("session_id", ""),
+                "source": "atomic_facts",
+            })
+    except Exception as exc:
+        logger.debug("Timeline atomic_facts query failed: %s", exc)
+
+    # 2. Temporal events
+    try:
+        rows = conn.execute(
+            "SELECT event_id, entity_id, fact_id, "
+            "observation_date, referenced_date, interval_start, description, "
+            "REPLACE(COALESCE(observation_date, referenced_date, interval_start), ' ', 'T') || 'Z' "
+            "AS event_date "
+            "FROM temporal_events "
+            "WHERE profile_id = ? "
+            "AND COALESCE(observation_date, referenced_date, interval_start) >= ? "
+            "ORDER BY event_date DESC LIMIT ?",
+            (pid, start_date, INTERNAL_CEILING),
+        ).fetchall()
+        for r in rows:
+            d = dict(r)
+            events.append({
+                "id": d.get("event_id", ""),
+                "type": "temporal_event",
+                "timestamp": d.get("event_date", ""),
+                "category": "temporal",
+                "community_id": None,
+                "content_preview": (d.get("description") or "")[:100],
+                "trust_score": None,
+                "lifecycle_zone": None,
+                "retention_score": None,
+                "session_id": None,
+                "source": "temporal_events",
+            })
+    except Exception as exc:
+        logger.debug("Timeline temporal_events query failed: %s", exc)
+
+    # 3. Consolidation log
+    try:
+        rows = conn.execute(
+            "SELECT action_id, action_type, new_fact_id, existing_fact_id, reason, "
+            "REPLACE(timestamp, ' ', 'T') || 'Z' AS timestamp "
+            "FROM consolidation_log "
+            "WHERE profile_id = ? AND timestamp >= ? "
+            "ORDER BY timestamp DESC LIMIT ?",
+            (pid, start_date, INTERNAL_CEILING),
+        ).fetchall()
+        for r in rows:
+            d = dict(r)
+            preview = f"{d.get('action_type', '')}: {(d.get('reason') or '')[:80]}"
+            events.append({
+                "id": d.get("action_id", ""),
+                "type": "consolidation",
+                "timestamp": d.get("timestamp", ""),
+                "category": "consolidation",
+                "community_id": None,
+                "content_preview": preview[:100],
+                "trust_score": None,
+                "lifecycle_zone": None,
+                "retention_score": None,
+                "session_id": None,
+                "source": "consolidation_log",
+            })
+    except Exception as exc:
+        logger.debug("Timeline consolidation_log query failed: %s", exc)
+
+    # Sort merged events by timestamp desc
+    events.sort(key=lambda e: e.get("timestamp", ""), reverse=True)
+    total_available = len(events)
+
+    # Paginate
+    events = events[offset:offset + limit]
+
+    # Enrich with community_id if group_by=community
+    fact_ids = [e["id"] for e in events if e["source"] == "atomic_facts"]
+    if group_by == "community" and fact_ids:
+        _enrich_communities(conn, pid, fact_ids, events)
+
+    # Enrich with trust scores + retention
+    if fact_ids:
+        _enrich_trust(conn, pid, fact_ids, events)
+        _enrich_retention(conn, pid, fact_ids, events)
+
+    conn.close()
+
+    return {
+        "range": range,
+        "group_by": group_by,
+        "total_available": total_available,
+        "count": len(events),
+        "offset": offset,
+        "events": events,
+    }
+
+
+def _enrich_communities(
+    conn: sqlite3.Connection, pid: str, fact_ids: list[str], events: list[dict],
+) -> None:
+    """Add community_id from fact_importance."""
+    try:
+        placeholders = ",".join("?" * len(fact_ids))
+        sql = (
+            "SELECT fact_id, community_id, pagerank_score "
+            "FROM fact_importance WHERE profile_id = ? AND fact_id IN ("
+            + placeholders + ")"
+        )
+        rows = conn.execute(sql, (pid, *fact_ids)).fetchall()
+        comm_map = {dict(r)["fact_id"]: dict(r).get("community_id") for r in rows}
+        for e in events:
+            if e["id"] in comm_map:
+                e["community_id"] = comm_map[e["id"]]
+    except Exception:
+        pass
+
+
+def _enrich_trust(
+    conn: sqlite3.Connection, pid: str, fact_ids: list[str], events: list[dict],
+) -> None:
+    """Add trust_score from trust_scores table."""
+    try:
+        placeholders = ",".join("?" * len(fact_ids))
+        sql = (
+            "SELECT target_id AS fact_id, trust_score "
+            "FROM trust_scores WHERE profile_id = ? AND target_type = 'fact' "
+            "AND target_id IN (" + placeholders + ")"
+        )
+        rows = conn.execute(sql, (pid, *fact_ids)).fetchall()
+        trust_map = {dict(r)["fact_id"]: round(float(dict(r).get("trust_score", 0)), 3) for r in rows}
+        for e in events:
+            if e["id"] in trust_map:
+                e["trust_score"] = trust_map[e["id"]]
+    except Exception:
+        pass
+
+
+def _enrich_retention(
+    conn: sqlite3.Connection, pid: str, fact_ids: list[str], events: list[dict],
+) -> None:
+    """Add lifecycle_zone + retention_score from fact_retention."""
+    try:
+        placeholders = ",".join("?" * len(fact_ids))
+        sql = (
+            "SELECT fact_id, lifecycle_zone, retention_score "
+            "FROM fact_retention WHERE profile_id = ? AND fact_id IN ("
+            + placeholders + ")"
+        )
+        rows = conn.execute(sql, (pid, *fact_ids)).fetchall()
+        ret_map = {dict(r)["fact_id"]: dict(r) for r in rows}
+        for e in events:
+            d = ret_map.get(e["id"])
+            if d:
+                e["lifecycle_zone"] = d.get("lifecycle_zone")
+                e["retention_score"] = round(float(d.get("retention_score", 0)), 3)
+    except Exception:
+        pass

package/src/superlocalmemory/server/routes/v3_api.py

@@ -1035,6 +1035,17 @@ async def trigger_consolidation(request: Request):
         engine = ConsolidationEngine(db=db, config=config.consolidation, slm_config=config)
         result = engine.consolidate(profile_id=pid, lightweight=lightweight)
 
+        # v3.4.1: Auto-trigger behavioral pattern mining after consolidation
+        try:
+            from superlocalmemory.learning.consolidation_worker import ConsolidationWorker
+            learning_db = config.base_dir / "learning.db"
+            cw = ConsolidationWorker(str(config.db_path), str(learning_db))
+            pattern_count = cw._generate_patterns(pid, False)
+            result["patterns_mined"] = pattern_count
+            logger.info("Auto-mined %d patterns after consolidation", pattern_count)
+        except Exception as exc:
+            logger.debug("Pattern mining after consolidation failed: %s", exc)
+
         return {"success": True, **result}
     except Exception as e:
         return JSONResponse({"error": str(e)}, status_code=500)
@@ -1561,6 +1572,156 @@ async def process_health(request: Request):
 
 # ── 1g. GET /api/v3/v33/overview ─────────────────────────────
 
+# ── v3.4.1: Graph Communities ──────────────────────────────────
+
+@router.get("/graph/communities")
+async def get_graph_communities(request: Request, profile: str = ""):
+    """Get community assignments with TF-IDF labels, entities, and colors.
+
+    v3.4.1: Uses TF-IDF labels from config table (computed by GraphAnalyzer
+    at consolidation time). Falls back to inline word frequency if labels
+    not yet computed.
+    """
+    try:
+        from superlocalmemory.server.routes.helpers import get_active_profile, DB_PATH
+        import sqlite3
+        pid = profile or get_active_profile()
+
+        if not DB_PATH.exists():
+            return {"communities": [], "total": 0}
+
+        conn = sqlite3.connect(str(DB_PATH))
+        conn.row_factory = sqlite3.Row
+
+        # Get community member counts and average pagerank
+        comm_rows = conn.execute(
+            "SELECT community_id, COUNT(*) AS member_count, "
+            "AVG(pagerank_score) AS pagerank_avg "
+            "FROM fact_importance "
+            "WHERE profile_id = ? AND community_id IS NOT NULL "
+            "GROUP BY community_id ORDER BY member_count DESC",
+            (pid,),
+        ).fetchall()
+
+        if not comm_rows:
+            conn.close()
+            return {"communities": [], "total": 0}
+
+        # Load TF-IDF labels from config table
+        tfidf_labels = {}
+        try:
+            label_key = "community_labels_" + pid
+            label_row = conn.execute(
+                "SELECT value FROM config WHERE key = ?",
+                (label_key,),
+            ).fetchone()
+            if label_row:
+                import json as _json
+                tfidf_labels = _json.loads(dict(label_row)["value"])
+        except Exception:
+            pass
+
+        cluster_colors = [
+            '#667eea', '#764ba2', '#43e97b', '#38f9d7',
+            '#4facfe', '#00f2fe', '#f093fb', '#f5576c',
+            '#fa709a', '#fee140', '#30cfd0', '#330867',
+        ]
+
+        communities = []
+        for row in comm_rows:
+            d = dict(row)
+            comm_id = d["community_id"]
+
+            # Use TF-IDF label if available, else generate inline
+            label = tfidf_labels.get(str(comm_id), "")
+            if not label:
+                # Inline fallback: word frequency from top facts
+                try:
+                    fact_rows = conn.execute(
+                        "SELECT af.content FROM fact_importance fi "
+                        "JOIN atomic_facts af ON fi.fact_id = af.fact_id "
+                        "WHERE fi.profile_id = ? AND fi.community_id = ? "
+                        "ORDER BY fi.pagerank_score DESC LIMIT 20",
+                        (pid, comm_id),
+                    ).fetchall()
+                    from collections import defaultdict as _ddict
+                    wf = _ddict(int)
+                    sw = {"the", "a", "an", "is", "was", "are", "to", "of",
+                          "in", "for", "on", "with", "at", "by", "from",
+                          "and", "but", "or", "not", "it", "this", "that",
+                          "i", "we", "they", "he", "she", "you", "my"}
+                    for fr in fact_rows:
+                        for w in (dict(fr).get("content", "")).lower().split():
+                            w = w.strip(".,;:!?\"'()[]{}")
+                            if len(w) > 2 and w not in sw:
+                                wf[w] += 1
+                    top = sorted(wf.items(), key=lambda x: x[1], reverse=True)[:3]
+                    label = ", ".join(w for w, _ in top) if top else f"Community {comm_id}"
+                except Exception:
+                    label = f"Community {comm_id}"
+
+            # Get top entities from canonical_entities_json
+            top_entities = []
+            try:
+                ent_rows = conn.execute(
+                    "SELECT af.canonical_entities_json FROM fact_importance fi "
+                    "JOIN atomic_facts af ON fi.fact_id = af.fact_id "
+                    "WHERE fi.profile_id = ? AND fi.community_id = ? "
+                    "ORDER BY fi.pagerank_score DESC LIMIT 10",
+                    (pid, comm_id),
+                ).fetchall()
+                import json as _json2
+                entity_counts: dict = {}
+                for er in ent_rows:
+                    raw = dict(er).get("canonical_entities_json", "")
+                    if raw:
+                        try:
+                            for ent in _json2.loads(raw):
+                                entity_counts[ent] = entity_counts.get(ent, 0) + 1
+                        except (ValueError, TypeError):
+                            pass
+                top_entities = sorted(
+                    entity_counts, key=entity_counts.get, reverse=True,
+                )[:5]
+            except Exception:
+                pass
+
+            communities.append({
+                "community_id": comm_id,
+                "label": label,
+                "member_count": d["member_count"],
+                "top_entities": top_entities,
+                "color": cluster_colors[comm_id % len(cluster_colors)],
+                "pagerank_avg": round(float(d["pagerank_avg"] or 0), 4),
+            })
+
+        conn.close()
+        return {"communities": communities, "total": len(communities)}
+
+    except Exception as e:
+        return JSONResponse({"error": str(e)}, status_code=500)
+
+
+@router.post("/graph/run-communities")
+async def run_community_detection(request: Request):
+    """Trigger community detection manually (runs GraphAnalyzer)."""
+    try:
+        from superlocalmemory.server.routes.helpers import get_active_profile, DB_PATH
+        from superlocalmemory.storage.database import DatabaseManager
+        from superlocalmemory.storage import schema as _schema
+        from superlocalmemory.core.graph_analyzer import GraphAnalyzer
+
+        pid = get_active_profile()
+        db = DatabaseManager(DB_PATH)
+        db.initialize(_schema)
+
+        analyzer = GraphAnalyzer(db)
+        result = analyzer.compute_and_store(pid)
+        return {"success": True, **result}
+    except Exception as e:
+        return JSONResponse({"error": str(e)}, status_code=500)
+
+
 @router.get("/v33/overview")
 async def v33_overview(request: Request, profile: str = ""):
     """Get SLM 3.3 feature overview -- all new capabilities at a glance."""

package/src/superlocalmemory/server/ui.py

@@ -152,6 +152,14 @@ def create_app() -> FastAPI:
     from superlocalmemory.server.routes.v3_api import router as v3_router
     application.include_router(v3_router)
 
+    # v3.4.1: Chat SSE + Insights + Timeline endpoints
+    for _module_name_v341 in ("chat",):
+        try:
+            _mod_v341 = __import__(f"superlocalmemory.server.routes.{_module_name_v341}", fromlist=["router"])
+            application.include_router(_mod_v341.router)
+        except (ImportError, Exception) as _exc:
+            logger.warning("Optional router %s failed: %s", _module_name_v341, _exc)
+
     # Graceful optional routers
     for _module_name in ("learning", "lifecycle", "behavioral", "compliance"):
         try: