superlocalmemory 3.8.0 → 3.8.1

package/src/superlocalmemory/server/routes/stats.py

@@ -4,21 +4,78 @@
 """SuperLocalMemory V3 - Stats Routes
  - AGPL-3.0-or-later
 
-Routes: /api/stats, /api/timeline, /api/patterns
+Routes: /api/stats, /api/timeline
 """
-import json
 import logging
-from collections import defaultdict
+import sqlite3
 from typing import Optional
 
 from fastapi import APIRouter, HTTPException, Query
 
-from .helpers import get_db_connection, dict_factory, get_active_profile, DB_PATH, MEMORY_DIR
+from .helpers import get_db_connection, dict_factory, get_active_profile, DB_PATH
 
 logger = logging.getLogger("superlocalmemory.routes.stats")
 router = APIRouter()
 
 
+def _query_ingestion_sources(cursor, profile_id: str) -> list[dict]:
+    """Query profile-scoped ingestion provenance without double-counting facts."""
+    cursor.execute(
+        """
+        SELECT COALESCE(NULLIF(TRIM(source_type), ''), 'unknown') AS source_type,
+               COUNT(DISTINCT fact_id) AS count
+        FROM provenance
+        WHERE profile_id = ?
+        GROUP BY COALESCE(NULLIF(TRIM(source_type), ''), 'unknown')
+        ORDER BY count DESC, source_type ASC
+        """,
+        (profile_id,),
+    )
+    return [dict(row) for row in cursor.fetchall()]
+
+
+def _load_ingestion_sources_with_status(
+    cursor, profile_id: str,
+) -> tuple[list[dict], dict]:
+    """Return provenance plus a truthful availability state.
+
+    Older databases may not have the provenance table.  In that case the
+    dashboard must report an empty/unknown state rather than inventing a split.
+    Lock or I/O failures are distinct from a legitimate legacy-empty state.
+    """
+    try:
+        return _query_ingestion_sources(cursor, profile_id), {
+            "available": True,
+            "state": "recorded",
+            "source": "memory.db:provenance",
+        }
+    except sqlite3.OperationalError as exc:
+        if "no such table" in str(exc).lower():
+            return [], {
+                "available": True,
+                "state": "legacy_not_recorded",
+                "source": "memory.db:provenance",
+            }
+        logger.warning("provenance metrics temporarily unavailable: %s", exc)
+        return [], {
+            "available": False,
+            "state": "temporarily_unavailable",
+            "source": "memory.db:provenance",
+        }
+    except sqlite3.Error as exc:
+        logger.warning("provenance metrics unavailable: %s", exc)
+        return [], {
+            "available": False,
+            "state": "temporarily_unavailable",
+            "source": "memory.db:provenance",
+        }
+
+
+def _load_ingestion_sources(cursor, profile_id: str) -> list[dict]:
+    """Compatibility helper returning only the provenance rows."""
+    return _load_ingestion_sources_with_status(cursor, profile_id)[0]
+
+
 def _internal_error(detail: str = "Internal server error") -> HTTPException:
     """SEC-H-02: log full traceback server-side; return a generic message to the client."""
     logger.exception("stats route error")
@@ -26,7 +83,7 @@ def _internal_error(detail: str = "Internal server error") -> HTTPException:
 
 
 @router.get("/api/stats")
-async def get_stats():
+def get_stats():
     """Get comprehensive system statistics."""
     try:
         conn = get_db_connection()
@@ -205,6 +262,9 @@ async def get_stats():
             importance_dist = cursor.fetchall()
 
         db_size = DB_PATH.stat().st_size if DB_PATH.exists() else 0
+        ingestion_sources, ingestion_sources_status = (
+            _load_ingestion_sources_with_status(cursor, active_profile)
+        )
 
         if total_graph_nodes > 1:
             max_edges = (total_graph_nodes * (total_graph_nodes - 1)) / 2
@@ -241,6 +301,8 @@ async def get_stats():
                     if total_graph_nodes > 0 else 0
                 ),
             },
+            "ingestion_sources": ingestion_sources,
+            "ingestion_sources_status": ingestion_sources_status,
         }
 
     except Exception:
@@ -248,11 +310,12 @@ async def get_stats():
 
 
 @router.get("/api/timeline")
-async def get_timeline(
+def get_timeline(
     days: int = Query(30, ge=1, le=365),
     group_by: str = Query("day", pattern="^(day|week|month)$"),
+    include_categories: bool = Query(True),
 ):
-    """Get temporal view of memory creation with flexible grouping."""
+    """Get temporal memory creation; dashboard callers may skip category scans."""
     try:
         conn = get_db_connection()
         conn.row_factory = dict_factory
@@ -286,22 +349,29 @@ async def get_timeline(
         """, (days, active_profile))
         timeline = cursor.fetchall()
 
-        cursor.execute(f"""
-            SELECT {date_group} as period, {cat_col} as category, COUNT(*) as count
-            FROM {table}
-            WHERE created_at >= datetime('now', '-' || ? || ' days')
-              AND {cat_col} IS NOT NULL AND {profile_col} = ?
-            GROUP BY {date_group}, {cat_col} ORDER BY period DESC, count DESC
-        """, (days, active_profile))
-        category_trend = cursor.fetchall()
-
-        cursor.execute(f"""
-            SELECT COUNT(*) as total_memories,
-                   COUNT(DISTINCT {cat_col}) as categories_used
-            FROM {table}
-            WHERE created_at >= datetime('now', '-' || ? || ' days') AND {profile_col} = ?
-        """, (days, active_profile))
-        period_stats = cursor.fetchone()
+        if include_categories:
+            cursor.execute(f"""
+                SELECT {date_group} as period, {cat_col} as category, COUNT(*) as count
+                FROM {table}
+                WHERE created_at >= datetime('now', '-' || ? || ' days')
+                  AND {cat_col} IS NOT NULL AND {profile_col} = ?
+                GROUP BY {date_group}, {cat_col} ORDER BY period DESC, count DESC
+            """, (days, active_profile))
+            category_trend = cursor.fetchall()
+
+            cursor.execute(f"""
+                SELECT COUNT(*) as total_memories,
+                       COUNT(DISTINCT {cat_col}) as categories_used
+                FROM {table}
+                WHERE created_at >= datetime('now', '-' || ? || ' days') AND {profile_col} = ?
+            """, (days, active_profile))
+            period_stats = cursor.fetchone()
+        else:
+            category_trend = []
+            period_stats = {
+                "total_memories": sum(int(row["count"] or 0) for row in timeline),
+                "categories_used": None,
+            }
 
         conn.close()
 
@@ -313,135 +383,3 @@ async def get_timeline(
 
     except Exception:
         raise _internal_error("Timeline error")
-
-
-@router.get("/api/patterns")
-async def get_patterns():
-    """Get learned patterns."""
-    try:
-        conn = get_db_connection()
-        conn.row_factory = dict_factory
-        cursor = conn.cursor()
-        active_profile = get_active_profile()
-
-        # Check for V3 learning tables or V2 identity_patterns
-        patterns = []
-        table_name = None
-        for candidate in ('learned_patterns', 'identity_patterns'):
-            cursor.execute(
-                "SELECT name FROM sqlite_master WHERE type='table' AND name=?",
-                (candidate,),
-            )
-            if cursor.fetchone():
-                table_name = candidate
-                break
-
-        if not table_name:
-            conn.close()
-            # Fall through to V3.1 behavioral pattern store
-            try:
-                from superlocalmemory.learning.behavioral import BehavioralPatternStore
-                store = BehavioralPatternStore(str(MEMORY_DIR / "learning.db"))
-                raw = store.get_patterns(profile_id=active_profile)
-                # v3.4.7: Map all pattern types to frontend categories
-                type_map = {
-                    "tech_preference": "preference",
-                    "interest": "preference",
-                    "entity_preferences": "preference",
-                    "style": "style",
-                    "fact_type_distribution": "style",
-                    "knowledge_structure": "style",
-                    "terminology": "terminology",
-                    "temporal": "workflow",
-                    "session_activity": "workflow",
-                    "workflow": "workflow",
-                    "co_retrieval_clusters": "workflow",
-                    "channel_performance": "performance",
-                }
-                grouped = defaultdict(list)
-                for p in raw:
-                    meta = p.get("metadata", {})
-                    data = meta  # metadata IS the data dict
-                    frontend_key = type_map.get(p.get("pattern_type", ""), "preference")
-                    # Extract human-readable value from data fields
-                    readable_value = (
-                        data.get("value")
-                        or data.get("topic")
-                        or data.get("pattern_key", "")
-                        or p.get("pattern_key", "")
-                    )
-                    readable_key = (
-                        data.get("pattern_key")
-                        or data.get("key")
-                        or p.get("pattern_key", "")
-                    )
-                    grouped[frontend_key].append({
-                        "pattern_type": p.get("pattern_type", ""),
-                        "key": readable_key,
-                        "value": readable_value,
-                        "confidence": p.get("confidence", 0),
-                        "evidence_count": data.get("evidence", p.get("evidence_count", 0)),
-                    })
-                all_patterns = [p for ps in grouped.values() for p in ps]
-                confs = [p["confidence"] for p in all_patterns if p.get("confidence")]
-                return {
-                    "patterns": dict(grouped),
-                    "total_patterns": len(all_patterns),
-                    "pattern_types": list(grouped.keys()),
-                    "confidence_stats": {
-                        "avg": sum(confs) / len(confs) if confs else 0,
-                        "min": min(confs) if confs else 0,
-                        "max": max(confs) if confs else 0,
-                    },
-                }
-            except Exception:
-                return {
-                    "patterns": {}, "total_patterns": 0, "pattern_types": [],
-                    "message": "Pattern learning not initialized.",
-                }
-
-        if table_name == 'identity_patterns':
-            cursor.execute("""
-                SELECT pattern_type, key, value, confidence, evidence_count,
-                       updated_at as last_updated
-                FROM identity_patterns WHERE profile = ?
-                ORDER BY confidence DESC, evidence_count DESC
-            """, (active_profile,))
-        else:
-            cursor.execute("""
-                SELECT pattern_type, key, value, confidence, evidence_count,
-                       last_updated
-                FROM learned_patterns WHERE is_active = 1
-                ORDER BY confidence DESC, evidence_count DESC
-            """)
-
-        patterns = cursor.fetchall()
-
-        for pattern in patterns:
-            if pattern.get('value'):
-                try:
-                    pattern['value'] = json.loads(pattern['value'])
-                except Exception:
-                    pass
-
-        grouped = defaultdict(list)
-        for pattern in patterns:
-            grouped[pattern['pattern_type']].append(pattern)
-
-        confidences = [p['confidence'] for p in patterns if p.get('confidence')]
-        confidence_stats = {
-            "avg": sum(confidences) / len(confidences) if confidences else 0,
-            "min": min(confidences) if confidences else 0,
-            "max": max(confidences) if confidences else 0,
-        }
-
-        conn.close()
-
-        return {
-            "patterns": dict(grouped), "total_patterns": len(patterns),
-            "pattern_types": list(grouped.keys()), "confidence_stats": confidence_stats,
-        }
-
-    except Exception:
-        logger.exception("patterns route error")
-        return {"patterns": {}, "total_patterns": 0, "error": "Internal server error"}

package/src/superlocalmemory/server/routes/token.py

@@ -31,21 +31,11 @@ logger = logging.getLogger(__name__)
 router = APIRouter(tags=["internal"])
 
 
-_ALLOWED_ORIGIN_PREFIXES = (
-    "http://127.0.0.1",
-    "https://127.0.0.1",
-    "http://localhost",
-    "https://localhost",
-    "http://[::1]",
-    "https://[::1]",
-)
-
-
 def _origin_is_loopback(origin: str) -> bool:
     """Return True iff ``origin`` is absent or a loopback URL."""
-    if not origin:
-        return True
-    return any(origin.startswith(p) for p in _ALLOWED_ORIGIN_PREFIXES)
+    from superlocalmemory.server.origin import origin_is_loopback
+
+    return origin_is_loopback(origin)
 
 
 @router.get("/internal/token")

package/src/superlocalmemory/server/routes/v3_api.py

@@ -52,6 +52,42 @@ async def _apply_runtime_config(request: Request, config, *, mode_change: bool)
     authorization.complete()
 
 
+def _require_manage(request: Request) -> None:
+    """Use one explicit authorization boundary for dashboard configuration."""
+    from superlocalmemory.server.rbac_enforce import require_manage
+
+    require_manage(request)
+
+
+def _resolve_profile(request: Request, requested: str = "") -> str:
+    """Resolve a profile override and enforce READ on its actual target."""
+    from superlocalmemory.access.rbac import Permission
+    from superlocalmemory.server.rbac_enforce import require_permission
+    from superlocalmemory.server.routes.helpers import get_active_profile
+
+    if requested and not isinstance(requested, str):
+        raise HTTPException(status_code=422, detail="profile must be a string")
+    profile = requested.strip() if requested else get_active_profile()
+    require_permission(request, Permission.READ, profile=profile)
+    return profile
+
+
+def _resolve_mutation_profile(requested: object) -> str:
+    """Resolve a body-supplied profile without silently coercing bad input."""
+    from superlocalmemory.server.routes.helpers import get_active_profile
+
+    if requested is not None and not isinstance(requested, str):
+        raise HTTPException(status_code=422, detail="profile must be a string")
+    return requested.strip() if requested and requested.strip() else get_active_profile()
+
+
+def _require_manage_for_profile(request: Request, profile: str) -> None:
+    """Enforce MANAGE on the resolved mutation target before any side effect."""
+    from superlocalmemory.server.rbac_enforce import require_manage
+
+    require_manage(request, profile=profile)
+
+
 # ── Dashboard ────────────────────────────────────────────────
 
 @router.get("/dashboard")
@@ -448,9 +484,9 @@ async def set_scope_config(request: Request):
 @router.post("/embedding/test")
 async def test_embedding_endpoint(request: Request):
     """Test connectivity to a custom embedding endpoint."""
+    _require_manage(request)
     try:
         import httpx
-        from urllib.parse import urlparse
         body = await request.json()
         endpoint = body.get("api_endpoint", "").rstrip("/")
         model = body.get("model_name", "test")
@@ -459,12 +495,10 @@ async def test_embedding_endpoint(request: Request):
         if not endpoint:
             return JSONResponse({"error": "No endpoint provided"}, status_code=400)
 
-        parsed = urlparse(endpoint)
-        if parsed.scheme not in ("http", "https"):
-            return JSONResponse({"error": "Only http/https endpoints supported"}, status_code=400)
-        host = parsed.hostname or ""
-        if host in ("169.254.169.254", "metadata.google.internal"):
-            return JSONResponse({"error": "Cloud metadata endpoints not allowed"}, status_code=400)
+        client_host = request.client.host if request.client else ""
+        error = _validate_provider_url(endpoint, client_host)
+        if error:
+            return JSONResponse({"error": error}, status_code=400)
 
         if not endpoint.endswith("/embeddings"):
             endpoint = f"{endpoint}/embeddings"
@@ -980,6 +1014,7 @@ async def get_auto_capture_config():
 @router.put("/auto-capture/config")
 async def set_auto_capture_config(request: Request):
     """Update auto-capture config. Body: {"enabled": true, "capture_decisions": true, ...}"""
+    _require_manage(request)
     try:
         body = await request.json()
         from superlocalmemory.hooks.rules_engine import RulesEngine
@@ -1010,6 +1045,7 @@ async def get_auto_recall_config():
 @router.put("/auto-recall/config")
 async def set_auto_recall_config(request: Request):
     """Update auto-recall config."""
+    _require_manage(request)
     try:
         body = await request.json()
         from superlocalmemory.hooks.rules_engine import RulesEngine
@@ -1096,6 +1132,7 @@ async def run_consolidation(request: Request):
 
         config = SLMConfig.load()
         learning_db = DB_PATH.parent / "learning.db"
+        _require_manage_for_profile(request, config.active_profile)
         authorization = authorize_route_mutation(
             request,
             operation="update",
@@ -1213,6 +1250,7 @@ async def set_auto_invoke_config(request: Request):
 
     Body: {"enabled": true, "min_score": 0.15, "weights": {...}}
     """
+    _require_manage(request)
     try:
         body = await request.json()
 
@@ -1254,10 +1292,10 @@ async def get_associations(
     profile: str = "",
 ):
     """Get association edges for a profile with content previews."""
+    pid = _resolve_profile(request, profile)
     try:
         from superlocalmemory.server.routes.helpers import get_active_profile, DB_PATH
         import sqlite3
-        pid = profile or get_active_profile()
 
         if not DB_PATH.exists():
             return {"edges": [], "total": 0}
@@ -1321,10 +1359,10 @@ async def get_associations(
 @router.get("/associations/stats")
 async def get_association_stats(request: Request, profile: str = ""):
     """Get association graph statistics."""
+    pid = _resolve_profile(request, profile)
     try:
         from superlocalmemory.server.routes.helpers import get_active_profile, DB_PATH
         import sqlite3
-        pid = profile or get_active_profile()
 
         if not DB_PATH.exists():
             return {
@@ -1422,12 +1460,12 @@ async def get_association_stats(request: Request, profile: str = ""):
 @router.get("/consolidation/status")
 async def get_consolidation_status(request: Request, profile: str = ""):
     """Get consolidation status and last run results."""
+    pid = _resolve_profile(request, profile)
     try:
         from superlocalmemory.server.routes.helpers import get_active_profile, DB_PATH
         from superlocalmemory.core.config import SLMConfig
         import sqlite3
 
-        pid = profile or get_active_profile()
         config = SLMConfig.load()
         cons_cfg = config.consolidation
 
@@ -1520,7 +1558,8 @@ async def trigger_consolidation(request: Request):
         profile = body.get("profile", "")
 
         from superlocalmemory.server.routes.helpers import get_active_profile
-        pid = profile or get_active_profile()
+        pid = _resolve_mutation_profile(profile)
+        _require_manage_for_profile(request, pid)
         authorization = authorize_route_mutation(
             request,
             operation="update",
@@ -1590,10 +1629,10 @@ async def trigger_consolidation(request: Request):
 @router.get("/core-memory")
 async def get_core_memory(request: Request, profile: str = ""):
     """Get all Core Memory blocks for a profile."""
+    pid = _resolve_profile(request, profile)
     try:
         from superlocalmemory.server.routes.helpers import get_active_profile, DB_PATH
         import sqlite3
-        pid = profile or get_active_profile()
 
         if not DB_PATH.exists():
             return {"blocks": [], "total_chars": 0, "char_limit": 2000}
@@ -1719,6 +1758,7 @@ async def update_core_memory_block(block_id: str, request: Request):
 @router.get("/vector-store/status")
 async def get_vector_store_status(request: Request, profile: str = ""):
     """Get VectorStore health and statistics."""
+    pid = _resolve_profile(request, profile)
     try:
         from superlocalmemory.core.config import SLMConfig
         from superlocalmemory.server.routes.helpers import DB_PATH
@@ -1749,7 +1789,8 @@ async def get_vector_store_status(request: Request, profile: str = ""):
             try:
                 conn = sqlite3.connect(str(DB_PATH))
                 count = conn.execute(
-                    "SELECT COUNT(*) FROM embedding_metadata"
+                    "SELECT COUNT(*) FROM embedding_metadata WHERE profile_id = ?",
+                    (pid,),
                 ).fetchone()[0]
                 result["total_vectors"] = count
                 conn.close()
@@ -1779,10 +1820,10 @@ async def get_vector_store_status(request: Request, profile: str = ""):
 @router.get("/forgetting/stats")
 async def forgetting_stats(request: Request, profile: str = ""):
     """Get memory retention zone distribution."""
+    pid = _resolve_profile(request, profile)
     try:
         from superlocalmemory.server.routes.helpers import get_active_profile, DB_PATH
         import sqlite3 as _sqlite3
-        pid = profile or get_active_profile()
 
         zones = {"active": 0, "warm": 0, "cold": 0, "archive": 0, "forgotten": 0}
         total = 0
@@ -1830,7 +1871,8 @@ async def run_forgetting(request: Request):
 
         from superlocalmemory.server.routes.helpers import get_active_profile, DB_PATH
         import sqlite3 as _sqlite3
-        pid = profile or get_active_profile()
+        pid = _resolve_mutation_profile(profile)
+        _require_manage_for_profile(request, pid)
 
         if not DB_PATH.exists():
             return {"success": False, "error": "Database not found"}
@@ -1907,10 +1949,10 @@ async def run_forgetting(request: Request):
 @router.get("/quantization/stats")
 async def quantization_stats(request: Request, profile: str = ""):
     """Get embedding quantization tier distribution."""
+    pid = _resolve_profile(request, profile)
     try:
         from superlocalmemory.server.routes.helpers import get_active_profile, DB_PATH
         import sqlite3 as _sqlite3
-        pid = profile or get_active_profile()
 
         tiers = {"float32": 0, "int8": 0, "polar4": 0, "polar2": 0}
         total = 0
@@ -1970,10 +2012,10 @@ async def quantization_stats(request: Request, profile: str = ""):
 @router.get("/ccq/blocks")
 async def ccq_blocks(request: Request, profile: str = "", limit: int = 50):
     """Get CCQ consolidated blocks."""
+    pid = _resolve_profile(request, profile)
     try:
         from superlocalmemory.server.routes.helpers import get_active_profile, DB_PATH
         import sqlite3 as _sqlite3
-        pid = profile or get_active_profile()
 
         if not DB_PATH.exists():
             return {"blocks": [], "total": 0}
@@ -2030,10 +2072,10 @@ async def ccq_blocks(request: Request, profile: str = "", limit: int = 50):
 @router.get("/soft-prompts")
 async def get_soft_prompts(request: Request, profile: str = ""):
     """Get active soft prompt templates."""
+    pid = _resolve_profile(request, profile)
     try:
         from superlocalmemory.server.routes.helpers import get_active_profile, DB_PATH
         import sqlite3 as _sqlite3
-        pid = profile or get_active_profile()
 
         if not DB_PATH.exists():
             return {"prompts": [], "total": 0, "total_tokens": 0}
@@ -2083,6 +2125,7 @@ async def get_soft_prompts(request: Request, profile: str = ""):
 @router.get("/health/processes")
 async def process_health(request: Request):
     """Get SLM process health status."""
+    _resolve_profile(request)
     try:
         import os as _os
 
@@ -2137,10 +2180,10 @@ async def get_graph_communities(request: Request, profile: str = ""):
     at consolidation time). Falls back to inline word frequency if labels
     not yet computed.
     """
+    pid = _resolve_profile(request, profile)
     try:
         from superlocalmemory.server.routes.helpers import get_active_profile, DB_PATH
         import sqlite3
-        pid = profile or get_active_profile()
 
         if not DB_PATH.exists():
             return {"communities": [], "total": 0}
@@ -2267,6 +2310,7 @@ async def run_community_detection(request: Request):
         from superlocalmemory.core.graph_analyzer import GraphAnalyzer
 
         pid = get_active_profile()
+        _require_manage_for_profile(request, pid)
         authorization = authorize_route_mutation(
             request,
             operation="update",
@@ -2289,10 +2333,10 @@ async def run_community_detection(request: Request):
 @router.get("/v33/overview")
 async def v33_overview(request: Request, profile: str = ""):
     """Get SLM 3.3 feature overview -- all new capabilities at a glance."""
+    pid = _resolve_profile(request, profile)
     try:
         from superlocalmemory.server.routes.helpers import get_active_profile, DB_PATH
         import sqlite3 as _sqlite3
-        pid = profile or get_active_profile()
 
         overview: dict = {
             "version": "3.3",