superlocalmemory 3.7.8 → 3.8.1

package/src/superlocalmemory/server/routes/stats.py

@@ -4,23 +4,86 @@
 """SuperLocalMemory V3 - Stats Routes
  - AGPL-3.0-or-later
 
-Routes: /api/stats, /api/timeline, /api/patterns
+Routes: /api/stats, /api/timeline
 """
-import json
 import logging
-from collections import defaultdict
+import sqlite3
 from typing import Optional
 
 from fastapi import APIRouter, HTTPException, Query
 
-from .helpers import get_db_connection, dict_factory, get_active_profile, DB_PATH, MEMORY_DIR
+from .helpers import get_db_connection, dict_factory, get_active_profile, DB_PATH
 
 logger = logging.getLogger("superlocalmemory.routes.stats")
 router = APIRouter()
 
 
+def _query_ingestion_sources(cursor, profile_id: str) -> list[dict]:
+    """Query profile-scoped ingestion provenance without double-counting facts."""
+    cursor.execute(
+        """
+        SELECT COALESCE(NULLIF(TRIM(source_type), ''), 'unknown') AS source_type,
+               COUNT(DISTINCT fact_id) AS count
+        FROM provenance
+        WHERE profile_id = ?
+        GROUP BY COALESCE(NULLIF(TRIM(source_type), ''), 'unknown')
+        ORDER BY count DESC, source_type ASC
+        """,
+        (profile_id,),
+    )
+    return [dict(row) for row in cursor.fetchall()]
+
+
+def _load_ingestion_sources_with_status(
+    cursor, profile_id: str,
+) -> tuple[list[dict], dict]:
+    """Return provenance plus a truthful availability state.
+
+    Older databases may not have the provenance table.  In that case the
+    dashboard must report an empty/unknown state rather than inventing a split.
+    Lock or I/O failures are distinct from a legitimate legacy-empty state.
+    """
+    try:
+        return _query_ingestion_sources(cursor, profile_id), {
+            "available": True,
+            "state": "recorded",
+            "source": "memory.db:provenance",
+        }
+    except sqlite3.OperationalError as exc:
+        if "no such table" in str(exc).lower():
+            return [], {
+                "available": True,
+                "state": "legacy_not_recorded",
+                "source": "memory.db:provenance",
+            }
+        logger.warning("provenance metrics temporarily unavailable: %s", exc)
+        return [], {
+            "available": False,
+            "state": "temporarily_unavailable",
+            "source": "memory.db:provenance",
+        }
+    except sqlite3.Error as exc:
+        logger.warning("provenance metrics unavailable: %s", exc)
+        return [], {
+            "available": False,
+            "state": "temporarily_unavailable",
+            "source": "memory.db:provenance",
+        }
+
+
+def _load_ingestion_sources(cursor, profile_id: str) -> list[dict]:
+    """Compatibility helper returning only the provenance rows."""
+    return _load_ingestion_sources_with_status(cursor, profile_id)[0]
+
+
+def _internal_error(detail: str = "Internal server error") -> HTTPException:
+    """SEC-H-02: log full traceback server-side; return a generic message to the client."""
+    logger.exception("stats route error")
+    return HTTPException(status_code=500, detail=detail)
+
+
 @router.get("/api/stats")
-async def get_stats():
+def get_stats():
     """Get comprehensive system statistics."""
     try:
         conn = get_db_connection()
@@ -199,6 +262,9 @@ async def get_stats():
             importance_dist = cursor.fetchall()
 
         db_size = DB_PATH.stat().st_size if DB_PATH.exists() else 0
+        ingestion_sources, ingestion_sources_status = (
+            _load_ingestion_sources_with_status(cursor, active_profile)
+        )
 
         if total_graph_nodes > 1:
             max_edges = (total_graph_nodes * (total_graph_nodes - 1)) / 2
@@ -235,18 +301,21 @@ async def get_stats():
                     if total_graph_nodes > 0 else 0
                 ),
             },
+            "ingestion_sources": ingestion_sources,
+            "ingestion_sources_status": ingestion_sources_status,
         }
 
-    except Exception as e:
-        raise HTTPException(status_code=500, detail=f"Stats error: {str(e)}")
+    except Exception:
+        raise _internal_error("Stats error")
 
 
 @router.get("/api/timeline")
-async def get_timeline(
+def get_timeline(
     days: int = Query(30, ge=1, le=365),
     group_by: str = Query("day", pattern="^(day|week|month)$"),
+    include_categories: bool = Query(True),
 ):
-    """Get temporal view of memory creation with flexible grouping."""
+    """Get temporal memory creation; dashboard callers may skip category scans."""
     try:
         conn = get_db_connection()
         conn.row_factory = dict_factory
@@ -280,22 +349,29 @@ async def get_timeline(
         """, (days, active_profile))
         timeline = cursor.fetchall()
 
-        cursor.execute(f"""
-            SELECT {date_group} as period, {cat_col} as category, COUNT(*) as count
-            FROM {table}
-            WHERE created_at >= datetime('now', '-' || ? || ' days')
-              AND {cat_col} IS NOT NULL AND {profile_col} = ?
-            GROUP BY {date_group}, {cat_col} ORDER BY period DESC, count DESC
-        """, (days, active_profile))
-        category_trend = cursor.fetchall()
-
-        cursor.execute(f"""
-            SELECT COUNT(*) as total_memories,
-                   COUNT(DISTINCT {cat_col}) as categories_used
-            FROM {table}
-            WHERE created_at >= datetime('now', '-' || ? || ' days') AND {profile_col} = ?
-        """, (days, active_profile))
-        period_stats = cursor.fetchone()
+        if include_categories:
+            cursor.execute(f"""
+                SELECT {date_group} as period, {cat_col} as category, COUNT(*) as count
+                FROM {table}
+                WHERE created_at >= datetime('now', '-' || ? || ' days')
+                  AND {cat_col} IS NOT NULL AND {profile_col} = ?
+                GROUP BY {date_group}, {cat_col} ORDER BY period DESC, count DESC
+            """, (days, active_profile))
+            category_trend = cursor.fetchall()
+
+            cursor.execute(f"""
+                SELECT COUNT(*) as total_memories,
+                       COUNT(DISTINCT {cat_col}) as categories_used
+                FROM {table}
+                WHERE created_at >= datetime('now', '-' || ? || ' days') AND {profile_col} = ?
+            """, (days, active_profile))
+            period_stats = cursor.fetchone()
+        else:
+            category_trend = []
+            period_stats = {
+                "total_memories": sum(int(row["count"] or 0) for row in timeline),
+                "categories_used": None,
+            }
 
         conn.close()
 
@@ -305,136 +381,5 @@ async def get_timeline(
             "parameters": {"days": days, "group_by": group_by},
         }
 
-    except Exception as e:
-        raise HTTPException(status_code=500, detail=f"Timeline error: {str(e)}")
-
-
-@router.get("/api/patterns")
-async def get_patterns():
-    """Get learned patterns."""
-    try:
-        conn = get_db_connection()
-        conn.row_factory = dict_factory
-        cursor = conn.cursor()
-        active_profile = get_active_profile()
-
-        # Check for V3 learning tables or V2 identity_patterns
-        patterns = []
-        table_name = None
-        for candidate in ('learned_patterns', 'identity_patterns'):
-            cursor.execute(
-                "SELECT name FROM sqlite_master WHERE type='table' AND name=?",
-                (candidate,),
-            )
-            if cursor.fetchone():
-                table_name = candidate
-                break
-
-        if not table_name:
-            conn.close()
-            # Fall through to V3.1 behavioral pattern store
-            try:
-                from superlocalmemory.learning.behavioral import BehavioralPatternStore
-                store = BehavioralPatternStore(str(MEMORY_DIR / "learning.db"))
-                raw = store.get_patterns(profile_id=active_profile)
-                # v3.4.7: Map all pattern types to frontend categories
-                type_map = {
-                    "tech_preference": "preference",
-                    "interest": "preference",
-                    "entity_preferences": "preference",
-                    "style": "style",
-                    "fact_type_distribution": "style",
-                    "knowledge_structure": "style",
-                    "terminology": "terminology",
-                    "temporal": "workflow",
-                    "session_activity": "workflow",
-                    "workflow": "workflow",
-                    "co_retrieval_clusters": "workflow",
-                    "channel_performance": "performance",
-                }
-                grouped = defaultdict(list)
-                for p in raw:
-                    meta = p.get("metadata", {})
-                    data = meta  # metadata IS the data dict
-                    frontend_key = type_map.get(p.get("pattern_type", ""), "preference")
-                    # Extract human-readable value from data fields
-                    readable_value = (
-                        data.get("value")
-                        or data.get("topic")
-                        or data.get("pattern_key", "")
-                        or p.get("pattern_key", "")
-                    )
-                    readable_key = (
-                        data.get("pattern_key")
-                        or data.get("key")
-                        or p.get("pattern_key", "")
-                    )
-                    grouped[frontend_key].append({
-                        "pattern_type": p.get("pattern_type", ""),
-                        "key": readable_key,
-                        "value": readable_value,
-                        "confidence": p.get("confidence", 0),
-                        "evidence_count": data.get("evidence", p.get("evidence_count", 0)),
-                    })
-                all_patterns = [p for ps in grouped.values() for p in ps]
-                confs = [p["confidence"] for p in all_patterns if p.get("confidence")]
-                return {
-                    "patterns": dict(grouped),
-                    "total_patterns": len(all_patterns),
-                    "pattern_types": list(grouped.keys()),
-                    "confidence_stats": {
-                        "avg": sum(confs) / len(confs) if confs else 0,
-                        "min": min(confs) if confs else 0,
-                        "max": max(confs) if confs else 0,
-                    },
-                }
-            except Exception:
-                return {
-                    "patterns": {}, "total_patterns": 0, "pattern_types": [],
-                    "message": "Pattern learning not initialized.",
-                }
-
-        if table_name == 'identity_patterns':
-            cursor.execute("""
-                SELECT pattern_type, key, value, confidence, evidence_count,
-                       updated_at as last_updated
-                FROM identity_patterns WHERE profile = ?
-                ORDER BY confidence DESC, evidence_count DESC
-            """, (active_profile,))
-        else:
-            cursor.execute("""
-                SELECT pattern_type, key, value, confidence, evidence_count,
-                       last_updated
-                FROM learned_patterns WHERE is_active = 1
-                ORDER BY confidence DESC, evidence_count DESC
-            """)
-
-        patterns = cursor.fetchall()
-
-        for pattern in patterns:
-            if pattern.get('value'):
-                try:
-                    pattern['value'] = json.loads(pattern['value'])
-                except Exception:
-                    pass
-
-        grouped = defaultdict(list)
-        for pattern in patterns:
-            grouped[pattern['pattern_type']].append(pattern)
-
-        confidences = [p['confidence'] for p in patterns if p.get('confidence')]
-        confidence_stats = {
-            "avg": sum(confidences) / len(confidences) if confidences else 0,
-            "min": min(confidences) if confidences else 0,
-            "max": max(confidences) if confidences else 0,
-        }
-
-        conn.close()
-
-        return {
-            "patterns": dict(grouped), "total_patterns": len(patterns),
-            "pattern_types": list(grouped.keys()), "confidence_stats": confidence_stats,
-        }
-
-    except Exception as e:
-        return {"patterns": {}, "total_patterns": 0, "error": str(e)}
+    except Exception:
+        raise _internal_error("Timeline error")

package/src/superlocalmemory/server/routes/tiers.py

@@ -20,7 +20,7 @@ from pydantic import BaseModel, Field
 
 from superlocalmemory.server.route_mutations import authorize_route_mutation
 
-from .helpers import DB_PATH
+from .helpers import DB_PATH, get_active_profile
 
 logger = logging.getLogger("superlocalmemory.routes.tiers")
 router = APIRouter()
@@ -55,9 +55,11 @@ def _validate_profile(profile_id: str) -> str:
 
 
 @router.get("/api/tiers/stats")
-async def tier_stats(profile_id: str = "default"):
+async def tier_stats(profile_id: str | None = None):
     """Get tier distribution stats."""
-    profile_id = _validate_profile(profile_id)
+    # Default to the ACTIVE profile, never the literal "default" — otherwise
+    # tier counts reflect the default profile regardless of which is active.
+    profile_id = _validate_profile(profile_id or get_active_profile())
 
     with _db() as conn:
         try:
@@ -98,13 +100,13 @@ async def tier_stats(profile_id: str = "default"):
 
 
 @router.post("/api/tiers/evaluate")
-async def evaluate_tiers_route(request: Request, profile_id: str = "default"):
+async def evaluate_tiers_route(request: Request, profile_id: str | None = None):
     """Manually trigger tier evaluation.
 
     Uses the shared engine (via lazy import) instead of re-initializing
     DatabaseManager on every request.
     """
-    profile_id = _validate_profile(profile_id)
+    profile_id = _validate_profile(profile_id or get_active_profile())
 
     try:
         authorization = authorize_route_mutation(
@@ -137,13 +139,13 @@ async def evaluate_tiers_route(request: Request, profile_id: str = "default"):
 async def pin_fact_route(
     body: PinRequest,
     request: Request,
-    profile_id: str = "default",
+    profile_id: str | None = None,
 ):
     """Pin a fact to stay in active tier forever.
 
     Validates fact exists in the specified profile before pinning.
     """
-    profile_id = _validate_profile(profile_id)
+    profile_id = _validate_profile(profile_id or get_active_profile())
     authorization = authorize_route_mutation(
         request,
         operation="update",
@@ -194,7 +196,7 @@ async def pin_fact_route(
 async def unpin_fact_route(
     body: PinRequest,
     request: Request,
-    profile_id: str = "default",
+    profile_id: str | None = None,
 ):
     """Unpin a fact, allowing normal tier demotion.
 
@@ -202,7 +204,7 @@ async def unpin_fact_route(
     based on access patterns. This is intentional — immediate demotion would
     surprise the user.
     """
-    profile_id = _validate_profile(profile_id)
+    profile_id = _validate_profile(profile_id or get_active_profile())
     authorization = authorize_route_mutation(
         request,
         operation="update",

package/src/superlocalmemory/server/routes/token.py

@@ -31,21 +31,11 @@ logger = logging.getLogger(__name__)
 router = APIRouter(tags=["internal"])
 
 
-_ALLOWED_ORIGIN_PREFIXES = (
-    "http://127.0.0.1",
-    "https://127.0.0.1",
-    "http://localhost",
-    "https://localhost",
-    "http://[::1]",
-    "https://[::1]",
-)
-
-
 def _origin_is_loopback(origin: str) -> bool:
     """Return True iff ``origin`` is absent or a loopback URL."""
-    if not origin:
-        return True
-    return any(origin.startswith(p) for p in _ALLOWED_ORIGIN_PREFIXES)
+    from superlocalmemory.server.origin import origin_is_loopback
+
+    return origin_is_loopback(origin)
 
 
 @router.get("/internal/token")