superlocalmemory 3.4.10 → 3.4.12

package/src/superlocalmemory/storage/schema_v3411.py

@@ -0,0 +1,149 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""SuperLocalMemory V3.4.11 "Scale-Ready" — Schema Extensions.
+
+New tables for tiered storage + future backend integrations:
+  - pinned_facts: User-pinned facts that stay in hot tier forever
+  - backend_status: Tracks initialization state of LanceDB/KùzuDB backends
+  - fact_consolidations: History of merged/consolidated facts
+
+Design rules (inherited):
+  - CREATE IF NOT EXISTS for idempotency
+  - profile_id where applicable
+  - Never ALTER existing column types
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+"""
+
+from __future__ import annotations
+
+import logging
+import sqlite3
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# DDL — Pinned Facts (user override on lifecycle demotion)
+# ---------------------------------------------------------------------------
+
+_PINNED_FACTS_DDL = """
+CREATE TABLE IF NOT EXISTS pinned_facts (
+    fact_id TEXT PRIMARY KEY,
+    profile_id TEXT DEFAULT 'default',
+    pinned_at TEXT NOT NULL,
+    reason TEXT DEFAULT ''
+);
+
+CREATE INDEX IF NOT EXISTS idx_pinned_facts_profile
+    ON pinned_facts(profile_id);
+"""
+
+# ---------------------------------------------------------------------------
+# DDL — Backend Status (LanceDB, KùzuDB, sqlite-vec tracking)
+# ---------------------------------------------------------------------------
+
+_BACKEND_STATUS_DDL = """
+CREATE TABLE IF NOT EXISTS backend_status (
+    backend_name TEXT PRIMARY KEY,
+    status TEXT DEFAULT 'not_initialized',
+    record_count INTEGER DEFAULT 0,
+    last_sync_at TEXT,
+    error_message TEXT DEFAULT '',
+    config TEXT DEFAULT '{}'
+);
+"""
+
+# ---------------------------------------------------------------------------
+# DDL — Fact Consolidations (merge history)
+# ---------------------------------------------------------------------------
+
+_FACT_CONSOLIDATIONS_DDL = """
+CREATE TABLE IF NOT EXISTS fact_consolidations (
+    consolidation_id TEXT PRIMARY KEY,
+    profile_id TEXT DEFAULT 'default',
+    consolidated_fact_id TEXT NOT NULL,
+    source_fact_ids TEXT NOT NULL,
+    strategy TEXT DEFAULT 'entity_cluster',
+    created_at TEXT NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS idx_fact_consolidations_profile
+    ON fact_consolidations(profile_id);
+CREATE INDEX IF NOT EXISTS idx_fact_consolidations_target
+    ON fact_consolidations(consolidated_fact_id);
+"""
+
+# ---------------------------------------------------------------------------
+# DDL — Index on lifecycle for fast tier queries
+# ---------------------------------------------------------------------------
+
+_LIFECYCLE_INDEX_DDL = """
+CREATE INDEX IF NOT EXISTS idx_atomic_facts_lifecycle
+    ON atomic_facts(lifecycle, profile_id);
+"""
+
+
+# ---------------------------------------------------------------------------
+# Migration runner
+# ---------------------------------------------------------------------------
+
+def apply_v3411_schema(db_path: str | sqlite3.Connection) -> dict:
+    """Apply all v3.4.11 schema changes. Idempotent."""
+    result = {"applied": [], "errors": []}
+
+    if isinstance(db_path, sqlite3.Connection):
+        conn = db_path
+        own_connection = False
+    else:
+        conn = sqlite3.connect(str(db_path))
+        own_connection = True
+
+    try:
+        for name, ddl in [
+            ("pinned_facts", _PINNED_FACTS_DDL),
+            ("backend_status", _BACKEND_STATUS_DDL),
+            ("fact_consolidations", _FACT_CONSOLIDATIONS_DDL),
+            ("lifecycle_index", _LIFECYCLE_INDEX_DDL),
+        ]:
+            try:
+                conn.executescript(ddl)
+                result["applied"].append(name)
+            except sqlite3.OperationalError as e:
+                result["errors"].append(f"{name}: {e}")
+
+        # Seed default backend entries
+        now = __import__("datetime").datetime.now().isoformat()
+        for backend in ("sqlite_vec", "sqlite_graph", "lancedb", "kuzu"):
+            try:
+                conn.execute(
+                    "INSERT OR IGNORE INTO backend_status "
+                    "(backend_name, status, config) VALUES (?, ?, ?)",
+                    (backend, "not_initialized" if backend in ("lancedb", "kuzu") else "active", "{}"),
+                )
+            except sqlite3.OperationalError:
+                pass
+
+        # Mark version
+        try:
+            conn.execute(
+                "INSERT OR IGNORE INTO schema_version (version, applied_at) VALUES (?, ?)",
+                ("3.4.11", now),
+            )
+        except sqlite3.OperationalError:
+            pass
+
+        conn.commit()
+
+        if result["applied"]:
+            logger.info("Schema v3.4.11 applied: %s", ", ".join(result["applied"]))
+
+    except Exception as e:
+        result["errors"].append(f"fatal: {e}")
+        logger.error("Schema v3.4.11 migration failed: %s", e)
+    finally:
+        if own_connection:
+            conn.close()
+
+    return result

package/src/superlocalmemory/ui/index.html

@@ -1519,10 +1519,12 @@
                     <h5 class="mb-0"><i class="bi bi-hourglass-split text-warning"></i> Memory Lifecycle</h5>
                     <div>
                         <span class="badge bg-secondary me-2" id="lifecycle-profile-badge">default</span>
-                        <button class="btn btn-sm btn-outline-info" onclick="compactDryRun()"><i class="bi bi-funnel"></i> Preview Compaction</button>
+                        <button class="btn btn-sm btn-outline-success" onclick="evaluateTiersNow()"><i class="bi bi-arrow-repeat"></i> Evaluate Tiers</button>
+                        <button class="btn btn-sm btn-outline-info ms-1" onclick="compactDryRun()"><i class="bi bi-funnel"></i> Preview Compaction</button>
                         <button class="btn btn-sm btn-outline-warning ms-1" onclick="compactExecute()"><i class="bi bi-lightning"></i> Compact Now</button>
                     </div>
                 </div>
+
                 <!-- State Distribution -->
                 <div class="row g-3 mb-4" id="lifecycle-states-row">
                     <div class="col-md-2"><div class="card p-3 text-center"><div class="text-success fw-bold fs-3" id="lc-active-count">-</div><small class="text-muted">Active</small></div></div>
@@ -1741,6 +1743,7 @@
                         <div style="margin-top:8px">Loading skill performance data...</div>
                     </div>
                 </div>
+                <div id="skill-lineage-container" class="mt-3"></div>
                 <div id="skill-detail-panel" style="display:none;margin-top:24px"></div>
             </div>
 
@@ -2150,7 +2153,7 @@
     <script src="static/js/ng-health.js?v=345"></script>
     <script src="static/js/ng-ingestion.js?v=345"></script>
     <script src="static/js/ng-entities.js?v=3410"></script>
-    <script src="static/js/ng-skills.js?v=3410"></script>
+    <script src="static/js/ng-skills.js?v=3411"></script>
     <script src="static/js/ng-mesh.js?v=345"></script>
     <script src="static/js/ng-shell.js?v=3410"></script>
 

package/src/superlocalmemory/ui/js/lifecycle.js

@@ -7,10 +7,27 @@ var _lifecycleData = null;
 
 async function loadLifecycle() {
     try {
+        // V3.4.11: Load tier stats from the authoritative source (atomic_facts.lifecycle)
+        // and merge into the existing lifecycle view for a unified display.
+        var tierResponse = await fetch('/api/tiers/stats');
+        var tierData = await tierResponse.json();
+
         var response = await fetch('/api/lifecycle/status');
         var data = await response.json();
         _lifecycleData = data;
 
+        // Override state counts with tier stats (atomic_facts is the truth)
+        if (tierData && tierData.total > 0) {
+            data.states = data.states || {};
+            data.states.active = tierData.active;
+            data.states.warm = tierData.warm;
+            data.states.cold = tierData.cold;
+            data.states.archived = tierData.archived;
+            data.total_memories = tierData.total;
+            data._pinned = tierData.pinned;
+            data.available = true;
+        }
+
         if (!data.available) {
             showEmpty('lifecycle-states-row', 'hourglass-split', 'Lifecycle engine not available. Upgrade to v2.8.');
             return;
@@ -21,6 +38,19 @@ async function loadLifecycle() {
         renderLifecycleAgeStats(data);
         renderLifecycleTransitions(data);
 
+        // V3.4.11: Show pinned count below the state cards
+        var pinnedInfo = document.getElementById('lifecycle-pinned-info');
+        if (!pinnedInfo) {
+            pinnedInfo = document.createElement('div');
+            pinnedInfo.id = 'lifecycle-pinned-info';
+            pinnedInfo.className = 'small text-muted mb-3';
+            var statesRow = document.getElementById('lifecycle-states-row');
+            if (statesRow) statesRow.parentNode.insertBefore(pinnedInfo, statesRow.nextSibling);
+        }
+        var pinCount = data._pinned || 0;
+        pinnedInfo.innerHTML = '<i class="bi bi-pin-angle"></i> ' + pinCount + ' pinned facts (protected from demotion). ' +
+            '<span class="text-muted" style="font-size:11px;">Active = full weight | Warm = 0.7x | Cold = 0.3x | Archived = deep recall only</span>';
+
         var badge = document.getElementById('lifecycle-profile-badge');
         if (badge) badge.textContent = data.active_profile || 'default';
     } catch (error) {
@@ -296,3 +326,56 @@ async function compactExecute() {
         console.error('Compaction error:', e);
     }
 }
+
+
+// ---- V3.4.11: Tier Management (pinning + evaluation) ----
+
+async function evaluateTiersNow() {
+    showToast('Evaluating tiers...');
+    try {
+        var response = await fetch('/api/tiers/evaluate', { method: 'POST' });
+        var data = await response.json();
+        if (data.success) {
+            var s = data.stats;
+            var demoted = (s.demoted_to_warm || 0) + (s.demoted_to_cold || 0) + (s.demoted_to_archive || 0);
+            showToast('Tier evaluation: ' + demoted + ' facts demoted, ' + s.pinned_protected + ' pinned protected');
+            loadTierStats();
+            loadLifecycle();
+        }
+    } catch (error) {
+        showToast('Tier evaluation failed');
+    }
+}
+
+async function pinFact(factId, reason) {
+    try {
+        var response = await fetch('/api/tiers/pin', {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({ fact_id: factId, reason: reason || '' })
+        });
+        var data = await response.json();
+        if (data.success) {
+            showToast('Fact pinned');
+            loadTierStats();
+        }
+    } catch (error) {
+        showToast('Failed to pin fact');
+    }
+}
+
+async function unpinFact(factId) {
+    try {
+        var response = await fetch('/api/tiers/unpin', {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({ fact_id: factId })
+        });
+        if (response.ok) {
+            showToast('Fact unpinned');
+            loadTierStats();
+        }
+    } catch (error) {
+        showToast('Failed to unpin fact');
+    }
+}