superlocalmemory 3.4.48 → 3.4.51

package/src/superlocalmemory/core/pruning_engine.py

@@ -0,0 +1,216 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""SuperLocalMemory v3.4.5 — Graph Pruning Engine.
+
+Reduces graph edge count without losing meaningful connections.
+Strategies:
+  1. Chain collapse: A→B→C → remove B→C if A→B exists with higher weight
+  2. Garbage entity removal: remove edges connected to garbage entities
+  3. Low-activity edge decay: edges between entities not accessed in 90+ days
+
+CRITICAL RULE: NEVER delete atomic_facts. Only prune graph_edges.
+Edges are derivable from facts — they can be regenerated.
+Facts are the permanent record.
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime, timedelta, UTC
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from superlocalmemory.storage.database import DatabaseManager
+
+logger = logging.getLogger(__name__)
+
+# Thresholds
+LOW_ACTIVITY_DAYS = 90
+CHAIN_COLLAPSE_MIN_WEIGHT_RATIO = 0.8
+BATCH_SIZE = 500
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+def prune_graph(
+    db: DatabaseManager,
+    profile_id: str = "default",
+    dry_run: bool = False,
+) -> dict[str, int]:
+    """Prune graph edges using all strategies.
+
+    Returns counts of edges removed per strategy.
+    Safe to run repeatedly — idempotent.
+    """
+    stats = {
+        "chain_collapsed": 0,
+        "garbage_removed": 0,
+        "low_activity_decayed": 0,
+        "total_removed": 0,
+        "edges_before": 0,
+        "edges_after": 0,
+    }
+
+    # Count before
+    rows = db.execute(
+        "SELECT COUNT(*) as c FROM graph_edges", ()
+    )
+    stats["edges_before"] = rows[0]["c"] if rows else 0
+
+    # Strategy 1: Chain collapse
+    collapsed = _collapse_chains(db, profile_id, dry_run)
+    stats["chain_collapsed"] = collapsed
+
+    # Strategy 2: Garbage entity edges
+    garbage = _remove_garbage_edges(db, profile_id, dry_run)
+    stats["garbage_removed"] = garbage
+
+    # Strategy 3: Low-activity edge decay
+    decayed = _decay_low_activity_edges(db, profile_id, dry_run)
+    stats["low_activity_decayed"] = decayed
+
+    stats["total_removed"] = collapsed + garbage + decayed
+
+    # Count after
+    rows = db.execute(
+        "SELECT COUNT(*) as c FROM graph_edges", ()
+    )
+    stats["edges_after"] = rows[0]["c"] if rows else 0
+
+    if stats["total_removed"] > 0:
+        logger.info(
+            "Graph pruning: %d edges removed (%d → %d)",
+            stats["total_removed"], stats["edges_before"], stats["edges_after"],
+        )
+
+    return stats
+
+
+# ---------------------------------------------------------------------------
+# Strategy 1: Chain Collapse
+# ---------------------------------------------------------------------------
+
+def _collapse_chains(db: DatabaseManager, profile_id: str, dry_run: bool) -> int:
+    """Collapse redundant chain edges.
+
+    If A→B (weight=0.9) and B→C (weight=0.5), and A→C also exists,
+    remove B→C if A→C weight >= B→C weight * threshold.
+
+    This preserves the semantic connection (A→C is stronger) while
+    removing intermediate edges.
+    """
+    try:
+        rows = db.execute("""
+            SELECT ge1.source_id as a, ge1.target_id as b, ge1.weight as w_ab,
+                   ge2.source_id as b2, ge2.target_id as c, ge2.weight as w_bc,
+                   ge3.weight as w_ac
+            FROM graph_edges ge1
+            JOIN graph_edges ge2 ON ge1.target_id = ge2.source_id
+            LEFT JOIN graph_edges ge3 ON ge1.source_id = ge3.source_id
+                                       AND ge2.target_id = ge3.target_id
+            WHERE ge3.weight >= ge2.weight * ?
+            LIMIT ?
+        """, (CHAIN_COLLAPSE_MIN_WEIGHT_RATIO, BATCH_SIZE))
+    except Exception as exc:
+        logger.warning("Chain collapse query failed: %s", exc)
+        return 0
+
+    remove_ids = []
+    for row in rows:
+        b_id = row["b"]
+        c_id = row["c"]
+        # Remove B→C edge
+        if not dry_run:
+            db.execute(
+                "DELETE FROM graph_edges WHERE source_id = ? AND target_id = ?",
+                (b_id, c_id),
+            )
+        remove_ids.append((b_id, c_id))
+
+    if remove_ids and not dry_run:
+        logger.info("Chain collapse: removed %d edges", len(remove_ids))
+
+    return len(remove_ids)
+
+
+# ---------------------------------------------------------------------------
+# Strategy 2: Garbage Entity Edges
+# ---------------------------------------------------------------------------
+
+def _remove_garbage_edges(db: DatabaseManager, profile_id: str, dry_run: bool) -> int:
+    """Remove edges connected to garbage/blacklisted entities."""
+    try:
+        rows = db.execute("""
+            SELECT ge.source_id, ge.target_id
+            FROM graph_edges ge
+            WHERE ge.source_id IN (SELECT term FROM entity_blacklist)
+               OR ge.target_id IN (SELECT term FROM entity_blacklist)
+            LIMIT ?
+        """, (BATCH_SIZE,))
+    except Exception:
+        return 0
+
+    count = 0
+    for row in rows:
+        if not dry_run:
+            db.execute(
+                "DELETE FROM graph_edges WHERE source_id = ? AND target_id = ?",
+                (row["source_id"], row["target_id"]),
+            )
+        count += 1
+
+    if count and not dry_run:
+        logger.info("Garbage edges removed: %d", count)
+
+    return count
+
+
+# ---------------------------------------------------------------------------
+# Strategy 3: Low-Activity Edge Decay
+# ---------------------------------------------------------------------------
+
+def _decay_low_activity_edges(
+    db: DatabaseManager, profile_id: str, dry_run: bool,
+) -> int:
+    """Remove edges between entities not accessed in 90+ days.
+
+    Only removes edges where BOTH connected entities have no recent access.
+    """
+    cutoff = (datetime.now(UTC) - timedelta(days=LOW_ACTIVITY_DAYS)).isoformat()
+
+    try:
+        rows = db.execute("""
+            SELECT ge.source_id, ge.target_id
+            FROM graph_edges ge
+            WHERE ge.source_id NOT IN (
+                SELECT DISTINCT entity_id FROM fact_access_log
+                WHERE accessed_at >= ?
+            )
+            AND ge.target_id NOT IN (
+                SELECT DISTINCT entity_id FROM fact_access_log
+                WHERE accessed_at >= ?
+            )
+            LIMIT ?
+        """, (cutoff, cutoff, BATCH_SIZE))
+    except Exception:
+        return 0
+
+    count = 0
+    for row in rows:
+        if not dry_run:
+            db.execute(
+                "DELETE FROM graph_edges WHERE source_id = ? AND target_id = ?",
+                (row["source_id"], row["target_id"]),
+            )
+        count += 1
+
+    if count and not dry_run:
+        logger.info("Low-activity edges decayed: %d", count)
+
+    return count

package/src/superlocalmemory/core/recall_pipeline.py

@@ -284,15 +284,27 @@ def apply_adaptive_ranking(
     from superlocalmemory.learning.ranker import AdaptiveRanker
     ranker = AdaptiveRanker(signal_count=signal_count)
 
+    from datetime import UTC, datetime as _dt
+    _now = _dt.now(UTC)
+
     result_dicts = []
     for r in response.results:
+        _age = 0.0
+        _created = getattr(r.fact, "created_at", None)
+        if _created:
+            try:
+                _age = max(0.0, (_now - _dt.fromisoformat(
+                    _created.replace("Z", "+00:00")
+                )).total_seconds() / 86400.0)
+            except (ValueError, TypeError):
+                pass
         result_dicts.append({
             "score": r.score,
             "cross_encoder_score": r.score,
             "trust_score": r.trust_score,
             "channel_scores": r.channel_scores or {},
             "fact": {
-                "age_days": 0,
+                "age_days": _age,
                 "access_count": r.fact.access_count,
             },
             "_original": r,
@@ -364,8 +376,20 @@ def apply_v2_adaptive_ranking(
         )
 
         # Build result-dict shape expected by the ranker's rerank() path.
+        from datetime import UTC, datetime as _dt
+        _now_v2 = _dt.now(UTC)
+
         result_dicts: list[dict] = []
         for r in response.results:
+            _age_v2 = 0.0
+            _created_v2 = getattr(r.fact, "created_at", None)
+            if _created_v2:
+                try:
+                    _age_v2 = max(0.0, (_now_v2 - _dt.fromisoformat(
+                        _created_v2.replace("Z", "+00:00")
+                    )).total_seconds() / 86400.0)
+                except (ValueError, TypeError):
+                    pass
             result_dicts.append({
                 "fact_id": r.fact.fact_id,
                 "score": r.score,
@@ -373,7 +397,7 @@ def apply_v2_adaptive_ranking(
                 "trust_score": r.trust_score,
                 "channel_scores": r.channel_scores or {},
                 "fact": {
-                    "age_days": 0,
+                    "age_days": _age_v2,
                     "access_count": r.fact.access_count,
                 },
                 "_original": r,

package/src/superlocalmemory/core/store_pipeline.py

@@ -498,6 +498,8 @@ def run_store_fact_direct(
         )
         fact.canonical_entities = list(canonical.values())
     db.store_fact(fact)
+    # v3.4.5: Incremental sync to CozoDB/LanceDB (F-04)
+    _sync_to_graph_backends(fact)
     if fact.embedding and ann_index:
         ann_index.add(fact.fact_id, fact.embedding)
     # V3.2: VectorStore upsert (dual-write)
@@ -567,3 +569,22 @@ def run_close_session(
         session_id, count, len(session_facts),
     )
     return count
+
+
+# ---------------------------------------------------------------------------
+# v3.4.5: Incremental sync to CozoDB/LanceDB (F-04)
+# ---------------------------------------------------------------------------
+
+def _sync_to_graph_backends(fact: Any) -> None:
+    """Sync a newly stored fact to CozoDB/LanceDB.
+
+    Non-blocking, best-effort. Called after SQLite write.
+    Failures are logged, not raised — SQLite is already committed.
+    """
+    try:
+        from superlocalmemory.core.backend_orchestrator import get_orchestrator
+        orch = get_orchestrator()
+        if orch is not None:
+            orch.sync_new_fact(fact)
+    except Exception:
+        pass  # Best-effort — daemon may not have initialized yet

package/src/superlocalmemory/core/tier_manager.py

@@ -323,3 +323,127 @@ def _demote_tier(
             )
 
     return len(demoted_ids)
+
+
+# ---------------------------------------------------------------------------
+# Hot-path access recording (v3.4.5 — Sprint 1)
+# ---------------------------------------------------------------------------
+
+# In-memory counter for batch-flush access recording.
+# Thread-safe. Flushed to SQLite every _ACCESS_FLUSH_THRESHOLD records
+# or _ACCESS_FLUSH_SECONDS, whichever comes first.
+import threading as _threading
+from datetime import datetime as _datetime, timedelta as _timedelta
+
+_pending_accesses: dict[str, int] = {}
+_access_lock = _threading.Lock()
+_last_access_flush = _datetime.now()
+
+_ACCESS_FLUSH_THRESHOLD = 100
+_ACCESS_FLUSH_SECONDS = 60
+
+
+def record_access_batch(db: DatabaseManager, fact_ids: list[str]) -> None:
+    """Hot-path access recording for recall. Must be < 1ms average.
+
+    Uses in-memory counter, batch-flushes to SQLite every
+    _ACCESS_FLUSH_THRESHOLD accesses or _ACCESS_FLUSH_SECONDS.
+    Promotes cold/archived facts on access.
+
+    Thread-safe — called from multiple MCP connections.
+    """
+    global _last_access_flush
+    if not fact_ids:
+        return
+
+    with _access_lock:
+        for fid in fact_ids:
+            _pending_accesses[fid] = _pending_accesses.get(fid, 0) + 1
+
+        now = _datetime.now()
+        should_flush = (
+            len(_pending_accesses) >= _ACCESS_FLUSH_THRESHOLD
+            or (now - _last_access_flush).total_seconds() > _ACCESS_FLUSH_SECONDS
+        )
+        if should_flush:
+            _flush_access_batch(db)
+            _last_access_flush = now
+
+    # Promote cold/archived facts on access (F-13: promote to warm)
+    try:
+        promote_on_access_batch(db, fact_ids)
+    except Exception:
+        pass
+
+
+def _flush_access_batch(db: DatabaseManager) -> None:
+    """Batch UPDATE access_count_30d and access_count in SQLite."""
+    if not _pending_accesses:
+        return
+    try:
+        for fid, count in list(_pending_accesses.items()):
+            db.execute(
+                "UPDATE atomic_facts SET access_count_30d = access_count_30d + ?, "
+                "access_count = access_count + ? WHERE fact_id = ?",
+                (count, count, fid),
+            )
+        _pending_accesses.clear()
+    except Exception as exc:
+        logger.warning("Failed to flush access counts: %s", exc)
+
+
+# ---------------------------------------------------------------------------
+# 30-day access window reset (F-14)
+# ---------------------------------------------------------------------------
+
+def reset_access_count_30d(db: DatabaseManager, profile_id: str = "default") -> int:
+    """Reset access_count_30d using actual 30-day window from fact_access_log.
+
+    Called during nightly rebalance. Returns number of facts updated.
+    """
+    try:
+        db.execute(
+            "UPDATE atomic_facts SET access_count_30d = COALESCE(("
+            "  SELECT COUNT(*) FROM fact_access_log "
+            "  WHERE fact_id = atomic_facts.fact_id "
+            "  AND accessed_at >= datetime('now', '-30 days')"
+            "), 0) WHERE profile_id = ?",
+            (profile_id,),
+        )
+        # Can't get rowcount reliably after subquery UPDATE
+        return -1
+    except Exception as exc:
+        logger.warning("Failed to reset access_count_30d: %s", exc)
+        return 0
+
+
+# ---------------------------------------------------------------------------
+# Backend sync stubs (v3.4.5 — Sprint 2+)
+# ---------------------------------------------------------------------------
+
+_cozo_backend: object | None = None
+_lancedb_backend: object | None = None
+
+
+def set_backends(cozo: object | None = None, lancedb: object | None = None) -> None:
+    """Register CozoDB/LanceDB backends for tier sync. Called by BackendOrchestrator."""
+    global _cozo_backend, _lancedb_backend
+    _cozo_backend = cozo
+    _lancedb_backend = lancedb
+
+
+def _sync_tiers_to_backends(
+    added: list[str], removed: list[str], db: DatabaseManager,
+) -> None:
+    """Sync tier changes to CozoDB/LanceDB. Non-fatal on failure."""
+    if _cozo_backend and hasattr(_cozo_backend, "sync_tier_changes"):
+        try:
+            _cozo_backend.sync_tier_changes(added=added, removed=removed)
+        except Exception as exc:
+            logger.warning("CozoDB tier sync failed: %s", exc)
+
+    if _lancedb_backend and hasattr(_lancedb_backend, "bulk_update_tiers_from_sqlite"):
+        try:
+            _lancedb_backend.bulk_update_tiers_from_sqlite(db.conn)
+        except Exception as exc:
+            logger.warning("LanceDB tier sync failed: %s", exc)

package/src/superlocalmemory/graph/__init__.py

@@ -0,0 +1,9 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Graph backends for SuperLocalMemory v3.4.5."""
+
+from superlocalmemory.graph.cozo_backend import CozoDBGraphBackend
+
+__all__ = ["CozoDBGraphBackend"]