superlocalmemory 3.4.30__tar.gz → 3.4.32__tar.gz

{superlocalmemory-3.4.30/src/superlocalmemory.egg-info → superlocalmemory-3.4.32}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: superlocalmemory
-Version: 3.4.30
+Version: 3.4.32
 Summary: Information-geometric agent memory with mathematical guarantees
 Author-email: Varun Pratap Bhardwaj <admin@superlocalmemory.com>
 License: AGPL-3.0-or-later

{superlocalmemory-3.4.30 → superlocalmemory-3.4.32}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "superlocalmemory"
-version = "3.4.30"
+version = "3.4.32"
 description = "Information-geometric agent memory with mathematical guarantees"
 readme = "README.md"
 license = {text = "AGPL-3.0-or-later"}

{superlocalmemory-3.4.30 → superlocalmemory-3.4.32}/src/superlocalmemory/__init__.py

@@ -1,3 +1,3 @@
 """SuperLocalMemory — information-geometric agent memory."""
 
-__version__ = "3.4.30"
+__version__ = "3.4.32"

{superlocalmemory-3.4.30 → superlocalmemory-3.4.32}/src/superlocalmemory/cli/pending_store.py

@@ -24,12 +24,21 @@ License: Elastic-2.0
 from __future__ import annotations
 
 import json
+import os
 import sqlite3
 import time
 from pathlib import Path
 
-_DEFAULT_DIR = Path.home() / ".superlocalmemory"
+
+def _default_dir() -> Path:
+    """Honor SLM_DATA_DIR so tests can isolate via tmp_path."""
+    return Path(os.environ.get("SLM_DATA_DIR") or Path.home() / ".superlocalmemory")
+
+
 _PENDING_DB = "pending.db"
+_MAX_RETRIES = 3
+_STUCK_DAYS = 7
+_DEAD_LETTER_DAYS = 30
 
 _SCHEMA = """
 CREATE TABLE IF NOT EXISTS pending_memories (
@@ -47,7 +56,7 @@ CREATE TABLE IF NOT EXISTS pending_memories (
 
 def _get_db(base_dir: Path | None = None) -> sqlite3.Connection:
     """Open pending.db with WAL mode. Creates if needed."""
-    d = base_dir or _DEFAULT_DIR
+    d = base_dir or _default_dir()
     d.mkdir(parents=True, exist_ok=True)
     db_path = d / _PENDING_DB
     conn = sqlite3.connect(str(db_path), timeout=5)
@@ -128,7 +137,7 @@ def mark_failed(row_id: int, error: str, base_dir: Path | None = None) -> None:
 
 def pending_count(base_dir: Path | None = None) -> int:
     """Count unprocessed pending memories."""
-    d = base_dir or _DEFAULT_DIR
+    d = base_dir or _default_dir()
     db_path = d / _PENDING_DB
     if not db_path.exists():
         return 0
@@ -156,3 +165,46 @@ def cleanup_done(days: int = 7, base_dir: Path | None = None) -> int:
         return cur.rowcount
     finally:
         conn.close()
+
+
+def cleanup_stale(base_dir: Path | None = None) -> dict[str, int]:
+    """Sweep stale rows from pending.db. Runs periodically from the daemon.
+
+    Removes:
+    - `done` rows older than 7 days (already processed)
+    - `failed` rows that exceeded max retries (moved to dead-letter via deletion)
+    - `pending` rows stuck more than 7 days (test pollution, crashed workers)
+    - Everything older than 30 days regardless of status (hard cap)
+    """
+    conn = _get_db(base_dir)
+    try:
+        done = conn.execute(
+            "DELETE FROM pending_memories WHERE status = 'done' "
+            "AND created_at < datetime('now', ?)",
+            (f"-{_STUCK_DAYS} days",),
+        ).rowcount
+        failed = conn.execute(
+            "DELETE FROM pending_memories WHERE status = 'failed' "
+            "AND retry_count >= ?",
+            (_MAX_RETRIES,),
+        ).rowcount
+        stuck = conn.execute(
+            "DELETE FROM pending_memories WHERE status = 'pending' "
+            "AND created_at < datetime('now', ?)",
+            (f"-{_STUCK_DAYS} days",),
+        ).rowcount
+        hard_cap = conn.execute(
+            "DELETE FROM pending_memories "
+            "WHERE created_at < datetime('now', ?)",
+            (f"-{_DEAD_LETTER_DAYS} days",),
+        ).rowcount
+        conn.commit()
+        return {
+            "done": done,
+            "failed_over_retries": failed,
+            "stuck_pending": stuck,
+            "hard_cap_expired": hard_cap,
+            "total": done + failed + stuck + hard_cap,
+        }
+    finally:
+        conn.close()

{superlocalmemory-3.4.30 → superlocalmemory-3.4.32}/src/superlocalmemory/core/maintenance_scheduler.py

@@ -116,6 +116,14 @@ class MaintenanceScheduler:
         except Exception as exc:
             logger.debug("Auto-backup check skipped: %s", exc)
 
+        try:
+            from superlocalmemory.cli.pending_store import cleanup_stale
+            stats = cleanup_stale()
+            if stats["total"] > 0:
+                logger.info("Pending cleanup: %s", stats)
+        except Exception as exc:
+            logger.debug("Pending cleanup skipped: %s", exc)
+
         self._schedule_next()
 
     def _sync_cloud_destinations(self, manager: object) -> None:

superlocalmemory-3.4.32/src/superlocalmemory/core/recall_gate.py

@@ -0,0 +1,36 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""v3.4.32: Recall-in-flight counter used to give /search priority over the
+pending materializer.
+
+Every recall handler calls ``begin_recall()`` on entry and ``end_recall()``
+in a finally block. The pending-memory materializer thread polls
+``in_flight()`` and sleeps while any recall is active, so the shared
+embedder worker never serves a materialization ahead of a user-initiated
+recall.
+"""
+from __future__ import annotations
+
+import threading
+
+_lock = threading.Lock()
+_active = 0
+
+
+def begin_recall() -> None:
+    global _active
+    with _lock:
+        _active += 1
+
+
+def end_recall() -> None:
+    global _active
+    with _lock:
+        _active = max(0, _active - 1)
+
+
+def in_flight() -> int:
+    with _lock:
+        return _active

{superlocalmemory-3.4.30 → superlocalmemory-3.4.32}/src/superlocalmemory/learning/signals.py

@@ -141,12 +141,18 @@ def reset_counters() -> None:
 
 
 def _drain_queue_for_tests() -> None:
-    """Drain the module queue — TEST-ONLY."""
+    """Drain the module queue — TEST-ONLY.
+
+    Swallows AttributeError so test harnesses that swap in a minimal
+    stub queue (e.g. exploding-queue fixtures) can still clean up.
+    """
     while True:
         try:
             _Q.get_nowait()
         except queue.Empty:
             return
+        except AttributeError:
+            return
 
 
 def queue_size() -> int:

{superlocalmemory-3.4.30 → superlocalmemory-3.4.32}/src/superlocalmemory/mcp/_pool_adapter.py

@@ -110,12 +110,24 @@ def pool_recall(query: str, limit: int = 10, **_: Any) -> PoolRecallResponse:
 
 
 def pool_store(content: str, metadata: dict | None = None) -> list[str]:
-    """Call pool.store and return the fact id list.
+    """Call pool.store and return fact id list (or pending tracker).
 
-    Raises :class:`PoolError` on worker death or any non-ok envelope.
+    v3.4.32: the daemon /remember endpoint is async by default — it
+    returns ``pending_id`` and queues the write. We surface this to
+    callers as ``["pending:<id>"]`` so they have a stable identifier
+    without blocking the remember on the embedder worker.
+
+    Legacy synchronous path (``?wait=true``) still returns real
+    ``fact_ids``. Worker death raises :class:`PoolError`.
     """
     raw = _pool().store(content=content, metadata=metadata or {})
     _unwrap_error(raw, "store")
-    if isinstance(raw, dict):
-        return list(raw.get("fact_ids", []))
+    if not isinstance(raw, dict):
+        return []
+    fact_ids = raw.get("fact_ids")
+    if fact_ids:
+        return list(fact_ids)
+    pending_id = raw.get("pending_id")
+    if pending_id is not None:
+        return [f"pending:{pending_id}"]
     return []

{superlocalmemory-3.4.30 → superlocalmemory-3.4.32}/src/superlocalmemory/mcp/tools_core.py

@@ -113,12 +113,11 @@ def register_core_tools(server, get_engine: Callable) -> None:
         """
         import asyncio
         try:
-            # V3.3.27: Store-first pattern — write to pending.db immediately
-            # (<100ms), then process through full pipeline in background.
-            # This eliminates the 30-40s blocking that Mode B users experience.
-            # Pending memories are auto-processed on next engine.initialize()
-            # or by the daemon's background loop.
-            from superlocalmemory.cli.pending_store import store_pending, mark_done
+            # v3.4.32: Store-first pattern. Write to pending.db and return
+            # immediately. The daemon's pending-materializer thread drains
+            # the queue with recall priority, so concurrent MCP remembers
+            # no longer contend with /search on the shared embedder.
+            from superlocalmemory.cli.pending_store import store_pending
 
             pending_id = store_pending(content, tags=tags, metadata={
                 "project": project,
@@ -127,39 +126,13 @@ def register_core_tools(server, get_engine: Callable) -> None:
                 "session_id": session_id,
             })
 
-            # Fire-and-forget: process in background thread
-            async def _process_in_background():
-                try:
-                    from superlocalmemory.core.worker_pool import WorkerPool
-                    pool = WorkerPool.shared()
-                    result = await asyncio.to_thread(
-                        pool.store, content, metadata={
-                            "tags": tags, "project": project,
-                            "importance": importance, "agent_id": agent_id,
-                            "session_id": session_id,
-                        },
-                    )
-                    if result.get("ok"):
-                        mark_done(pending_id)
-                        _emit_event("memory.created", {
-                            "content_preview": content[:80],
-                            "agent_id": agent_id,
-                            "fact_count": result.get("count", 0),
-                        }, source_agent=agent_id)
-                except Exception as _bg_exc:
-                    logger.warning(
-                        "Background store failed (pending_id=%s): %s",
-                        pending_id, _bg_exc,
-                    )
-
-            asyncio.create_task(_process_in_background())
-
             return {
                 "success": True,
                 "fact_ids": [f"pending:{pending_id}"],
                 "count": 1,
                 "pending": True,
-                "message": "Stored to pending — processing in background.",
+                "pending_id": pending_id,
+                "message": "Stored — facts will appear in the dashboard shortly.",
             }
         except Exception as exc:
             logger.exception("remember failed")

{superlocalmemory-3.4.30 → superlocalmemory-3.4.32}/src/superlocalmemory/server/routes/data_io.py

@@ -28,11 +28,11 @@ router = APIRouter()
 
 @router.get("/api/export")
 async def export_memories(
-    format: str = Query("json", pattern="^(json|jsonl)$"),
+    format: str = Query("json", pattern="^(json|jsonl|csv)$"),
     category: Optional[str] = None,
     project_name: Optional[str] = None,
 ):
-    """Export memories as JSON or JSONL."""
+    """Export memories as JSON, JSONL, or CSV."""
     try:
         conn = get_db_connection()
         conn.row_factory = dict_factory
@@ -76,6 +76,25 @@ async def export_memories(
         if format == "jsonl":
             content = "\n".join(json.dumps(m) for m in memories)
             media_type = "application/x-ndjson"
+        elif format == "csv":
+            import csv
+            import io as _io
+            if memories:
+                buf = _io.StringIO()
+                fieldnames = list(memories[0].keys())
+                writer = csv.DictWriter(
+                    buf, fieldnames=fieldnames, extrasaction="ignore",
+                )
+                writer.writeheader()
+                for m in memories:
+                    writer.writerow({
+                        k: (json.dumps(v) if isinstance(v, (dict, list)) else v)
+                        for k, v in m.items()
+                    })
+                content = buf.getvalue()
+            else:
+                content = ""
+            media_type = "text/csv"
         else:
             content = json.dumps({
                 "version": "3.0.0",

{superlocalmemory-3.4.30 → superlocalmemory-3.4.32}/src/superlocalmemory/server/routes/memories.py

@@ -398,7 +398,12 @@ async def get_graph(
 
 @router.post("/api/search")
 async def search_memories(request: Request, body: SearchRequest):
-    """Semantic search via subprocess worker pool (memory-isolated)."""
+    """Semantic search via subprocess worker pool (memory-isolated).
+
+    v3.4.32: marks recall in-flight so the pending materializer yields.
+    """
+    from superlocalmemory.core.recall_gate import begin_recall, end_recall
+    begin_recall()
     try:
         from superlocalmemory.core.worker_pool import WorkerPool
         pool = WorkerPool.shared()
@@ -435,6 +440,8 @@ async def search_memories(request: Request, body: SearchRequest):
 
     except Exception as e:
         raise HTTPException(status_code=500, detail=f"Search error: {str(e)}")
+    finally:
+        end_recall()
 
 
 @router.get("/api/clusters")
@@ -607,6 +614,97 @@ async def get_memory_facts(request: Request, memory_id: str):
         raise HTTPException(status_code=500, detail=f"Error: {str(e)}")
 
 
+@router.get("/api/memories/{memory_id}/detail")
+async def get_memory_detail(request: Request, memory_id: str):
+    """Full memory row + all child atomic facts (for dashboard modal)."""
+    try:
+        conn = get_db_connection()
+        conn.row_factory = dict_factory
+        cursor = conn.cursor()
+        active_profile = get_active_profile()
+
+        cursor.execute(
+            "SELECT memory_id, content, session_id, speaker, role, "
+            "session_date, created_at, metadata_json "
+            "FROM memories WHERE memory_id = ? AND profile_id = ?",
+            (memory_id, active_profile),
+        )
+        mem = cursor.fetchone()
+        if not mem:
+            conn.close()
+            raise HTTPException(status_code=404, detail="Memory not found")
+
+        cursor.execute(
+            "SELECT fact_id, content, fact_type, confidence, importance, "
+            "access_count, created_at, entities_json "
+            "FROM atomic_facts WHERE memory_id = ? AND profile_id = ? "
+            "ORDER BY created_at ASC",
+            (memory_id, active_profile),
+        )
+        facts = cursor.fetchall()
+        conn.close()
+
+        try:
+            mem["metadata"] = json.loads(mem.pop("metadata_json") or "{}")
+        except Exception:
+            mem["metadata"] = {}
+        for f in facts:
+            try:
+                f["entities"] = json.loads(f.pop("entities_json") or "[]")
+            except Exception:
+                f["entities"] = []
+
+        return {
+            "memory": mem,
+            "facts": facts,
+            "fact_count": len(facts),
+        }
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Detail error: {str(e)}")
+
+
+@router.get("/api/facts/{fact_id}")
+async def get_fact_detail(request: Request, fact_id: str):
+    """Single atomic fact detail (for fact popup)."""
+    try:
+        conn = get_db_connection()
+        conn.row_factory = dict_factory
+        cursor = conn.cursor()
+        active_profile = get_active_profile()
+
+        cursor.execute(
+            "SELECT f.fact_id, f.memory_id, f.content, f.fact_type, "
+            "f.confidence, f.importance, f.access_count, f.created_at, "
+            "f.entities_json, f.canonical_entities_json, f.session_id, "
+            "m.content AS source_memory_content "
+            "FROM atomic_facts f "
+            "LEFT JOIN memories m ON f.memory_id = m.memory_id "
+            "WHERE f.fact_id = ? AND f.profile_id = ?",
+            (fact_id, active_profile),
+        )
+        row = cursor.fetchone()
+        conn.close()
+        if not row:
+            raise HTTPException(status_code=404, detail="Fact not found")
+        try:
+            row["entities"] = json.loads(row.pop("entities_json") or "[]")
+        except Exception:
+            row["entities"] = []
+        try:
+            row["canonical_entities"] = json.loads(
+                row.pop("canonical_entities_json") or "[]"
+            )
+        except Exception:
+            row["canonical_entities"] = []
+        return row
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Fact detail error: {str(e)}")
+
+
 @router.delete("/api/memories/{fact_id}")
 async def delete_memory(request: Request, fact_id: str):
     """Delete a specific memory (atomic fact) by ID."""

{superlocalmemory-3.4.30 → superlocalmemory-3.4.32}/src/superlocalmemory/server/routes/stats.py

@@ -40,6 +40,12 @@ async def get_stats():
                 "SELECT COUNT(*) as total FROM atomic_facts WHERE profile_id = ?",
                 (active_profile,),
             )
+            total_facts = cursor.fetchone()['total']
+
+            cursor.execute(
+                "SELECT COUNT(*) as total FROM memories WHERE profile_id = ?",
+                (active_profile,),
+            )
             total_memories = cursor.fetchone()['total']
 
             total_sessions = 0
@@ -52,7 +58,7 @@ async def get_stats():
             except Exception:
                 pass
 
-            total_graph_nodes = total_memories
+            total_graph_nodes = total_facts
             total_graph_edges = 0
             try:
                 cursor.execute(
@@ -121,12 +127,13 @@ async def get_stats():
             importance_dist = []
 
         else:
-            # V2 fallback
+            # V2 fallback — no atomic_facts; facts == memories
             cursor.execute(
                 "SELECT COUNT(*) as total FROM memories WHERE profile = ?",
                 (active_profile,),
             )
             total_memories = cursor.fetchone()['total']
+            total_facts = total_memories
 
             try:
                 cursor.execute("SELECT COUNT(*) as total FROM sessions")
@@ -201,9 +208,16 @@ async def get_stats():
 
         conn.close()
 
+        facts_per_memory = (
+            round(total_facts / total_memories, 1)
+            if total_memories > 0 else 0.0
+        )
+
         return {
             "overview": {
                 "total_memories": total_memories,
+                "total_facts": total_facts,
+                "facts_per_memory": facts_per_memory,
                 "total_sessions": total_sessions,
                 "total_clusters": total_clusters,
                 "graph_nodes": total_graph_nodes,

{superlocalmemory-3.4.30 → superlocalmemory-3.4.32}/src/superlocalmemory/server/unified_daemon.py

@@ -66,6 +66,20 @@ class ObserveRequest(BaseModel):
     content: str
 
 
+# ---------------------------------------------------------------------------
+# v3.4.32: Recall-priority gate for the pending materializer.
+# All /remember writes go to pending.db and return fast; a background
+# thread drains pending while yielding to any in-flight /search.
+# See ``superlocalmemory.core.recall_gate``.
+# ---------------------------------------------------------------------------
+
+from superlocalmemory.core.recall_gate import (
+    begin_recall as _begin_recall,
+    end_recall as _end_recall,
+    in_flight as _recalls_in_flight,
+)
+
+
 # ---------------------------------------------------------------------------
 # Observation debounce buffer (migrated from daemon.py)
 # ---------------------------------------------------------------------------
@@ -949,6 +963,8 @@ def _register_daemon_routes(application: FastAPI) -> None:
         if not effective_sid:
             import time as _t
             effective_sid = f"http:{int(_t.time() * 1000)}"
+        # v3.4.32: mark recall in-flight so the pending materializer pauses
+        _begin_recall()
         try:
             response = engine.recall(
                 search_query, limit=limit, session_id=effective_sid,
@@ -1006,18 +1022,47 @@ def _register_daemon_routes(application: FastAPI) -> None:
             }
         except Exception as exc:
             raise HTTPException(500, detail=str(exc))
+        finally:
+            _end_recall()
 
     @application.post("/remember")
-    async def remember(req: RememberRequest):
+    async def remember(req: RememberRequest, wait: bool = False):
+        """v3.4.32: Async by default — writes to pending.db, returns pending_id
+        in <100ms. Materializer thread drains at low priority, yielding to
+        /search. Pass ``?wait=true`` for legacy synchronous behavior (blocks
+        on the embedder until facts are written).
+        """
         _update_activity()
         engine = _get_engine_or_503()
+
+        if wait:
+            try:
+                metadata = {"tags": req.tags} if req.tags else {}
+                extra = getattr(req, "metadata", None)
+                if isinstance(extra, dict):
+                    metadata.update(extra)
+                fact_ids = engine.store(req.content, metadata=metadata)
+                return {"ok": True, "fact_ids": fact_ids, "count": len(fact_ids)}
+            except Exception as exc:
+                raise HTTPException(500, detail=str(exc))
+
         try:
-            metadata = {"tags": req.tags} if req.tags else {}
+            from superlocalmemory.cli.pending_store import store_pending
+            meta = {}
+            if req.tags:
+                meta["tags"] = req.tags
             extra = getattr(req, "metadata", None)
             if isinstance(extra, dict):
-                metadata.update(extra)
-            fact_ids = engine.store(req.content, metadata=metadata)
-            return {"ok": True, "fact_ids": fact_ids, "count": len(fact_ids)}
+                meta.update(extra)
+            pending_id = store_pending(
+                req.content, tags=req.tags or "", metadata=meta,
+            )
+            return {
+                "ok": True,
+                "pending_id": pending_id,
+                "status": "queued",
+                "note": "materialized async; pass ?wait=true for legacy sync",
+            }
         except Exception as exc:
             raise HTTPException(500, detail=str(exc))
 
@@ -1189,6 +1234,70 @@ def _start_memory_watchdog() -> None:
     logger.info("Memory watchdog started (limit: %d MB per worker)", MAX_WORKER_MB)
 
 
+_materializer_stop = threading.Event()
+_materializer_thread: threading.Thread | None = None
+
+
+def _start_pending_materializer() -> None:
+    """Background thread: drains pending.db, yields to active /search calls.
+
+    Poll loop:
+    1. Fetch up to 5 pending rows.
+    2. For each row: if any /search is in flight, sleep 500ms (yield priority).
+    3. Call engine.store(), mark_done or mark_failed.
+    4. Sleep 2s between polls when idle (empty queue).
+    """
+    global _materializer_thread
+
+    def _loop():
+        from superlocalmemory.cli.pending_store import (
+            get_pending, mark_done, mark_failed,
+        )
+        while not _materializer_stop.is_set():
+            try:
+                engine = _engine  # may be None briefly at startup
+                if engine is None:
+                    time.sleep(2.0)
+                    continue
+                pending = get_pending(limit=5)
+                if not pending:
+                    time.sleep(2.0)
+                    continue
+                for item in pending:
+                    if _materializer_stop.is_set():
+                        break
+                    # Yield to recalls: wait until none in flight
+                    waits = 0
+                    while _recalls_in_flight() > 0 and waits < 60:
+                        time.sleep(0.5)
+                        waits += 1
+                    try:
+                        import json as _json
+                        md_str = item.get("metadata") or "{}"
+                        try:
+                            md = _json.loads(md_str)
+                        except Exception:
+                            md = {}
+                        if item.get("tags"):
+                            md.setdefault("tags", item["tags"])
+                        engine.store(item["content"], metadata=md)
+                        mark_done(item["id"])
+                    except Exception as exc:
+                        logger.warning(
+                            "Pending %d failed: %s", item["id"], exc,
+                        )
+                        mark_failed(item["id"], str(exc))
+            except Exception as exc:
+                logger.warning("materializer loop error: %s", exc)
+                time.sleep(5.0)
+
+    _materializer_thread = threading.Thread(
+        target=_loop, daemon=True, name="pending-materializer",
+    )
+    _materializer_thread.start()
+    logger.info("Pending materializer started (recall-priority)")
+
+
 def start_server(port: int = _DEFAULT_PORT) -> None:
     """Start the unified daemon. Blocks until stopped."""
     global _start_time
@@ -1223,6 +1332,9 @@ def start_server(port: int = _DEFAULT_PORT) -> None:
     # v3.4.7: Start memory watchdog to prevent runaway workers
     _start_memory_watchdog()
 
+    # v3.4.32: Continuous pending-queue materializer with recall priority.
+    _start_pending_materializer()
+
     log_dir = Path.home() / ".superlocalmemory" / "logs"
     log_dir.mkdir(parents=True, exist_ok=True)