superlocalmemory 3.4.37 → 3.4.39

package/CHANGELOG.md

@@ -10,6 +10,64 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ---
 
+## [3.4.38] - 2026-04-26
+
+**P0 silent data loss fix.** The async `/remember` pipeline was broken since
+v3.4.32 — memories were being marked "queued" and acknowledged but never
+actually persisting to memory.db during runtime. Only daemon-restart drained
+the pending queue (limit 20 per restart). 18 memories were permanently lost
+to a NoneType iterable crash between April 15-26, 2026, all recoverable
+because the content was preserved in pending.db.
+
+### Fixed
+- **Materializer `_engine` NameError** (`unified_daemon.py`). The background
+  pending materializer thread referenced a module-level `_engine` global
+  that was never declared. Result: every iteration threw `NameError: name
+  '_engine' is not defined`, the exception was caught and logged as
+  "materializer loop error", and the thread slept 5s and retried forever
+  without ever processing pending memories. Bug present since v3.4.32.
+  Fixed by declaring `_engine = None` at module level and assigning
+  `_engine = engine` in the FastAPI lifespan after `engine.initialize()`.
+- **scene_builder NoneType crash** (`encoding/scene_builder.py:assign_to_scene`).
+  When the embedding worker was unavailable (cold-start timeout, crash),
+  `embedder.embed()` returned None. The code checked `theme_emb is None`
+  but never checked `fact_emb is None`, so `_cosine(None, theme_emb)`
+  called `zip(None, theme_emb)` → `'NoneType' object is not iterable`,
+  propagating up through `engine.store()` → mark_failed → permanent loss.
+  Fixed by guarding `fact_emb is None` (skip scene assignment, still create
+  scene) and adding defensive `None` check to `_cosine()` itself.
+- **Retry-aware mark_failed** (`cli/pending_store.py`). Previously, ANY
+  exception during materialization permanently marked the memory as
+  failed — even transient errors like embedding worker timeout. Now uses
+  the existing `retry_count` column: keeps status as `pending` until 3
+  retries, only marks `failed` after all retries are exhausted.
+
+### Added
+- **Diagnostic logging in materializer** — "Materializer: waiting for
+  engine to init...", "engine acquired, starting drain loop", "processing
+  N pending memories" — so operators can verify the materializer is alive
+  without grepping for absence of error messages.
+- **`tests/test_integration/test_async_remember_e2e.py`** — full
+  production pipeline test: POST `/remember` (async, default mode) →
+  wait up to 60s → verify content in `memory.db` → recall returns it.
+  This is the test that was missing for 8+ months. The 4,501 existing
+  test functions test components in isolation (mocking `store_pending`)
+  and never exercise the full async flow that real users hit.
+
+### Recovery
+On install, if you have existing failed records in `pending.db`, they will
+be auto-retried on the next daemon restart by `engine._process_pending_memories()`.
+To manually recover, run:
+```python
+import sqlite3
+db = sqlite3.connect('~/.superlocalmemory/pending.db')
+db.execute("UPDATE pending_memories SET status='pending', retry_count=0, error=NULL WHERE status='failed'")
+db.commit()
+```
+Then `slm restart`.
+
+---
+
 ## [3.4.37] - 2026-04-26
 
 **P0 RAM fix.** Total SLM footprint reduced from ~14 GB peak to ~2.3 GB peak

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "superlocalmemory",
-  "version": "3.4.37",
+  "version": "3.4.39",
   "description": "Information-geometric agent memory with mathematical guarantees. 4-channel retrieval, Fisher-Rao similarity, zero-LLM mode, EU AI Act compliant. Works with Claude, Cursor, Windsurf, and 17+ AI tools.",
   "keywords": [
     "ai-memory",

package/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "superlocalmemory"
-version = "3.4.37"
+version = "3.4.39"
 description = "Information-geometric agent memory with mathematical guarantees"
 readme = "README.md"
 license = {text = "AGPL-3.0-or-later"}

package/src/superlocalmemory/__init__.py

@@ -1,3 +1,3 @@
 """SuperLocalMemory — information-geometric agent memory."""
 
-__version__ = "3.4.37"
+__version__ = "3.4.39"

package/src/superlocalmemory/cli/pending_store.py

@@ -122,13 +122,22 @@ def mark_done(row_id: int, base_dir: Path | None = None) -> None:
 
 
 def mark_failed(row_id: int, error: str, base_dir: Path | None = None) -> None:
-    """Mark a pending memory as failed with error message."""
+    """Mark a pending memory as failed with error message.
+
+    v3.4.38: Now retry-aware. If retry_count < _MAX_RETRIES, keeps status as
+    'pending' so the materializer will retry on next iteration. Only marks
+    permanently failed after _MAX_RETRIES (3) attempts. The previous behavior
+    permanently lost 18 memories between April 15-26, 2026 to transient errors.
+    """
     conn = _get_db(base_dir)
     try:
+        # Increment retry count and conditionally update status
         conn.execute(
-            "UPDATE pending_memories SET status = 'failed', error = ?, "
-            "retry_count = retry_count + 1 WHERE id = ?",
-            (error, row_id),
+            "UPDATE pending_memories SET error = ?, "
+            "retry_count = retry_count + 1, "
+            "status = CASE WHEN retry_count + 1 >= ? THEN 'failed' ELSE 'pending' END "
+            "WHERE id = ?",
+            (error, _MAX_RETRIES, row_id),
         )
         conn.commit()
     finally:

package/src/superlocalmemory/core/store_pipeline.py

@@ -167,6 +167,15 @@ def run_store(
         session_date=parsed_date, speaker_a=speaker,
     )
 
+    # v3.4.38: Defensive None guard. extract_facts() returns None on transient
+    # failures (embedding worker timeout, LLM call fail). Without this guard,
+    # line 201's `{f.content for f in facts}` raises 'NoneType' object is not
+    # iterable, causing the caller to mark_failed permanently — even though
+    # the content is still recoverable. 18 memories were lost to this between
+    # April 15-26, 2026.
+    if facts is None:
+        facts = []
+
     # V3.3.11: Also store raw content as a verbatim fact to preserve details
     # that fact extraction may abstract away (dates, names, specifics).
     # This ensures BM25 and semantic search can always find the original text.

package/src/superlocalmemory/encoding/scene_builder.py

@@ -56,6 +56,15 @@ class SceneBuilder:
         # Always compute fact embedding first — needed for comparisons
         fact_emb = self._embedder.embed(new_fact.content)
 
+        # v3.4.38: Defensive None guard. embedder.embed() returns None when
+        # the embedding worker is unavailable (timeout, crash). Without this
+        # guard, _cosine(None, theme_emb) → zip(None, ...) → 'NoneType'
+        # object is not iterable, propagating up to engine.store() and
+        # causing the entire memory to be lost. Better to skip scene
+        # assignment than lose the memory.
+        if fact_emb is None:
+            return self._create_scene(new_fact, profile_id)
+
         scenes = self._get_scenes(profile_id)
         if not scenes:
             return self._create_scene(new_fact, profile_id)
@@ -189,7 +198,12 @@ class SceneBuilder:
         )
 
 
-def _cosine(a: list[float], b: list[float]) -> float:
+def _cosine(a: list[float] | None, b: list[float] | None) -> float:
+    # v3.4.38: Defensive None guard — embedder can return None on worker
+    # unavailability. Returning 0.0 is correct: zero similarity means no
+    # match, which falls back to creating a new scene.
+    if a is None or b is None:
+        return 0.0
     dot = sum(x * y for x, y in zip(a, b))
     na = sum(x * x for x in a) ** 0.5
     nb = sum(x * x for x in b) ** 0.5

package/src/superlocalmemory/mcp/tools_active.py

@@ -17,6 +17,7 @@ Part of Qualixar | Author: Varun Pratap Bhardwaj
 from __future__ import annotations
 
 import logging
+import os
 from pathlib import Path
 from typing import Callable
 
@@ -26,17 +27,32 @@ MEMORY_DIR = Path.home() / ".superlocalmemory"
 DB_PATH = MEMORY_DIR / "memory.db"
 
 
+def _get_agent_id(default: str = "mcp_client") -> str:
+    """Resolve the calling agent's ID for attribution.
+
+    Each Avenger (Claude, Codex, Gemini, Kimi, GLM, Qwen, etc.) sets the
+    ``SLM_AGENT_ID`` env var in its MCP server config so that memories,
+    observations, and registry entries are tagged with the actual source
+    agent — not the legacy ``"mcp_client"`` default.
+
+    v3.4.39+: enables proper cross-Avenger attribution in ``session_init``,
+    ``observe``, and event emissions.
+    """
+    return os.environ.get("SLM_AGENT_ID", default)
+
+
 def _emit_event(event_type: str, payload: dict | None = None,
-                source_agent: str = "mcp_client") -> None:  # V3.3.12: see also mcp/shared.py
+                source_agent: str | None = None) -> None:  # V3.3.12: see also mcp/shared.py
     """Emit an event to the EventBus (best-effort, never raises).
 
     Dashboard visibility is load-bearing per the v3.4.26 user contract,
     so we log on failure rather than silently dropping the signal.
     """
+    resolved_agent = source_agent if source_agent is not None else _get_agent_id()
     try:
         from superlocalmemory.infra.event_bus import EventBus
         bus = EventBus.get_instance(str(DB_PATH))
-        bus.emit(event_type, payload=payload, source_agent=source_agent,
+        bus.emit(event_type, payload=payload, source_agent=resolved_agent,
                  source_protocol="mcp")
     except Exception as exc:
         logger.warning("event emit failed: type=%s err=%s", event_type, exc)
@@ -116,10 +132,11 @@ def register_active_tools(server, get_engine: Callable) -> None:
                     "session_init feedback_count read failed: %s", exc,
                 )
 
-            # Register agent + emit event
-            _register_agent("mcp_client", pid)
+            # Register agent + emit event (v3.4.39: SLM_AGENT_ID env support)
+            agent_id = _get_agent_id()
+            _register_agent(agent_id, pid)
             _emit_event("agent.connected", {
-                "agent_id": "mcp_client",
+                "agent_id": agent_id,
                 "project_path": project_path,
                 "memory_count": len(memories),
             })
@@ -145,7 +162,7 @@ def register_active_tools(server, get_engine: Callable) -> None:
     @server.tool()
     async def observe(
         content: str,
-        agent_id: str = "mcp_client",
+        agent_id: str | None = None,
     ) -> dict:
         """Observe conversation content for automatic memory capture.
 
@@ -155,7 +172,13 @@ def register_active_tools(server, get_engine: Callable) -> None:
 
         Call this after making decisions, fixing bugs, or expressing preferences.
         The system will NOT store low-confidence or irrelevant content.
+
+        v3.4.39: ``agent_id`` now defaults to the ``SLM_AGENT_ID`` env var
+        (set by each Avenger's MCP config) so observations carry proper
+        cross-Avenger attribution.
         """
+        if agent_id is None:
+            agent_id = _get_agent_id()
         try:
             from superlocalmemory.hooks.auto_capture import AutoCapture
             from superlocalmemory.hooks.rules_engine import RulesEngine

package/src/superlocalmemory/server/unified_daemon.py

@@ -148,6 +148,13 @@ from superlocalmemory.core.recall_gate import (
     in_flight as _recalls_in_flight,
 )
 
+# v3.4.38: Module-level engine reference for the pending materializer.
+# Set by the FastAPI lifespan after engine.initialize(). Was missing before,
+# causing "name '_engine' is not defined" errors that blocked materialization
+# of pending memories — they accumulated forever, only being processed at
+# daemon startup via engine._process_pending_memories().
+_engine = None
+
 
 # ---------------------------------------------------------------------------
 # Observation debounce buffer (migrated from daemon.py)
@@ -420,6 +427,9 @@ async def lifespan(application: FastAPI):
 
         application.state.engine = engine
         application.state.config = config
+        # v3.4.38: Wire module-level _engine for the pending materializer.
+        global _engine
+        _engine = engine
         logger.info("Unified daemon: MemoryEngine initialized (mode=%s)", config.mode.value)
 
         # LLD-07 §4 — deferred migrations (e.g. M006 reward column) need to
@@ -1378,16 +1388,31 @@ def _start_pending_materializer() -> None:
         from superlocalmemory.cli.pending_store import (
             get_pending, mark_done, mark_failed,
         )
+        # v3.4.38: log first engine acquisition so we know materializer is alive
+        _engine_logged = False
+        _waiting_logged = False
         while not _materializer_stop.is_set():
             try:
-                engine = _engine  # may be None briefly at startup
+                # v3.4.38: Read fresh module global on every iteration so we
+                # pick up the engine after lifespan sets it. Use the import
+                # trick to ensure we're reading the live module attribute,
+                # not a stale local reference.
+                import superlocalmemory.server.unified_daemon as _ud
+                engine = _ud._engine
                 if engine is None:
+                    if not _waiting_logged:
+                        logger.info("Materializer: waiting for engine to init...")
+                        _waiting_logged = True
                     time.sleep(2.0)
                     continue
+                if not _engine_logged:
+                    logger.info("Materializer: engine acquired, starting drain loop")
+                    _engine_logged = True
                 pending = get_pending(limit=5)
                 if not pending:
                     time.sleep(2.0)
                     continue
+                logger.info("Materializer: processing %d pending memories", len(pending))
                 for item in pending:
                     if _materializer_stop.is_set():
                         break

package/src/superlocalmemory.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: superlocalmemory
-Version: 3.4.37
+Version: 3.4.39
 Summary: Information-geometric agent memory with mathematical guarantees
 Author-email: Varun Pratap Bhardwaj <admin@superlocalmemory.com>
 License: AGPL-3.0-or-later