superlocalmemory 3.4.37 → 3.4.38

package/CHANGELOG.md

@@ -10,6 +10,64 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ---
 
+## [3.4.38] - 2026-04-26
+
+**P0 silent data loss fix.** The async `/remember` pipeline was broken since
+v3.4.32 — memories were being marked "queued" and acknowledged but never
+actually persisting to memory.db during runtime. Only daemon-restart drained
+the pending queue (limit 20 per restart). 18 memories were permanently lost
+to a NoneType iterable crash between April 15-26, 2026, all recoverable
+because the content was preserved in pending.db.
+
+### Fixed
+- **Materializer `_engine` NameError** (`unified_daemon.py`). The background
+  pending materializer thread referenced a module-level `_engine` global
+  that was never declared. Result: every iteration threw `NameError: name
+  '_engine' is not defined`, the exception was caught and logged as
+  "materializer loop error", and the thread slept 5s and retried forever
+  without ever processing pending memories. Bug present since v3.4.32.
+  Fixed by declaring `_engine = None` at module level and assigning
+  `_engine = engine` in the FastAPI lifespan after `engine.initialize()`.
+- **scene_builder NoneType crash** (`encoding/scene_builder.py:assign_to_scene`).
+  When the embedding worker was unavailable (cold-start timeout, crash),
+  `embedder.embed()` returned None. The code checked `theme_emb is None`
+  but never checked `fact_emb is None`, so `_cosine(None, theme_emb)`
+  called `zip(None, theme_emb)` → `'NoneType' object is not iterable`,
+  propagating up through `engine.store()` → mark_failed → permanent loss.
+  Fixed by guarding `fact_emb is None` (skip scene assignment, still create
+  scene) and adding defensive `None` check to `_cosine()` itself.
+- **Retry-aware mark_failed** (`cli/pending_store.py`). Previously, ANY
+  exception during materialization permanently marked the memory as
+  failed — even transient errors like embedding worker timeout. Now uses
+  the existing `retry_count` column: keeps status as `pending` until 3
+  retries, only marks `failed` after all retries are exhausted.
+
+### Added
+- **Diagnostic logging in materializer** — "Materializer: waiting for
+  engine to init...", "engine acquired, starting drain loop", "processing
+  N pending memories" — so operators can verify the materializer is alive
+  without grepping for absence of error messages.
+- **`tests/test_integration/test_async_remember_e2e.py`** — full
+  production pipeline test: POST `/remember` (async, default mode) →
+  wait up to 60s → verify content in `memory.db` → recall returns it.
+  This is the test that was missing for 8+ months. The 4,501 existing
+  test functions test components in isolation (mocking `store_pending`)
+  and never exercise the full async flow that real users hit.
+
+### Recovery
+On install, if you have existing failed records in `pending.db`, they will
+be auto-retried on the next daemon restart by `engine._process_pending_memories()`.
+To manually recover, run:
+```python
+import sqlite3
+db = sqlite3.connect('~/.superlocalmemory/pending.db')
+db.execute("UPDATE pending_memories SET status='pending', retry_count=0, error=NULL WHERE status='failed'")
+db.commit()
+```
+Then `slm restart`.
+
+---
+
 ## [3.4.37] - 2026-04-26
 
 **P0 RAM fix.** Total SLM footprint reduced from ~14 GB peak to ~2.3 GB peak

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "superlocalmemory",
-  "version": "3.4.37",
+  "version": "3.4.38",
   "description": "Information-geometric agent memory with mathematical guarantees. 4-channel retrieval, Fisher-Rao similarity, zero-LLM mode, EU AI Act compliant. Works with Claude, Cursor, Windsurf, and 17+ AI tools.",
   "keywords": [
     "ai-memory",

package/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "superlocalmemory"
-version = "3.4.37"
+version = "3.4.38"
 description = "Information-geometric agent memory with mathematical guarantees"
 readme = "README.md"
 license = {text = "AGPL-3.0-or-later"}

package/src/superlocalmemory/__init__.py

@@ -1,3 +1,3 @@
 """SuperLocalMemory — information-geometric agent memory."""
 
-__version__ = "3.4.37"
+__version__ = "3.4.38"

package/src/superlocalmemory/cli/pending_store.py

@@ -122,13 +122,22 @@ def mark_done(row_id: int, base_dir: Path | None = None) -> None:
 
 
 def mark_failed(row_id: int, error: str, base_dir: Path | None = None) -> None:
-    """Mark a pending memory as failed with error message."""
+    """Mark a pending memory as failed with error message.
+
+    v3.4.38: Now retry-aware. If retry_count < _MAX_RETRIES, keeps status as
+    'pending' so the materializer will retry on next iteration. Only marks
+    permanently failed after _MAX_RETRIES (3) attempts. The previous behavior
+    permanently lost 18 memories between April 15-26, 2026 to transient errors.
+    """
     conn = _get_db(base_dir)
     try:
+        # Increment retry count and conditionally update status
         conn.execute(
-            "UPDATE pending_memories SET status = 'failed', error = ?, "
-            "retry_count = retry_count + 1 WHERE id = ?",
-            (error, row_id),
+            "UPDATE pending_memories SET error = ?, "
+            "retry_count = retry_count + 1, "
+            "status = CASE WHEN retry_count + 1 >= ? THEN 'failed' ELSE 'pending' END "
+            "WHERE id = ?",
+            (error, _MAX_RETRIES, row_id),
         )
         conn.commit()
     finally:

package/src/superlocalmemory/core/store_pipeline.py

@@ -167,6 +167,15 @@ def run_store(
         session_date=parsed_date, speaker_a=speaker,
     )
 
+    # v3.4.38: Defensive None guard. extract_facts() returns None on transient
+    # failures (embedding worker timeout, LLM call fail). Without this guard,
+    # line 201's `{f.content for f in facts}` raises 'NoneType' object is not
+    # iterable, causing the caller to mark_failed permanently — even though
+    # the content is still recoverable. 18 memories were lost to this between
+    # April 15-26, 2026.
+    if facts is None:
+        facts = []
+
     # V3.3.11: Also store raw content as a verbatim fact to preserve details
     # that fact extraction may abstract away (dates, names, specifics).
     # This ensures BM25 and semantic search can always find the original text.

package/src/superlocalmemory/encoding/scene_builder.py

@@ -56,6 +56,15 @@ class SceneBuilder:
         # Always compute fact embedding first — needed for comparisons
         fact_emb = self._embedder.embed(new_fact.content)
 
+        # v3.4.38: Defensive None guard. embedder.embed() returns None when
+        # the embedding worker is unavailable (timeout, crash). Without this
+        # guard, _cosine(None, theme_emb) → zip(None, ...) → 'NoneType'
+        # object is not iterable, propagating up to engine.store() and
+        # causing the entire memory to be lost. Better to skip scene
+        # assignment than lose the memory.
+        if fact_emb is None:
+            return self._create_scene(new_fact, profile_id)
+
         scenes = self._get_scenes(profile_id)
         if not scenes:
             return self._create_scene(new_fact, profile_id)
@@ -189,7 +198,12 @@ class SceneBuilder:
         )
 
 
-def _cosine(a: list[float], b: list[float]) -> float:
+def _cosine(a: list[float] | None, b: list[float] | None) -> float:
+    # v3.4.38: Defensive None guard — embedder can return None on worker
+    # unavailability. Returning 0.0 is correct: zero similarity means no
+    # match, which falls back to creating a new scene.
+    if a is None or b is None:
+        return 0.0
     dot = sum(x * y for x, y in zip(a, b))
     na = sum(x * x for x in a) ** 0.5
     nb = sum(x * x for x in b) ** 0.5

package/src/superlocalmemory/server/unified_daemon.py

@@ -148,6 +148,13 @@ from superlocalmemory.core.recall_gate import (
     in_flight as _recalls_in_flight,
 )
 
+# v3.4.38: Module-level engine reference for the pending materializer.
+# Set by the FastAPI lifespan after engine.initialize(). Was missing before,
+# causing "name '_engine' is not defined" errors that blocked materialization
+# of pending memories — they accumulated forever, only being processed at
+# daemon startup via engine._process_pending_memories().
+_engine = None
+
 
 # ---------------------------------------------------------------------------
 # Observation debounce buffer (migrated from daemon.py)
@@ -420,6 +427,9 @@ async def lifespan(application: FastAPI):
 
         application.state.engine = engine
         application.state.config = config
+        # v3.4.38: Wire module-level _engine for the pending materializer.
+        global _engine
+        _engine = engine
         logger.info("Unified daemon: MemoryEngine initialized (mode=%s)", config.mode.value)
 
         # LLD-07 §4 — deferred migrations (e.g. M006 reward column) need to
@@ -1378,16 +1388,31 @@ def _start_pending_materializer() -> None:
         from superlocalmemory.cli.pending_store import (
             get_pending, mark_done, mark_failed,
         )
+        # v3.4.38: log first engine acquisition so we know materializer is alive
+        _engine_logged = False
+        _waiting_logged = False
         while not _materializer_stop.is_set():
             try:
-                engine = _engine  # may be None briefly at startup
+                # v3.4.38: Read fresh module global on every iteration so we
+                # pick up the engine after lifespan sets it. Use the import
+                # trick to ensure we're reading the live module attribute,
+                # not a stale local reference.
+                import superlocalmemory.server.unified_daemon as _ud
+                engine = _ud._engine
                 if engine is None:
+                    if not _waiting_logged:
+                        logger.info("Materializer: waiting for engine to init...")
+                        _waiting_logged = True
                     time.sleep(2.0)
                     continue
+                if not _engine_logged:
+                    logger.info("Materializer: engine acquired, starting drain loop")
+                    _engine_logged = True
                 pending = get_pending(limit=5)
                 if not pending:
                     time.sleep(2.0)
                     continue
+                logger.info("Materializer: processing %d pending memories", len(pending))
                 for item in pending:
                     if _materializer_stop.is_set():
                         break

package/src/superlocalmemory.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: superlocalmemory
-Version: 3.4.37
+Version: 3.4.38
 Summary: Information-geometric agent memory with mathematical guarantees
 Author-email: Varun Pratap Bhardwaj <admin@superlocalmemory.com>
 License: AGPL-3.0-or-later