superlocalmemory 3.7.6 → 3.7.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/CHANGELOG.md +30 -0
  2. package/README.md +2 -2
  3. package/package.json +2 -2
  4. package/plugin/.claude-plugin/plugin.json +1 -1
  5. package/plugin/requirements.txt +1 -1
  6. package/plugin-src/manifest.json +1 -1
  7. package/plugin-src/requirements.txt +1 -1
  8. package/pyproject.toml +6 -7
  9. package/src/superlocalmemory/__init__.py +1 -1
  10. package/src/superlocalmemory/cli/commands.py +171 -11
  11. package/src/superlocalmemory/cli/setup_wizard.py +18 -1
  12. package/src/superlocalmemory/infra/auth_middleware.py +33 -5
  13. package/src/superlocalmemory/mcp/_daemon_proxy.py +8 -10
  14. package/src/superlocalmemory/mcp/server.py +1 -0
  15. package/src/superlocalmemory/mcp/tools_core.py +216 -20
  16. package/src/superlocalmemory/optimize/cache/centroid_store.py +21 -3
  17. package/src/superlocalmemory/optimize/cache/manager.py +7 -0
  18. package/src/superlocalmemory/optimize/cache/semantic.py +27 -10
  19. package/src/superlocalmemory/server/api.py +17 -0
  20. package/src/superlocalmemory/server/profile_runtime.py +384 -0
  21. package/src/superlocalmemory/server/recall_health.py +12 -6
  22. package/src/superlocalmemory/server/routes/chat.py +63 -12
  23. package/src/superlocalmemory/server/routes/helpers.py +9 -16
  24. package/src/superlocalmemory/server/routes/memories.py +58 -11
  25. package/src/superlocalmemory/server/routes/profiles.py +24 -14
  26. package/src/superlocalmemory/server/routes/v3_api.py +128 -52
  27. package/src/superlocalmemory/server/ui.py +10 -0
  28. package/src/superlocalmemory/server/unified_daemon.py +290 -74
  29. package/src/superlocalmemory/storage/migration_runner.py +17 -3
  30. package/src/superlocalmemory/storage/migrations/M002_model_state_history.py +32 -3
  31. package/src/superlocalmemory/storage/schema_v32.py +0 -9
  32. package/src/superlocalmemory/ui/index.html +32 -1
  33. package/src/superlocalmemory/ui/js/auto-settings.js +48 -0
  34. package/src/superlocalmemory/ui/js/profiles.js +11 -2
@@ -182,10 +182,26 @@ class EngineRecallAdapter:
182
182
  loaded a SECOND MemoryEngine. This adapter eliminates that duplication.
183
183
  """
184
184
 
185
- def __init__(self, engine) -> None:
185
+ def __init__(self, engine, profile_runtime=None) -> None:
186
+ self._engine = engine
187
+ self._profile_runtime = profile_runtime
188
+
189
+ def set_engine(self, engine) -> None:
190
+ """Replace the engine while the profile runtime is exclusive."""
186
191
  self._engine = engine
187
192
 
188
193
  def recall(self, query: str, limit: int = 10, session_id: str = "") -> dict:
194
+ from contextlib import nullcontext
195
+
196
+ lease = (
197
+ self._profile_runtime.operation()
198
+ if self._profile_runtime is not None
199
+ else nullcontext()
200
+ )
201
+ with lease:
202
+ return self._recall(query, limit=limit, session_id=session_id)
203
+
204
+ def _recall(self, query: str, limit: int = 10, session_id: str = "") -> dict:
189
205
  response = self._engine.recall(
190
206
  query, limit=limit, session_id=session_id or None,
191
207
  )
@@ -229,6 +245,84 @@ class EngineRecallAdapter:
229
245
  }
230
246
 
231
247
 
248
+ def _configure_scale_backends(engine, config) -> None:
249
+ """Attach optional graph/vector backends to one initialized engine."""
250
+ try:
251
+ from superlocalmemory.core.backend_orchestrator import (
252
+ BackendOrchestrator,
253
+ set_orchestrator,
254
+ )
255
+
256
+ orchestrator = BackendOrchestrator(config=config, db=engine._db)
257
+ orchestrator.on_daemon_start()
258
+ set_orchestrator(orchestrator)
259
+ cozo_backend = orchestrator.get_graph_backend()
260
+ lancedb_backend = orchestrator.get_vector_backend()
261
+ retrieval = getattr(engine, "_retrieval_engine", None)
262
+ if retrieval is not None:
263
+ entity_graph = getattr(retrieval, "_entity", None)
264
+ if (
265
+ entity_graph is not None
266
+ and cozo_backend is not None
267
+ and orchestrator.graph_retrieval_ready()
268
+ ):
269
+ entity_graph._cozo = cozo_backend
270
+ semantic = getattr(retrieval, "_semantic", None)
271
+ if semantic is not None and lancedb_backend is not None:
272
+ semantic.set_scale_vector_backend(lancedb_backend)
273
+ logger.info(
274
+ "BackendOrchestrator: ready (cozo=%s, lancedb=%s)",
275
+ "active" if cozo_backend else "off",
276
+ "active" if lancedb_backend else "off",
277
+ )
278
+ except Exception as exc:
279
+ logger.warning("BackendOrchestrator init failed (non-fatal): %s", exc)
280
+
281
+
282
+ def _hot_reconfigure_engine(application, new_config, *, mode_change: bool) -> None:
283
+ """Build and publish one coherent daemon engine for a saved config."""
284
+ from superlocalmemory.core.engine import MemoryEngine
285
+
286
+ old_engine = getattr(application.state, "engine", None)
287
+ new_engine = MemoryEngine(new_config)
288
+ try:
289
+ new_engine.initialize()
290
+ new_config.save(mode_change=mode_change)
291
+ except BaseException:
292
+ new_engine.close()
293
+ raise
294
+
295
+ # The profile transition barrier is exclusive here. Publish every
296
+ # long-lived reference before closing the former engine.
297
+ application.state.engine = new_engine
298
+ application.state.config = new_config
299
+ global _engine
300
+ _engine = new_engine
301
+ _observe_buffer.set_engine(new_engine)
302
+ adapter = getattr(application.state, "engine_recall_adapter", None)
303
+ if adapter is not None:
304
+ adapter.set_engine(new_engine)
305
+ _configure_scale_backends(new_engine, new_config)
306
+
307
+ old_health_stop = getattr(application.state, "recall_health_stop", None)
308
+ if old_health_stop is not None:
309
+ old_health_stop.set()
310
+ try:
311
+ from superlocalmemory.server.recall_health import start_recall_health_monitor
312
+
313
+ runtime = getattr(application.state, "profile_runtime", None)
314
+ _thread, health_stop, _state = start_recall_health_monitor(
315
+ new_engine, runtime=runtime,
316
+ )
317
+ application.state.recall_health_stop = health_stop
318
+ except Exception as exc:
319
+ application.state.recall_health_stop = None
320
+ logger.warning("recall-health restart failed (non-fatal): %s", exc)
321
+
322
+ if old_engine is not None and old_engine is not new_engine:
323
+ old_engine.close()
324
+
325
+
232
326
  # ---------------------------------------------------------------------------
233
327
  # v3.4.32: Recall-priority gate for the pending materializer.
234
328
  # All /remember writes go to pending.db and return fast; a background
@@ -248,6 +342,7 @@ from superlocalmemory.core.recall_gate import (
248
342
  # of pending memories — they accumulated forever, only being processed at
249
343
  # daemon startup via engine._process_pending_memories().
250
344
  _engine = None
345
+ _profile_runtime = None
251
346
 
252
347
 
253
348
  def _emit_event(
@@ -679,54 +774,24 @@ async def lifespan(application: FastAPI):
679
774
  except Exception:
680
775
  pass
681
776
 
682
- application.state.engine = engine
683
- application.state.config = config
777
+ from superlocalmemory.server.profile_runtime import bind_profile_runtime
778
+
779
+ profile_runtime = bind_profile_runtime(application.state, engine, config)
780
+ application.state.reconfigure_engine = (
781
+ lambda new_config, mode_change=False: _hot_reconfigure_engine(
782
+ application, new_config, mode_change=mode_change,
783
+ )
784
+ )
684
785
  # v3.4.38: Wire module-level _engine for the pending materializer.
685
- global _engine
786
+ global _engine, _profile_runtime
787
+ _profile_runtime = profile_runtime
686
788
  _engine = engine
687
789
  logger.info("Unified daemon: MemoryEngine initialized (mode=%s)", config.mode.value)
688
790
 
689
791
  # v3.5.0: Backend Orchestrator — CozoDB (graph) + LanceDB (vector) backends.
690
792
  # Initialise AFTER engine so the retrieval channels exist to receive backends.
691
793
  # Migrates edges/embeddings automatically; fail-soft (non-blocking).
692
- _cozo_backend = None
693
- _lancedb_backend = None
694
- try:
695
- from superlocalmemory.core.backend_orchestrator import (
696
- BackendOrchestrator, set_orchestrator,
697
- )
698
- orch = BackendOrchestrator(config=config, db=engine._db)
699
- orch.on_daemon_start()
700
- set_orchestrator(orch)
701
- _cozo_backend = orch.get_graph_backend()
702
- _lancedb_backend = orch.get_vector_backend()
703
- # Cozo storage may be active before its canonical-entity retrieval
704
- # projection is parity-proven. Never route mismatched ID spaces.
705
- re = getattr(engine, '_retrieval_engine', None)
706
- if re is not None:
707
- eg = getattr(re, '_entity', None)
708
- if (
709
- eg is not None
710
- and _cozo_backend is not None
711
- and orch.graph_retrieval_ready()
712
- ):
713
- try:
714
- eg._cozo = _cozo_backend
715
- logger.info("CozoDB backend wired into entity_graph channel")
716
- except Exception as exc:
717
- logger.warning("CozoDB channel injection failed: %s", exc)
718
- semantic = getattr(re, '_semantic', None)
719
- if semantic is not None and _lancedb_backend is not None:
720
- try:
721
- semantic.set_scale_vector_backend(_lancedb_backend)
722
- logger.info("LanceDB backend wired into semantic channel with SQLite shadow")
723
- except Exception as exc:
724
- logger.warning("LanceDB channel injection failed: %s", exc)
725
- logger.info("BackendOrchestrator: ready (cozo=%s, lancedb=%s)",
726
- "active" if _cozo_backend else "off",
727
- "active" if _lancedb_backend else "off")
728
- except Exception as exc:
729
- logger.warning("BackendOrchestrator init failed (non-fatal): %s", exc)
794
+ _configure_scale_backends(engine, config)
730
795
 
731
796
  # LLD-07 §4 — deferred migrations (e.g. M006 reward column) need to
732
797
  # run AFTER MemoryEngine.initialize() has bootstrapped runtime tables
@@ -855,8 +920,9 @@ async def lifespan(application: FastAPI):
855
920
  # Fire 2 warmup queries: one to load the graph page cache,
856
921
  # second to warm the reranker subprocess + all producers.
857
922
  # Without this, dashboard POST /api/search hits 11s cold.
858
- for wq in ("memory recall performance", "context injection retrieval"):
859
- engine.recall(wq, limit=5)
923
+ with profile_runtime.operation():
924
+ for wq in ("memory recall performance", "context injection retrieval"):
925
+ engine.recall(wq, limit=5)
860
926
  elapsed = round((_t.monotonic() - t0) * 1000)
861
927
  logger.info(
862
928
  "Recall engine pre-warmed in %dms", elapsed,
@@ -928,7 +994,9 @@ async def lifespan(application: FastAPI):
928
994
  from superlocalmemory.server.recall_health import (
929
995
  start_recall_health_monitor,
930
996
  )
931
- _rh_thread, _rh_stop, _ = start_recall_health_monitor(engine)
997
+ _rh_thread, _rh_stop, _ = start_recall_health_monitor(
998
+ engine, runtime=profile_runtime,
999
+ )
932
1000
  application.state.recall_health_stop = _rh_stop
933
1001
  except Exception as _rh_exc:
934
1002
  logger.warning(
@@ -944,12 +1012,14 @@ async def lifespan(application: FastAPI):
944
1012
  from superlocalmemory.core.recall_queue import RecallQueue
945
1013
  _queue_db = state_path("recall_queue.db")
946
1014
  _recall_queue = RecallQueue(_queue_db)
1015
+ _engine_recall_adapter = EngineRecallAdapter(engine, profile_runtime)
947
1016
  _queue_consumer = QueueConsumer(
948
1017
  queue=_recall_queue,
949
- pool=EngineRecallAdapter(engine),
1018
+ pool=_engine_recall_adapter,
950
1019
  )
951
1020
  _queue_consumer.start()
952
1021
  application.state.queue_consumer = _queue_consumer
1022
+ application.state.engine_recall_adapter = _engine_recall_adapter
953
1023
  application.state.recall_queue = _recall_queue
954
1024
  logger.info("QueueConsumer started (recall_queue.db)")
955
1025
 
@@ -966,6 +1036,7 @@ async def lifespan(application: FastAPI):
966
1036
  except Exception as _qc_exc:
967
1037
  logger.warning("QueueConsumer start failed (non-fatal): %s", _qc_exc)
968
1038
  application.state.queue_consumer = None
1039
+ application.state.engine_recall_adapter = None
969
1040
  application.state.recall_queue = None
970
1041
 
971
1042
  except Exception:
@@ -1315,11 +1386,21 @@ async def lifespan(application: FastAPI):
1315
1386
  except Exception as exc: # pragma: no cover — defensive
1316
1387
  logger.warning("perf_log flush failed: %s", exc)
1317
1388
 
1318
- if engine is not None:
1389
+ materializer_stopped = _stop_pending_materializer()
1390
+ _profile_runtime = None
1391
+ _engine = None
1392
+ if engine is not None and materializer_stopped:
1319
1393
  try:
1320
1394
  engine.close()
1321
1395
  except Exception:
1322
1396
  pass
1397
+ elif engine is not None:
1398
+ # The process is already shutting down. Do not close a database/model
1399
+ # object still owned by an admitted background operation; OS process
1400
+ # teardown is safer than racing that writer with engine.close().
1401
+ logger.warning(
1402
+ "Engine close deferred because pending materializer is still active"
1403
+ )
1323
1404
  _cleanup_process_descriptor(
1324
1405
  getattr(application.state, "daemon_descriptor", None),
1325
1406
  )
@@ -1362,9 +1443,19 @@ def create_app() -> FastAPI:
1362
1443
  application.state.daemon_descriptor = _process_descriptor(
1363
1444
  identity_port, SLM_VERSION, "starting",
1364
1445
  )
1446
+ application.state.reconfigure_engine = (
1447
+ lambda new_config, mode_change=False: _hot_reconfigure_engine(
1448
+ application, new_config, mode_change=mode_change,
1449
+ )
1450
+ )
1365
1451
 
1366
1452
  # -- Middleware --
1453
+ from superlocalmemory.server.profile_runtime import ProfileRuntimeMiddleware
1367
1454
  from superlocalmemory.server.security_middleware import SecurityHeadersMiddleware
1455
+ application.add_middleware(
1456
+ ProfileRuntimeMiddleware,
1457
+ app_state=application.state,
1458
+ )
1368
1459
  application.add_middleware(SecurityHeadersMiddleware)
1369
1460
  application.add_middleware(GZipMiddleware, minimum_size=1000)
1370
1461
  application.add_middleware(
@@ -1635,6 +1726,7 @@ def _register_dashboard_routes(application: FastAPI) -> None:
1635
1726
  from superlocalmemory.infra.auth_middleware import (
1636
1727
  authorize_http_mcp_request,
1637
1728
  check_api_key,
1729
+ loopback_strict_mode_enabled,
1638
1730
  )
1639
1731
  from superlocalmemory.server.write_identity import (
1640
1732
  require_http_mutation_actor,
@@ -1710,25 +1802,52 @@ def _register_dashboard_routes(application: FastAPI) -> None:
1710
1802
  content={"error": str(_identity_exc.detail)},
1711
1803
  )
1712
1804
  raise
1713
- # v3.7.6 (#71/#73/#74): require_http_mutation_actor above is the
1714
- # authoritative write-auth boundary — it accepts the daemon
1715
- # capability, the dashboard install token, a matching X-SLM-API-Key,
1716
- # or an uncredentialed loopback caller, and fails closed for everyone
1717
- # else. The legacy check_api_key gate only understands X-SLM-API-Key,
1718
- # so running it as a second gate 401'd write paths that stage 1 had
1719
- # already authorized: capability-authenticated daemon write-throughs
1720
- # (MCP `remember`, #71) and install-token dashboard writes / config
1721
- # tests (#73/#74) whenever an api_key file exists. Only fall back to
1722
- # check_api_key when the mutation-actor gate did not run — i.e. for
1723
- # non-write, non-recall requests, where it is a no-op for reads.
1724
- if not requires_mutation_actor and not check_api_key(
1725
- headers, is_write=is_write
1726
- ):
1727
- from fastapi.responses import JSONResponse
1728
- return JSONResponse(
1729
- status_code=401,
1730
- content={"error": "Invalid or missing API key."},
1731
- )
1805
+ # v3.7.6 (#71/#73/#74): require_http_mutation_actor above is
1806
+ # the authoritative write-auth boundary — it accepts the
1807
+ # daemon capability, the dashboard install token, a matching
1808
+ # X-SLM-API-Key, or an uncredentialed loopback caller, and
1809
+ # fails closed for everyone else. Running check_api_key as an
1810
+ # unconditional second gate used to 401 write paths stage 1
1811
+ # had already authorized: capability-authenticated daemon
1812
+ # write-throughs (MCP `remember`, #71) and install-token
1813
+ # dashboard writes / config tests (#73/#74) whenever an
1814
+ # api_key file existed.
1815
+ #
1816
+ # v3.7.8 (F1/F2): that fix silently narrowed the api_key
1817
+ # feature — a configured api_key file used to force even
1818
+ # loopback writes to present a credential; after #71/#73/#74
1819
+ # loopback writes need none, with no opt-back-in. This is the
1820
+ # single enforcement point for the opt-in strict posture
1821
+ # (SLM_REQUIRE_API_KEY_LOOPBACK): re-run check_api_key, but
1822
+ # ONLY for the uncredentialed-loopback case — a caller who
1823
+ # presented none of the three credential headers above and
1824
+ # was authorized purely by being loopback. Credentialed
1825
+ # callers (capability / install token / api key) were already
1826
+ # validated by require_http_mutation_actor and are never
1827
+ # re-gated here, so #71/#73/#74 stay fixed.
1828
+ if loopback_strict_mode_enabled():
1829
+ _has_credential = any(
1830
+ headers.get(_h)
1831
+ for _h in (
1832
+ "x-slm-daemon-capability",
1833
+ "x-install-token",
1834
+ "x-slm-api-key",
1835
+ )
1836
+ )
1837
+ if not _has_credential and not check_api_key(
1838
+ headers, is_write=True
1839
+ ):
1840
+ from fastapi.responses import JSONResponse
1841
+ return JSONResponse(
1842
+ status_code=401,
1843
+ content={
1844
+ "error": (
1845
+ "SLM_REQUIRE_API_KEY_LOOPBACK is set: "
1846
+ "loopback writes must present a matching "
1847
+ "X-SLM-API-Key."
1848
+ )
1849
+ },
1850
+ )
1732
1851
  return await call_next(request)
1733
1852
  except Exception as _auth_exc:
1734
1853
  # v3.6.12 (failopen-1): security middleware must NEVER fail open silently.
@@ -1968,6 +2087,9 @@ def _register_daemon_routes(application: FastAPI) -> None:
1968
2087
  except Exception:
1969
2088
  _recall_health = {"recall_healthy": None}
1970
2089
  identity = getattr(application.state, "daemon_descriptor", None)
2090
+ from superlocalmemory.server.profile_runtime import get_profile_runtime
2091
+
2092
+ profile_snapshot = get_profile_runtime(application.state).snapshot
1971
2093
  return {
1972
2094
  "status": "ok",
1973
2095
  "ready": fully_ready,
@@ -1985,6 +2107,8 @@ def _register_daemon_routes(application: FastAPI) -> None:
1985
2107
  # Runtime readiness is more precise than descriptor lifecycle.
1986
2108
  # A process can be alive and identity-valid while retrieval warms.
1987
2109
  "state": runtime_state,
2110
+ "active_profile": profile_snapshot.profile_id,
2111
+ "profile_generation": profile_snapshot.generation,
1988
2112
  }
1989
2113
 
1990
2114
  @application.get("/recall")
@@ -2076,8 +2200,13 @@ def _register_daemon_routes(application: FastAPI) -> None:
2076
2200
  )
2077
2201
  for _r in results:
2078
2202
  _r["content"] = _sanitize_json_text(_r.get("content", ""))
2203
+ from superlocalmemory.server.profile_runtime import get_profile_runtime
2204
+
2205
+ profile_snapshot = get_profile_runtime(application.state).snapshot
2079
2206
  return {
2080
2207
  "ok": True,
2208
+ "profile": profile_snapshot.profile_id,
2209
+ "profile_generation": profile_snapshot.generation,
2081
2210
  "query": search_query,
2082
2211
  "query_type": response.query_type,
2083
2212
  "result_count": len(results),
@@ -2312,17 +2441,54 @@ def _register_daemon_routes(application: FastAPI) -> None:
2312
2441
  _update_activity()
2313
2442
  # Non-blocking peek — status must never force a re-init.
2314
2443
  engine = getattr(application.state, "engine", None)
2315
- fact_count = engine.fact_count if engine else 0
2316
- mode = engine._config.mode.value if engine and hasattr(engine, '_config') else "unknown"
2444
+ from superlocalmemory.server.profile_runtime import get_profile_runtime
2445
+
2446
+ profile_snapshot = get_profile_runtime(application.state).snapshot
2447
+ config = getattr(application.state, "config", None)
2448
+ fact_count = 0
2449
+ entity_count = 0
2450
+ edge_count = 0
2451
+ if engine is not None:
2452
+ try:
2453
+ fact_count = engine._db.get_fact_count(profile_snapshot.profile_id)
2454
+ entities = engine._db.execute(
2455
+ "SELECT COUNT(*) AS c FROM canonical_entities "
2456
+ "WHERE profile_id = ?",
2457
+ (profile_snapshot.profile_id,),
2458
+ )
2459
+ entity_count = int(dict(entities[0])["c"]) if entities else 0
2460
+ edges = engine._db.execute(
2461
+ "SELECT COUNT(*) AS c FROM graph_edges WHERE profile_id = ?",
2462
+ (profile_snapshot.profile_id,),
2463
+ )
2464
+ edge_count = int(dict(edges[0])["c"]) if edges else 0
2465
+ except Exception:
2466
+ logger.debug("daemon status count query failed", exc_info=True)
2467
+ db_path = getattr(config, "db_path", None)
2468
+ db_size_mb = (
2469
+ round(db_path.stat().st_size / 1024 / 1024, 2)
2470
+ if db_path is not None and db_path.exists()
2471
+ else 0.0
2472
+ )
2473
+ mode = getattr(getattr(config, "mode", None), "value", "unknown")
2474
+ provider = getattr(getattr(config, "llm", None), "provider", "") or "none"
2317
2475
  return {
2318
2476
  "status": "running",
2319
2477
  "pid": os.getpid(),
2320
2478
  "uptime_s": round(time.monotonic() - (_start_time or time.monotonic())),
2321
2479
  "mode": mode,
2480
+ "provider": provider,
2322
2481
  "fact_count": fact_count,
2482
+ "entity_count": entity_count,
2483
+ "edge_count": edge_count,
2484
+ "base_dir": str(getattr(config, "base_dir", "")),
2485
+ "db_path": str(db_path or ""),
2486
+ "db_size_mb": db_size_mb,
2323
2487
  "idle_s": round(time.monotonic() - _last_activity),
2324
2488
  "port": application.state.daemon_descriptor.port,
2325
2489
  "legacy_port": _LEGACY_PORT,
2490
+ "profile": profile_snapshot.profile_id,
2491
+ "profile_generation": profile_snapshot.generation,
2326
2492
  }
2327
2493
 
2328
2494
  @application.get("/list")
@@ -2508,6 +2674,18 @@ def _materializer_actor_id() -> str:
2508
2674
  return f"daemon-capability:{descriptor.capability_fingerprint}"
2509
2675
 
2510
2676
 
2677
+ def _run_materializer_operation(runtime, engine_supplier, operation):
2678
+ """Run one bounded background unit against an admitted engine snapshot."""
2679
+ with runtime.operation():
2680
+ # Resolve the engine only after admission. A concurrent mode/provider
2681
+ # reconfiguration may have replaced the module-level engine while this
2682
+ # worker was waiting at the transition barrier.
2683
+ engine = engine_supplier()
2684
+ if engine is None:
2685
+ return None
2686
+ return operation(engine)
2687
+
2688
+
2511
2689
  def _materialize_ingestion_one_pass(
2512
2690
  engine,
2513
2691
  *,
@@ -2609,6 +2787,10 @@ def _start_pending_materializer() -> None:
2609
2787
  """Drain M018 operations and backfill the legacy pending.db queue."""
2610
2788
  global _materializer_thread
2611
2789
 
2790
+ if _materializer_thread is not None and _materializer_thread.is_alive():
2791
+ return
2792
+ _materializer_stop.clear()
2793
+
2612
2794
  def _loop():
2613
2795
  from superlocalmemory.cli.pending_store import (
2614
2796
  get_pending, mark_done, mark_failed,
@@ -2624,9 +2806,12 @@ def _start_pending_materializer() -> None:
2624
2806
  # not a stale local reference.
2625
2807
  import superlocalmemory.server.unified_daemon as _ud
2626
2808
  engine = _ud._engine
2627
- if engine is None:
2809
+ runtime = _ud._profile_runtime
2810
+ if engine is None or runtime is None:
2628
2811
  if not _waiting_logged:
2629
- logger.info("Materializer: waiting for engine to init...")
2812
+ logger.info(
2813
+ "Materializer: waiting for engine/runtime to init..."
2814
+ )
2630
2815
  _waiting_logged = True
2631
2816
  time.sleep(0.5)
2632
2817
  continue
@@ -2634,10 +2819,18 @@ def _start_pending_materializer() -> None:
2634
2819
  logger.info("Materializer: engine acquired, starting drain loop")
2635
2820
  _engine_logged = True
2636
2821
 
2637
- durable_complete, durable_failed = _materialize_ingestion_one_pass(
2638
- engine,
2639
- limit=50,
2822
+ cycle_result = _run_materializer_operation(
2823
+ runtime,
2824
+ lambda: _ud._engine,
2825
+ lambda admitted_engine: _materialize_ingestion_one_pass(
2826
+ admitted_engine,
2827
+ # One operation per lease bounds profile-switch wait
2828
+ # time without allowing engine components to rebind
2829
+ # halfway through an enrichment pipeline.
2830
+ limit=1,
2831
+ ),
2640
2832
  )
2833
+ durable_complete, durable_failed = cycle_result or (0, 0)
2641
2834
  pending = get_pending(limit=50)
2642
2835
  if not pending and not durable_complete and not durable_failed:
2643
2836
  time.sleep(1.0)
@@ -2655,7 +2848,15 @@ def _start_pending_materializer() -> None:
2655
2848
  time.sleep(0.5)
2656
2849
  waits += 1
2657
2850
  try:
2658
- operation_id = _materialize_legacy_pending_item(engine, item)
2851
+ operation_id = _run_materializer_operation(
2852
+ runtime,
2853
+ lambda: _ud._engine,
2854
+ lambda admitted_engine: _materialize_legacy_pending_item(
2855
+ admitted_engine, item,
2856
+ ),
2857
+ )
2858
+ if operation_id is None:
2859
+ raise RuntimeError("resident engine became unavailable")
2659
2860
  mark_done(item["id"])
2660
2861
  _emit_event(
2661
2862
  "memory.stored",
@@ -2683,6 +2884,21 @@ def _start_pending_materializer() -> None:
2683
2884
  logger.info("Pending materializer started (recall-priority)")
2684
2885
 
2685
2886
 
2887
+ def _stop_pending_materializer(timeout: float = 5.0) -> bool:
2888
+ """Stop and join the background writer before closing its engine."""
2889
+ global _materializer_thread
2890
+
2891
+ _materializer_stop.set()
2892
+ thread = _materializer_thread
2893
+ if thread is not None and thread.is_alive():
2894
+ thread.join(timeout=timeout)
2895
+ if thread.is_alive():
2896
+ logger.warning("Pending materializer did not stop within %.1fs", timeout)
2897
+ return False
2898
+ _materializer_thread = None
2899
+ return True
2900
+
2901
+
2686
2902
  def start_server(port: int = _DEFAULT_PORT) -> None:
2687
2903
  """Start the unified daemon. Blocks until stopped."""
2688
2904
  global _start_time
@@ -120,6 +120,17 @@ _MODULES = {
120
120
 
121
121
  logger = logging.getLogger(__name__)
122
122
 
123
+ # Exact historical DDL fingerprints whose resulting schema is intentionally
124
+ # accepted by the current migration. Unknown hashes are never reconciled.
125
+ _KNOWN_EQUIVALENT_DDL_HASHES: dict[str, frozenset[str]] = {
126
+ _M002.NAME: frozenset({
127
+ # v3.4.21 hardened copy-forward variant.
128
+ "347eeb2ec8aac89f7cbf373da49ac9446be9ed150e6105c382c656cd22426d4b",
129
+ # v3.4.22 model_version-default variant shipped through 3.6.x.
130
+ "d28666fa1dfa66e6514efd288e6748363513da2255a4cee95d80f233e6728ae7",
131
+ }),
132
+ }
133
+
123
134
 
124
135
  @dataclass(frozen=True, slots=True)
125
136
  class Migration:
@@ -307,11 +318,14 @@ def _apply_single(
307
318
  # in place, reconcile the log to the current hash and treat as
308
319
  # already-applied instead of failing the daemon into permanent
309
320
  # not_ready. Absent/failing verify keeps the hard failure.
321
+ allowed_hashes = _KNOWN_EQUIVALENT_DDL_HASHES.get(
322
+ migration.name, frozenset(),
323
+ )
310
324
  mod = _MODULES.get(migration.name)
311
325
  verify_fn = (
312
326
  getattr(mod, "verify", None) if mod is not None else None
313
327
  )
314
- if verify_fn is not None:
328
+ if logged_hash in allowed_hashes and verify_fn is not None:
315
329
  try:
316
330
  if verify_fn(conn):
317
331
  if not dry_run:
@@ -323,8 +337,8 @@ def _apply_single(
323
337
  pass
324
338
  return (
325
339
  "skipped",
326
- "drift reconciled via verify schema present, "
327
- "log re-hashed to current DDL",
340
+ "allowlisted historical DDL reconciled after "
341
+ "full schema verification",
328
342
  )
329
343
  except sqlite3.Error: # pragma: no cover
330
344
  pass
@@ -29,14 +29,43 @@ _REQUIRED_COLS = frozenset({
29
29
 
30
30
 
31
31
  def verify(conn: sqlite3.Connection) -> bool:
32
- """Return True if the rebuilt model_state schema is in place."""
32
+ """Verify columns plus both indexes promised by this migration."""
33
33
  try:
34
- cols = {r[1] for r in conn.execute(
34
+ cols = {r[1]: r for r in conn.execute(
35
35
  "PRAGMA table_info(learning_model_state)"
36
36
  ).fetchall()}
37
+ index_rows = conn.execute(
38
+ "PRAGMA index_list(learning_model_state)"
39
+ ).fetchall()
37
40
  except sqlite3.Error:
38
41
  return False
39
- return _REQUIRED_COLS <= cols
42
+ if not _REQUIRED_COLS <= set(cols):
43
+ return False
44
+
45
+ indexes = {row[1]: row for row in index_rows}
46
+ active = indexes.get("idx_model_active")
47
+ profile_time = indexes.get("idx_model_profile_time")
48
+ if active is None or profile_time is None:
49
+ return False
50
+ # idx_model_active must remain a UNIQUE partial index.
51
+ if int(active[2]) != 1 or int(active[4]) != 1:
52
+ return False
53
+ active_cols = [row[2] for row in conn.execute(
54
+ "PRAGMA index_info(idx_model_active)"
55
+ ).fetchall()]
56
+ time_cols = [row[2] for row in conn.execute(
57
+ "PRAGMA index_info(idx_model_profile_time)"
58
+ ).fetchall()]
59
+ if active_cols != ["profile_id"]:
60
+ return False
61
+ if time_cols != ["profile_id", "trained_at"]:
62
+ return False
63
+ sql_row = conn.execute(
64
+ "SELECT sql FROM sqlite_master WHERE type='index' AND name=?",
65
+ ("idx_model_active",),
66
+ ).fetchone()
67
+ normalized = " ".join(str(sql_row[0] if sql_row else "").lower().split())
68
+ return "where is_active = 1" in normalized
40
69
 
41
70
 
42
71
  # IMPORTANT: this DDL shipped in V3.4.21. Migration hashes are immutable
@@ -365,15 +365,6 @@ V32_DDL: list[str] = [
365
365
  """,
366
366
  ]
367
367
 
368
- # vec0 virtual table DDL — executed by VectorStore ONLY (requires extension loaded first).
369
- # NOT in V32_DDL because executescript cannot load extensions mid-script.
370
- V32_VEC0_DDL: Final[str] = """
371
- CREATE VIRTUAL TABLE IF NOT EXISTS fact_embeddings USING vec0(
372
- profile_id TEXT PARTITION KEY,
373
- embedding float[768] distance_metric=cosine
374
- );
375
- """
376
-
377
368
  # ---------------------------------------------------------------------------
378
369
  # Rollback DDL (reverse FK order -- Rule 20)
379
370
  # ---------------------------------------------------------------------------