superlocalmemory 3.7.6 → 3.7.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/README.md +2 -2
- package/package.json +2 -2
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/requirements.txt +1 -1
- package/plugin-src/manifest.json +1 -1
- package/plugin-src/requirements.txt +1 -1
- package/pyproject.toml +6 -7
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/cli/commands.py +171 -11
- package/src/superlocalmemory/cli/setup_wizard.py +18 -1
- package/src/superlocalmemory/infra/auth_middleware.py +33 -5
- package/src/superlocalmemory/mcp/_daemon_proxy.py +8 -10
- package/src/superlocalmemory/mcp/server.py +1 -0
- package/src/superlocalmemory/mcp/tools_core.py +216 -20
- package/src/superlocalmemory/optimize/cache/centroid_store.py +21 -3
- package/src/superlocalmemory/optimize/cache/manager.py +7 -0
- package/src/superlocalmemory/optimize/cache/semantic.py +27 -10
- package/src/superlocalmemory/server/api.py +17 -0
- package/src/superlocalmemory/server/profile_runtime.py +384 -0
- package/src/superlocalmemory/server/recall_health.py +12 -6
- package/src/superlocalmemory/server/routes/chat.py +63 -12
- package/src/superlocalmemory/server/routes/helpers.py +9 -16
- package/src/superlocalmemory/server/routes/memories.py +58 -11
- package/src/superlocalmemory/server/routes/profiles.py +24 -14
- package/src/superlocalmemory/server/routes/v3_api.py +128 -52
- package/src/superlocalmemory/server/ui.py +10 -0
- package/src/superlocalmemory/server/unified_daemon.py +290 -74
- package/src/superlocalmemory/storage/migration_runner.py +17 -3
- package/src/superlocalmemory/storage/migrations/M002_model_state_history.py +32 -3
- package/src/superlocalmemory/storage/schema_v32.py +0 -9
- package/src/superlocalmemory/ui/index.html +32 -1
- package/src/superlocalmemory/ui/js/auto-settings.js +48 -0
- package/src/superlocalmemory/ui/js/profiles.js +11 -2
|
@@ -182,10 +182,26 @@ class EngineRecallAdapter:
|
|
|
182
182
|
loaded a SECOND MemoryEngine. This adapter eliminates that duplication.
|
|
183
183
|
"""
|
|
184
184
|
|
|
185
|
-
def __init__(self, engine) -> None:
|
|
185
|
+
def __init__(self, engine, profile_runtime=None) -> None:
|
|
186
|
+
self._engine = engine
|
|
187
|
+
self._profile_runtime = profile_runtime
|
|
188
|
+
|
|
189
|
+
def set_engine(self, engine) -> None:
|
|
190
|
+
"""Replace the engine while the profile runtime is exclusive."""
|
|
186
191
|
self._engine = engine
|
|
187
192
|
|
|
188
193
|
def recall(self, query: str, limit: int = 10, session_id: str = "") -> dict:
|
|
194
|
+
from contextlib import nullcontext
|
|
195
|
+
|
|
196
|
+
lease = (
|
|
197
|
+
self._profile_runtime.operation()
|
|
198
|
+
if self._profile_runtime is not None
|
|
199
|
+
else nullcontext()
|
|
200
|
+
)
|
|
201
|
+
with lease:
|
|
202
|
+
return self._recall(query, limit=limit, session_id=session_id)
|
|
203
|
+
|
|
204
|
+
def _recall(self, query: str, limit: int = 10, session_id: str = "") -> dict:
|
|
189
205
|
response = self._engine.recall(
|
|
190
206
|
query, limit=limit, session_id=session_id or None,
|
|
191
207
|
)
|
|
@@ -229,6 +245,84 @@ class EngineRecallAdapter:
|
|
|
229
245
|
}
|
|
230
246
|
|
|
231
247
|
|
|
248
|
+
def _configure_scale_backends(engine, config) -> None:
|
|
249
|
+
"""Attach optional graph/vector backends to one initialized engine."""
|
|
250
|
+
try:
|
|
251
|
+
from superlocalmemory.core.backend_orchestrator import (
|
|
252
|
+
BackendOrchestrator,
|
|
253
|
+
set_orchestrator,
|
|
254
|
+
)
|
|
255
|
+
|
|
256
|
+
orchestrator = BackendOrchestrator(config=config, db=engine._db)
|
|
257
|
+
orchestrator.on_daemon_start()
|
|
258
|
+
set_orchestrator(orchestrator)
|
|
259
|
+
cozo_backend = orchestrator.get_graph_backend()
|
|
260
|
+
lancedb_backend = orchestrator.get_vector_backend()
|
|
261
|
+
retrieval = getattr(engine, "_retrieval_engine", None)
|
|
262
|
+
if retrieval is not None:
|
|
263
|
+
entity_graph = getattr(retrieval, "_entity", None)
|
|
264
|
+
if (
|
|
265
|
+
entity_graph is not None
|
|
266
|
+
and cozo_backend is not None
|
|
267
|
+
and orchestrator.graph_retrieval_ready()
|
|
268
|
+
):
|
|
269
|
+
entity_graph._cozo = cozo_backend
|
|
270
|
+
semantic = getattr(retrieval, "_semantic", None)
|
|
271
|
+
if semantic is not None and lancedb_backend is not None:
|
|
272
|
+
semantic.set_scale_vector_backend(lancedb_backend)
|
|
273
|
+
logger.info(
|
|
274
|
+
"BackendOrchestrator: ready (cozo=%s, lancedb=%s)",
|
|
275
|
+
"active" if cozo_backend else "off",
|
|
276
|
+
"active" if lancedb_backend else "off",
|
|
277
|
+
)
|
|
278
|
+
except Exception as exc:
|
|
279
|
+
logger.warning("BackendOrchestrator init failed (non-fatal): %s", exc)
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def _hot_reconfigure_engine(application, new_config, *, mode_change: bool) -> None:
|
|
283
|
+
"""Build and publish one coherent daemon engine for a saved config."""
|
|
284
|
+
from superlocalmemory.core.engine import MemoryEngine
|
|
285
|
+
|
|
286
|
+
old_engine = getattr(application.state, "engine", None)
|
|
287
|
+
new_engine = MemoryEngine(new_config)
|
|
288
|
+
try:
|
|
289
|
+
new_engine.initialize()
|
|
290
|
+
new_config.save(mode_change=mode_change)
|
|
291
|
+
except BaseException:
|
|
292
|
+
new_engine.close()
|
|
293
|
+
raise
|
|
294
|
+
|
|
295
|
+
# The profile transition barrier is exclusive here. Publish every
|
|
296
|
+
# long-lived reference before closing the former engine.
|
|
297
|
+
application.state.engine = new_engine
|
|
298
|
+
application.state.config = new_config
|
|
299
|
+
global _engine
|
|
300
|
+
_engine = new_engine
|
|
301
|
+
_observe_buffer.set_engine(new_engine)
|
|
302
|
+
adapter = getattr(application.state, "engine_recall_adapter", None)
|
|
303
|
+
if adapter is not None:
|
|
304
|
+
adapter.set_engine(new_engine)
|
|
305
|
+
_configure_scale_backends(new_engine, new_config)
|
|
306
|
+
|
|
307
|
+
old_health_stop = getattr(application.state, "recall_health_stop", None)
|
|
308
|
+
if old_health_stop is not None:
|
|
309
|
+
old_health_stop.set()
|
|
310
|
+
try:
|
|
311
|
+
from superlocalmemory.server.recall_health import start_recall_health_monitor
|
|
312
|
+
|
|
313
|
+
runtime = getattr(application.state, "profile_runtime", None)
|
|
314
|
+
_thread, health_stop, _state = start_recall_health_monitor(
|
|
315
|
+
new_engine, runtime=runtime,
|
|
316
|
+
)
|
|
317
|
+
application.state.recall_health_stop = health_stop
|
|
318
|
+
except Exception as exc:
|
|
319
|
+
application.state.recall_health_stop = None
|
|
320
|
+
logger.warning("recall-health restart failed (non-fatal): %s", exc)
|
|
321
|
+
|
|
322
|
+
if old_engine is not None and old_engine is not new_engine:
|
|
323
|
+
old_engine.close()
|
|
324
|
+
|
|
325
|
+
|
|
232
326
|
# ---------------------------------------------------------------------------
|
|
233
327
|
# v3.4.32: Recall-priority gate for the pending materializer.
|
|
234
328
|
# All /remember writes go to pending.db and return fast; a background
|
|
@@ -248,6 +342,7 @@ from superlocalmemory.core.recall_gate import (
|
|
|
248
342
|
# of pending memories — they accumulated forever, only being processed at
|
|
249
343
|
# daemon startup via engine._process_pending_memories().
|
|
250
344
|
_engine = None
|
|
345
|
+
_profile_runtime = None
|
|
251
346
|
|
|
252
347
|
|
|
253
348
|
def _emit_event(
|
|
@@ -679,54 +774,24 @@ async def lifespan(application: FastAPI):
|
|
|
679
774
|
except Exception:
|
|
680
775
|
pass
|
|
681
776
|
|
|
682
|
-
|
|
683
|
-
|
|
777
|
+
from superlocalmemory.server.profile_runtime import bind_profile_runtime
|
|
778
|
+
|
|
779
|
+
profile_runtime = bind_profile_runtime(application.state, engine, config)
|
|
780
|
+
application.state.reconfigure_engine = (
|
|
781
|
+
lambda new_config, mode_change=False: _hot_reconfigure_engine(
|
|
782
|
+
application, new_config, mode_change=mode_change,
|
|
783
|
+
)
|
|
784
|
+
)
|
|
684
785
|
# v3.4.38: Wire module-level _engine for the pending materializer.
|
|
685
|
-
global _engine
|
|
786
|
+
global _engine, _profile_runtime
|
|
787
|
+
_profile_runtime = profile_runtime
|
|
686
788
|
_engine = engine
|
|
687
789
|
logger.info("Unified daemon: MemoryEngine initialized (mode=%s)", config.mode.value)
|
|
688
790
|
|
|
689
791
|
# v3.5.0: Backend Orchestrator — CozoDB (graph) + LanceDB (vector) backends.
|
|
690
792
|
# Initialise AFTER engine so the retrieval channels exist to receive backends.
|
|
691
793
|
# Migrates edges/embeddings automatically; fail-soft (non-blocking).
|
|
692
|
-
|
|
693
|
-
_lancedb_backend = None
|
|
694
|
-
try:
|
|
695
|
-
from superlocalmemory.core.backend_orchestrator import (
|
|
696
|
-
BackendOrchestrator, set_orchestrator,
|
|
697
|
-
)
|
|
698
|
-
orch = BackendOrchestrator(config=config, db=engine._db)
|
|
699
|
-
orch.on_daemon_start()
|
|
700
|
-
set_orchestrator(orch)
|
|
701
|
-
_cozo_backend = orch.get_graph_backend()
|
|
702
|
-
_lancedb_backend = orch.get_vector_backend()
|
|
703
|
-
# Cozo storage may be active before its canonical-entity retrieval
|
|
704
|
-
# projection is parity-proven. Never route mismatched ID spaces.
|
|
705
|
-
re = getattr(engine, '_retrieval_engine', None)
|
|
706
|
-
if re is not None:
|
|
707
|
-
eg = getattr(re, '_entity', None)
|
|
708
|
-
if (
|
|
709
|
-
eg is not None
|
|
710
|
-
and _cozo_backend is not None
|
|
711
|
-
and orch.graph_retrieval_ready()
|
|
712
|
-
):
|
|
713
|
-
try:
|
|
714
|
-
eg._cozo = _cozo_backend
|
|
715
|
-
logger.info("CozoDB backend wired into entity_graph channel")
|
|
716
|
-
except Exception as exc:
|
|
717
|
-
logger.warning("CozoDB channel injection failed: %s", exc)
|
|
718
|
-
semantic = getattr(re, '_semantic', None)
|
|
719
|
-
if semantic is not None and _lancedb_backend is not None:
|
|
720
|
-
try:
|
|
721
|
-
semantic.set_scale_vector_backend(_lancedb_backend)
|
|
722
|
-
logger.info("LanceDB backend wired into semantic channel with SQLite shadow")
|
|
723
|
-
except Exception as exc:
|
|
724
|
-
logger.warning("LanceDB channel injection failed: %s", exc)
|
|
725
|
-
logger.info("BackendOrchestrator: ready (cozo=%s, lancedb=%s)",
|
|
726
|
-
"active" if _cozo_backend else "off",
|
|
727
|
-
"active" if _lancedb_backend else "off")
|
|
728
|
-
except Exception as exc:
|
|
729
|
-
logger.warning("BackendOrchestrator init failed (non-fatal): %s", exc)
|
|
794
|
+
_configure_scale_backends(engine, config)
|
|
730
795
|
|
|
731
796
|
# LLD-07 §4 — deferred migrations (e.g. M006 reward column) need to
|
|
732
797
|
# run AFTER MemoryEngine.initialize() has bootstrapped runtime tables
|
|
@@ -855,8 +920,9 @@ async def lifespan(application: FastAPI):
|
|
|
855
920
|
# Fire 2 warmup queries: one to load the graph page cache,
|
|
856
921
|
# second to warm the reranker subprocess + all producers.
|
|
857
922
|
# Without this, dashboard POST /api/search hits 11s cold.
|
|
858
|
-
|
|
859
|
-
|
|
923
|
+
with profile_runtime.operation():
|
|
924
|
+
for wq in ("memory recall performance", "context injection retrieval"):
|
|
925
|
+
engine.recall(wq, limit=5)
|
|
860
926
|
elapsed = round((_t.monotonic() - t0) * 1000)
|
|
861
927
|
logger.info(
|
|
862
928
|
"Recall engine pre-warmed in %dms", elapsed,
|
|
@@ -928,7 +994,9 @@ async def lifespan(application: FastAPI):
|
|
|
928
994
|
from superlocalmemory.server.recall_health import (
|
|
929
995
|
start_recall_health_monitor,
|
|
930
996
|
)
|
|
931
|
-
_rh_thread, _rh_stop, _ = start_recall_health_monitor(
|
|
997
|
+
_rh_thread, _rh_stop, _ = start_recall_health_monitor(
|
|
998
|
+
engine, runtime=profile_runtime,
|
|
999
|
+
)
|
|
932
1000
|
application.state.recall_health_stop = _rh_stop
|
|
933
1001
|
except Exception as _rh_exc:
|
|
934
1002
|
logger.warning(
|
|
@@ -944,12 +1012,14 @@ async def lifespan(application: FastAPI):
|
|
|
944
1012
|
from superlocalmemory.core.recall_queue import RecallQueue
|
|
945
1013
|
_queue_db = state_path("recall_queue.db")
|
|
946
1014
|
_recall_queue = RecallQueue(_queue_db)
|
|
1015
|
+
_engine_recall_adapter = EngineRecallAdapter(engine, profile_runtime)
|
|
947
1016
|
_queue_consumer = QueueConsumer(
|
|
948
1017
|
queue=_recall_queue,
|
|
949
|
-
pool=
|
|
1018
|
+
pool=_engine_recall_adapter,
|
|
950
1019
|
)
|
|
951
1020
|
_queue_consumer.start()
|
|
952
1021
|
application.state.queue_consumer = _queue_consumer
|
|
1022
|
+
application.state.engine_recall_adapter = _engine_recall_adapter
|
|
953
1023
|
application.state.recall_queue = _recall_queue
|
|
954
1024
|
logger.info("QueueConsumer started (recall_queue.db)")
|
|
955
1025
|
|
|
@@ -966,6 +1036,7 @@ async def lifespan(application: FastAPI):
|
|
|
966
1036
|
except Exception as _qc_exc:
|
|
967
1037
|
logger.warning("QueueConsumer start failed (non-fatal): %s", _qc_exc)
|
|
968
1038
|
application.state.queue_consumer = None
|
|
1039
|
+
application.state.engine_recall_adapter = None
|
|
969
1040
|
application.state.recall_queue = None
|
|
970
1041
|
|
|
971
1042
|
except Exception:
|
|
@@ -1315,11 +1386,21 @@ async def lifespan(application: FastAPI):
|
|
|
1315
1386
|
except Exception as exc: # pragma: no cover — defensive
|
|
1316
1387
|
logger.warning("perf_log flush failed: %s", exc)
|
|
1317
1388
|
|
|
1318
|
-
|
|
1389
|
+
materializer_stopped = _stop_pending_materializer()
|
|
1390
|
+
_profile_runtime = None
|
|
1391
|
+
_engine = None
|
|
1392
|
+
if engine is not None and materializer_stopped:
|
|
1319
1393
|
try:
|
|
1320
1394
|
engine.close()
|
|
1321
1395
|
except Exception:
|
|
1322
1396
|
pass
|
|
1397
|
+
elif engine is not None:
|
|
1398
|
+
# The process is already shutting down. Do not close a database/model
|
|
1399
|
+
# object still owned by an admitted background operation; OS process
|
|
1400
|
+
# teardown is safer than racing that writer with engine.close().
|
|
1401
|
+
logger.warning(
|
|
1402
|
+
"Engine close deferred because pending materializer is still active"
|
|
1403
|
+
)
|
|
1323
1404
|
_cleanup_process_descriptor(
|
|
1324
1405
|
getattr(application.state, "daemon_descriptor", None),
|
|
1325
1406
|
)
|
|
@@ -1362,9 +1443,19 @@ def create_app() -> FastAPI:
|
|
|
1362
1443
|
application.state.daemon_descriptor = _process_descriptor(
|
|
1363
1444
|
identity_port, SLM_VERSION, "starting",
|
|
1364
1445
|
)
|
|
1446
|
+
application.state.reconfigure_engine = (
|
|
1447
|
+
lambda new_config, mode_change=False: _hot_reconfigure_engine(
|
|
1448
|
+
application, new_config, mode_change=mode_change,
|
|
1449
|
+
)
|
|
1450
|
+
)
|
|
1365
1451
|
|
|
1366
1452
|
# -- Middleware --
|
|
1453
|
+
from superlocalmemory.server.profile_runtime import ProfileRuntimeMiddleware
|
|
1367
1454
|
from superlocalmemory.server.security_middleware import SecurityHeadersMiddleware
|
|
1455
|
+
application.add_middleware(
|
|
1456
|
+
ProfileRuntimeMiddleware,
|
|
1457
|
+
app_state=application.state,
|
|
1458
|
+
)
|
|
1368
1459
|
application.add_middleware(SecurityHeadersMiddleware)
|
|
1369
1460
|
application.add_middleware(GZipMiddleware, minimum_size=1000)
|
|
1370
1461
|
application.add_middleware(
|
|
@@ -1635,6 +1726,7 @@ def _register_dashboard_routes(application: FastAPI) -> None:
|
|
|
1635
1726
|
from superlocalmemory.infra.auth_middleware import (
|
|
1636
1727
|
authorize_http_mcp_request,
|
|
1637
1728
|
check_api_key,
|
|
1729
|
+
loopback_strict_mode_enabled,
|
|
1638
1730
|
)
|
|
1639
1731
|
from superlocalmemory.server.write_identity import (
|
|
1640
1732
|
require_http_mutation_actor,
|
|
@@ -1710,25 +1802,52 @@ def _register_dashboard_routes(application: FastAPI) -> None:
|
|
|
1710
1802
|
content={"error": str(_identity_exc.detail)},
|
|
1711
1803
|
)
|
|
1712
1804
|
raise
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1805
|
+
# v3.7.6 (#71/#73/#74): require_http_mutation_actor above is
|
|
1806
|
+
# the authoritative write-auth boundary — it accepts the
|
|
1807
|
+
# daemon capability, the dashboard install token, a matching
|
|
1808
|
+
# X-SLM-API-Key, or an uncredentialed loopback caller, and
|
|
1809
|
+
# fails closed for everyone else. Running check_api_key as an
|
|
1810
|
+
# unconditional second gate used to 401 write paths stage 1
|
|
1811
|
+
# had already authorized: capability-authenticated daemon
|
|
1812
|
+
# write-throughs (MCP `remember`, #71) and install-token
|
|
1813
|
+
# dashboard writes / config tests (#73/#74) whenever an
|
|
1814
|
+
# api_key file existed.
|
|
1815
|
+
#
|
|
1816
|
+
# v3.7.8 (F1/F2): that fix silently narrowed the api_key
|
|
1817
|
+
# feature — a configured api_key file used to force even
|
|
1818
|
+
# loopback writes to present a credential; after #71/#73/#74
|
|
1819
|
+
# loopback writes need none, with no opt-back-in. This is the
|
|
1820
|
+
# single enforcement point for the opt-in strict posture
|
|
1821
|
+
# (SLM_REQUIRE_API_KEY_LOOPBACK): re-run check_api_key, but
|
|
1822
|
+
# ONLY for the uncredentialed-loopback case — a caller who
|
|
1823
|
+
# presented none of the three credential headers above and
|
|
1824
|
+
# was authorized purely by being loopback. Credentialed
|
|
1825
|
+
# callers (capability / install token / api key) were already
|
|
1826
|
+
# validated by require_http_mutation_actor and are never
|
|
1827
|
+
# re-gated here, so #71/#73/#74 stay fixed.
|
|
1828
|
+
if loopback_strict_mode_enabled():
|
|
1829
|
+
_has_credential = any(
|
|
1830
|
+
headers.get(_h)
|
|
1831
|
+
for _h in (
|
|
1832
|
+
"x-slm-daemon-capability",
|
|
1833
|
+
"x-install-token",
|
|
1834
|
+
"x-slm-api-key",
|
|
1835
|
+
)
|
|
1836
|
+
)
|
|
1837
|
+
if not _has_credential and not check_api_key(
|
|
1838
|
+
headers, is_write=True
|
|
1839
|
+
):
|
|
1840
|
+
from fastapi.responses import JSONResponse
|
|
1841
|
+
return JSONResponse(
|
|
1842
|
+
status_code=401,
|
|
1843
|
+
content={
|
|
1844
|
+
"error": (
|
|
1845
|
+
"SLM_REQUIRE_API_KEY_LOOPBACK is set: "
|
|
1846
|
+
"loopback writes must present a matching "
|
|
1847
|
+
"X-SLM-API-Key."
|
|
1848
|
+
)
|
|
1849
|
+
},
|
|
1850
|
+
)
|
|
1732
1851
|
return await call_next(request)
|
|
1733
1852
|
except Exception as _auth_exc:
|
|
1734
1853
|
# v3.6.12 (failopen-1): security middleware must NEVER fail open silently.
|
|
@@ -1968,6 +2087,9 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
1968
2087
|
except Exception:
|
|
1969
2088
|
_recall_health = {"recall_healthy": None}
|
|
1970
2089
|
identity = getattr(application.state, "daemon_descriptor", None)
|
|
2090
|
+
from superlocalmemory.server.profile_runtime import get_profile_runtime
|
|
2091
|
+
|
|
2092
|
+
profile_snapshot = get_profile_runtime(application.state).snapshot
|
|
1971
2093
|
return {
|
|
1972
2094
|
"status": "ok",
|
|
1973
2095
|
"ready": fully_ready,
|
|
@@ -1985,6 +2107,8 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
1985
2107
|
# Runtime readiness is more precise than descriptor lifecycle.
|
|
1986
2108
|
# A process can be alive and identity-valid while retrieval warms.
|
|
1987
2109
|
"state": runtime_state,
|
|
2110
|
+
"active_profile": profile_snapshot.profile_id,
|
|
2111
|
+
"profile_generation": profile_snapshot.generation,
|
|
1988
2112
|
}
|
|
1989
2113
|
|
|
1990
2114
|
@application.get("/recall")
|
|
@@ -2076,8 +2200,13 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
2076
2200
|
)
|
|
2077
2201
|
for _r in results:
|
|
2078
2202
|
_r["content"] = _sanitize_json_text(_r.get("content", ""))
|
|
2203
|
+
from superlocalmemory.server.profile_runtime import get_profile_runtime
|
|
2204
|
+
|
|
2205
|
+
profile_snapshot = get_profile_runtime(application.state).snapshot
|
|
2079
2206
|
return {
|
|
2080
2207
|
"ok": True,
|
|
2208
|
+
"profile": profile_snapshot.profile_id,
|
|
2209
|
+
"profile_generation": profile_snapshot.generation,
|
|
2081
2210
|
"query": search_query,
|
|
2082
2211
|
"query_type": response.query_type,
|
|
2083
2212
|
"result_count": len(results),
|
|
@@ -2312,17 +2441,54 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
2312
2441
|
_update_activity()
|
|
2313
2442
|
# Non-blocking peek — status must never force a re-init.
|
|
2314
2443
|
engine = getattr(application.state, "engine", None)
|
|
2315
|
-
|
|
2316
|
-
|
|
2444
|
+
from superlocalmemory.server.profile_runtime import get_profile_runtime
|
|
2445
|
+
|
|
2446
|
+
profile_snapshot = get_profile_runtime(application.state).snapshot
|
|
2447
|
+
config = getattr(application.state, "config", None)
|
|
2448
|
+
fact_count = 0
|
|
2449
|
+
entity_count = 0
|
|
2450
|
+
edge_count = 0
|
|
2451
|
+
if engine is not None:
|
|
2452
|
+
try:
|
|
2453
|
+
fact_count = engine._db.get_fact_count(profile_snapshot.profile_id)
|
|
2454
|
+
entities = engine._db.execute(
|
|
2455
|
+
"SELECT COUNT(*) AS c FROM canonical_entities "
|
|
2456
|
+
"WHERE profile_id = ?",
|
|
2457
|
+
(profile_snapshot.profile_id,),
|
|
2458
|
+
)
|
|
2459
|
+
entity_count = int(dict(entities[0])["c"]) if entities else 0
|
|
2460
|
+
edges = engine._db.execute(
|
|
2461
|
+
"SELECT COUNT(*) AS c FROM graph_edges WHERE profile_id = ?",
|
|
2462
|
+
(profile_snapshot.profile_id,),
|
|
2463
|
+
)
|
|
2464
|
+
edge_count = int(dict(edges[0])["c"]) if edges else 0
|
|
2465
|
+
except Exception:
|
|
2466
|
+
logger.debug("daemon status count query failed", exc_info=True)
|
|
2467
|
+
db_path = getattr(config, "db_path", None)
|
|
2468
|
+
db_size_mb = (
|
|
2469
|
+
round(db_path.stat().st_size / 1024 / 1024, 2)
|
|
2470
|
+
if db_path is not None and db_path.exists()
|
|
2471
|
+
else 0.0
|
|
2472
|
+
)
|
|
2473
|
+
mode = getattr(getattr(config, "mode", None), "value", "unknown")
|
|
2474
|
+
provider = getattr(getattr(config, "llm", None), "provider", "") or "none"
|
|
2317
2475
|
return {
|
|
2318
2476
|
"status": "running",
|
|
2319
2477
|
"pid": os.getpid(),
|
|
2320
2478
|
"uptime_s": round(time.monotonic() - (_start_time or time.monotonic())),
|
|
2321
2479
|
"mode": mode,
|
|
2480
|
+
"provider": provider,
|
|
2322
2481
|
"fact_count": fact_count,
|
|
2482
|
+
"entity_count": entity_count,
|
|
2483
|
+
"edge_count": edge_count,
|
|
2484
|
+
"base_dir": str(getattr(config, "base_dir", "")),
|
|
2485
|
+
"db_path": str(db_path or ""),
|
|
2486
|
+
"db_size_mb": db_size_mb,
|
|
2323
2487
|
"idle_s": round(time.monotonic() - _last_activity),
|
|
2324
2488
|
"port": application.state.daemon_descriptor.port,
|
|
2325
2489
|
"legacy_port": _LEGACY_PORT,
|
|
2490
|
+
"profile": profile_snapshot.profile_id,
|
|
2491
|
+
"profile_generation": profile_snapshot.generation,
|
|
2326
2492
|
}
|
|
2327
2493
|
|
|
2328
2494
|
@application.get("/list")
|
|
@@ -2508,6 +2674,18 @@ def _materializer_actor_id() -> str:
|
|
|
2508
2674
|
return f"daemon-capability:{descriptor.capability_fingerprint}"
|
|
2509
2675
|
|
|
2510
2676
|
|
|
2677
|
+
def _run_materializer_operation(runtime, engine_supplier, operation):
|
|
2678
|
+
"""Run one bounded background unit against an admitted engine snapshot."""
|
|
2679
|
+
with runtime.operation():
|
|
2680
|
+
# Resolve the engine only after admission. A concurrent mode/provider
|
|
2681
|
+
# reconfiguration may have replaced the module-level engine while this
|
|
2682
|
+
# worker was waiting at the transition barrier.
|
|
2683
|
+
engine = engine_supplier()
|
|
2684
|
+
if engine is None:
|
|
2685
|
+
return None
|
|
2686
|
+
return operation(engine)
|
|
2687
|
+
|
|
2688
|
+
|
|
2511
2689
|
def _materialize_ingestion_one_pass(
|
|
2512
2690
|
engine,
|
|
2513
2691
|
*,
|
|
@@ -2609,6 +2787,10 @@ def _start_pending_materializer() -> None:
|
|
|
2609
2787
|
"""Drain M018 operations and backfill the legacy pending.db queue."""
|
|
2610
2788
|
global _materializer_thread
|
|
2611
2789
|
|
|
2790
|
+
if _materializer_thread is not None and _materializer_thread.is_alive():
|
|
2791
|
+
return
|
|
2792
|
+
_materializer_stop.clear()
|
|
2793
|
+
|
|
2612
2794
|
def _loop():
|
|
2613
2795
|
from superlocalmemory.cli.pending_store import (
|
|
2614
2796
|
get_pending, mark_done, mark_failed,
|
|
@@ -2624,9 +2806,12 @@ def _start_pending_materializer() -> None:
|
|
|
2624
2806
|
# not a stale local reference.
|
|
2625
2807
|
import superlocalmemory.server.unified_daemon as _ud
|
|
2626
2808
|
engine = _ud._engine
|
|
2627
|
-
|
|
2809
|
+
runtime = _ud._profile_runtime
|
|
2810
|
+
if engine is None or runtime is None:
|
|
2628
2811
|
if not _waiting_logged:
|
|
2629
|
-
logger.info(
|
|
2812
|
+
logger.info(
|
|
2813
|
+
"Materializer: waiting for engine/runtime to init..."
|
|
2814
|
+
)
|
|
2630
2815
|
_waiting_logged = True
|
|
2631
2816
|
time.sleep(0.5)
|
|
2632
2817
|
continue
|
|
@@ -2634,10 +2819,18 @@ def _start_pending_materializer() -> None:
|
|
|
2634
2819
|
logger.info("Materializer: engine acquired, starting drain loop")
|
|
2635
2820
|
_engine_logged = True
|
|
2636
2821
|
|
|
2637
|
-
|
|
2638
|
-
|
|
2639
|
-
|
|
2822
|
+
cycle_result = _run_materializer_operation(
|
|
2823
|
+
runtime,
|
|
2824
|
+
lambda: _ud._engine,
|
|
2825
|
+
lambda admitted_engine: _materialize_ingestion_one_pass(
|
|
2826
|
+
admitted_engine,
|
|
2827
|
+
# One operation per lease bounds profile-switch wait
|
|
2828
|
+
# time without allowing engine components to rebind
|
|
2829
|
+
# halfway through an enrichment pipeline.
|
|
2830
|
+
limit=1,
|
|
2831
|
+
),
|
|
2640
2832
|
)
|
|
2833
|
+
durable_complete, durable_failed = cycle_result or (0, 0)
|
|
2641
2834
|
pending = get_pending(limit=50)
|
|
2642
2835
|
if not pending and not durable_complete and not durable_failed:
|
|
2643
2836
|
time.sleep(1.0)
|
|
@@ -2655,7 +2848,15 @@ def _start_pending_materializer() -> None:
|
|
|
2655
2848
|
time.sleep(0.5)
|
|
2656
2849
|
waits += 1
|
|
2657
2850
|
try:
|
|
2658
|
-
operation_id =
|
|
2851
|
+
operation_id = _run_materializer_operation(
|
|
2852
|
+
runtime,
|
|
2853
|
+
lambda: _ud._engine,
|
|
2854
|
+
lambda admitted_engine: _materialize_legacy_pending_item(
|
|
2855
|
+
admitted_engine, item,
|
|
2856
|
+
),
|
|
2857
|
+
)
|
|
2858
|
+
if operation_id is None:
|
|
2859
|
+
raise RuntimeError("resident engine became unavailable")
|
|
2659
2860
|
mark_done(item["id"])
|
|
2660
2861
|
_emit_event(
|
|
2661
2862
|
"memory.stored",
|
|
@@ -2683,6 +2884,21 @@ def _start_pending_materializer() -> None:
|
|
|
2683
2884
|
logger.info("Pending materializer started (recall-priority)")
|
|
2684
2885
|
|
|
2685
2886
|
|
|
2887
|
+
def _stop_pending_materializer(timeout: float = 5.0) -> bool:
|
|
2888
|
+
"""Stop and join the background writer before closing its engine."""
|
|
2889
|
+
global _materializer_thread
|
|
2890
|
+
|
|
2891
|
+
_materializer_stop.set()
|
|
2892
|
+
thread = _materializer_thread
|
|
2893
|
+
if thread is not None and thread.is_alive():
|
|
2894
|
+
thread.join(timeout=timeout)
|
|
2895
|
+
if thread.is_alive():
|
|
2896
|
+
logger.warning("Pending materializer did not stop within %.1fs", timeout)
|
|
2897
|
+
return False
|
|
2898
|
+
_materializer_thread = None
|
|
2899
|
+
return True
|
|
2900
|
+
|
|
2901
|
+
|
|
2686
2902
|
def start_server(port: int = _DEFAULT_PORT) -> None:
|
|
2687
2903
|
"""Start the unified daemon. Blocks until stopped."""
|
|
2688
2904
|
global _start_time
|
|
@@ -120,6 +120,17 @@ _MODULES = {
|
|
|
120
120
|
|
|
121
121
|
logger = logging.getLogger(__name__)
|
|
122
122
|
|
|
123
|
+
# Exact historical DDL fingerprints whose resulting schema is intentionally
|
|
124
|
+
# accepted by the current migration. Unknown hashes are never reconciled.
|
|
125
|
+
_KNOWN_EQUIVALENT_DDL_HASHES: dict[str, frozenset[str]] = {
|
|
126
|
+
_M002.NAME: frozenset({
|
|
127
|
+
# v3.4.21 hardened copy-forward variant.
|
|
128
|
+
"347eeb2ec8aac89f7cbf373da49ac9446be9ed150e6105c382c656cd22426d4b",
|
|
129
|
+
# v3.4.22 model_version-default variant shipped through 3.6.x.
|
|
130
|
+
"d28666fa1dfa66e6514efd288e6748363513da2255a4cee95d80f233e6728ae7",
|
|
131
|
+
}),
|
|
132
|
+
}
|
|
133
|
+
|
|
123
134
|
|
|
124
135
|
@dataclass(frozen=True, slots=True)
|
|
125
136
|
class Migration:
|
|
@@ -307,11 +318,14 @@ def _apply_single(
|
|
|
307
318
|
# in place, reconcile the log to the current hash and treat as
|
|
308
319
|
# already-applied instead of failing the daemon into permanent
|
|
309
320
|
# not_ready. Absent/failing verify keeps the hard failure.
|
|
321
|
+
allowed_hashes = _KNOWN_EQUIVALENT_DDL_HASHES.get(
|
|
322
|
+
migration.name, frozenset(),
|
|
323
|
+
)
|
|
310
324
|
mod = _MODULES.get(migration.name)
|
|
311
325
|
verify_fn = (
|
|
312
326
|
getattr(mod, "verify", None) if mod is not None else None
|
|
313
327
|
)
|
|
314
|
-
if verify_fn is not None:
|
|
328
|
+
if logged_hash in allowed_hashes and verify_fn is not None:
|
|
315
329
|
try:
|
|
316
330
|
if verify_fn(conn):
|
|
317
331
|
if not dry_run:
|
|
@@ -323,8 +337,8 @@ def _apply_single(
|
|
|
323
337
|
pass
|
|
324
338
|
return (
|
|
325
339
|
"skipped",
|
|
326
|
-
"
|
|
327
|
-
"
|
|
340
|
+
"allowlisted historical DDL reconciled after "
|
|
341
|
+
"full schema verification",
|
|
328
342
|
)
|
|
329
343
|
except sqlite3.Error: # pragma: no cover
|
|
330
344
|
pass
|
|
@@ -29,14 +29,43 @@ _REQUIRED_COLS = frozenset({
|
|
|
29
29
|
|
|
30
30
|
|
|
31
31
|
def verify(conn: sqlite3.Connection) -> bool:
|
|
32
|
-
"""
|
|
32
|
+
"""Verify columns plus both indexes promised by this migration."""
|
|
33
33
|
try:
|
|
34
|
-
cols = {r[1] for r in conn.execute(
|
|
34
|
+
cols = {r[1]: r for r in conn.execute(
|
|
35
35
|
"PRAGMA table_info(learning_model_state)"
|
|
36
36
|
).fetchall()}
|
|
37
|
+
index_rows = conn.execute(
|
|
38
|
+
"PRAGMA index_list(learning_model_state)"
|
|
39
|
+
).fetchall()
|
|
37
40
|
except sqlite3.Error:
|
|
38
41
|
return False
|
|
39
|
-
|
|
42
|
+
if not _REQUIRED_COLS <= set(cols):
|
|
43
|
+
return False
|
|
44
|
+
|
|
45
|
+
indexes = {row[1]: row for row in index_rows}
|
|
46
|
+
active = indexes.get("idx_model_active")
|
|
47
|
+
profile_time = indexes.get("idx_model_profile_time")
|
|
48
|
+
if active is None or profile_time is None:
|
|
49
|
+
return False
|
|
50
|
+
# idx_model_active must remain a UNIQUE partial index.
|
|
51
|
+
if int(active[2]) != 1 or int(active[4]) != 1:
|
|
52
|
+
return False
|
|
53
|
+
active_cols = [row[2] for row in conn.execute(
|
|
54
|
+
"PRAGMA index_info(idx_model_active)"
|
|
55
|
+
).fetchall()]
|
|
56
|
+
time_cols = [row[2] for row in conn.execute(
|
|
57
|
+
"PRAGMA index_info(idx_model_profile_time)"
|
|
58
|
+
).fetchall()]
|
|
59
|
+
if active_cols != ["profile_id"]:
|
|
60
|
+
return False
|
|
61
|
+
if time_cols != ["profile_id", "trained_at"]:
|
|
62
|
+
return False
|
|
63
|
+
sql_row = conn.execute(
|
|
64
|
+
"SELECT sql FROM sqlite_master WHERE type='index' AND name=?",
|
|
65
|
+
("idx_model_active",),
|
|
66
|
+
).fetchone()
|
|
67
|
+
normalized = " ".join(str(sql_row[0] if sql_row else "").lower().split())
|
|
68
|
+
return "where is_active = 1" in normalized
|
|
40
69
|
|
|
41
70
|
|
|
42
71
|
# IMPORTANT: this DDL shipped in V3.4.21. Migration hashes are immutable
|
|
@@ -365,15 +365,6 @@ V32_DDL: list[str] = [
|
|
|
365
365
|
""",
|
|
366
366
|
]
|
|
367
367
|
|
|
368
|
-
# vec0 virtual table DDL — executed by VectorStore ONLY (requires extension loaded first).
|
|
369
|
-
# NOT in V32_DDL because executescript cannot load extensions mid-script.
|
|
370
|
-
V32_VEC0_DDL: Final[str] = """
|
|
371
|
-
CREATE VIRTUAL TABLE IF NOT EXISTS fact_embeddings USING vec0(
|
|
372
|
-
profile_id TEXT PARTITION KEY,
|
|
373
|
-
embedding float[768] distance_metric=cosine
|
|
374
|
-
);
|
|
375
|
-
"""
|
|
376
|
-
|
|
377
368
|
# ---------------------------------------------------------------------------
|
|
378
369
|
# Rollback DDL (reverse FK order -- Rule 20)
|
|
379
370
|
# ---------------------------------------------------------------------------
|