superlocalmemory 4.0.8 → 4.0.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/CHANGELOG.md +146 -1
  2. package/README.md +6 -6
  3. package/package.json +1 -1
  4. package/plugin/.claude-plugin/plugin.json +1 -1
  5. package/plugin/CLAUDE.md +3 -3
  6. package/plugin/agents/slm-governance-advisor.md +1 -1
  7. package/plugin/agents/slm-loop-runner.md +1 -1
  8. package/plugin/agents/slm-memory-advisor.md +1 -1
  9. package/plugin/agents/slm-optimize-advisor.md +1 -1
  10. package/plugin/requirements.txt +1 -1
  11. package/plugin/scripts/ensure-venv.sh +1 -1
  12. package/plugin/skills/slm-cache/SKILL.md +1 -1
  13. package/plugin/skills/slm-compress/SKILL.md +1 -1
  14. package/plugin/skills/slm-governance/SKILL.md +1 -1
  15. package/plugin/skills/slm-graph/SKILL.md +1 -1
  16. package/plugin/skills/slm-loop/SKILL.md +1 -1
  17. package/plugin/skills/slm-mesh/SKILL.md +1 -1
  18. package/plugin/skills/slm-profile/SKILL.md +1 -1
  19. package/plugin/skills/slm-recall/SKILL.md +1 -1
  20. package/plugin/skills/slm-remember/SKILL.md +1 -1
  21. package/plugin/skills/slm-scope/SKILL.md +1 -1
  22. package/plugin/skills/slm-session/SKILL.md +1 -1
  23. package/plugin/skills/slm-status/SKILL.md +3 -3
  24. package/plugin-src/rules/AGENTS.md +1 -1
  25. package/plugin-src/skills/slm-status/SKILL.md +2 -2
  26. package/pyproject.toml +1 -1
  27. package/scripts/postinstall.js +4 -0
  28. package/src/superlocalmemory/__init__.py +1 -1
  29. package/src/superlocalmemory/cli/_lazy_init.py +1 -1
  30. package/src/superlocalmemory/cli/commands.py +119 -9
  31. package/src/superlocalmemory/cli/db_migrate.py +0 -2
  32. package/src/superlocalmemory/cli/gdpr_io.py +1 -1
  33. package/src/superlocalmemory/cli/main.py +5 -5
  34. package/src/superlocalmemory/cli/service_installer.py +2 -1
  35. package/src/superlocalmemory/cli/setup_wizard.py +1 -1
  36. package/src/superlocalmemory/core/config.py +41 -7
  37. package/src/superlocalmemory/core/context_cache.py +0 -2
  38. package/src/superlocalmemory/core/engine.py +371 -63
  39. package/src/superlocalmemory/core/evidence_bundle.py +3 -1
  40. package/src/superlocalmemory/core/install_detector.py +131 -0
  41. package/src/superlocalmemory/core/progressive_abstraction.py +1 -1
  42. package/src/superlocalmemory/core/security_primitives.py +3 -6
  43. package/src/superlocalmemory/core/store_pipeline.py +94 -26
  44. package/src/superlocalmemory/core/topic_signature.py +0 -2
  45. package/src/superlocalmemory/core/transactions/concrete_owners.py +15 -8
  46. package/src/superlocalmemory/dynamics/eap_scheduler.py +17 -6
  47. package/src/superlocalmemory/encoding/graph_builder.py +2 -2
  48. package/src/superlocalmemory/encoding/scene_builder.py +8 -2
  49. package/src/superlocalmemory/hooks/adapter_base.py +0 -2
  50. package/src/superlocalmemory/hooks/context_payload.py +0 -2
  51. package/src/superlocalmemory/hooks/portable_kit.py +8 -8
  52. package/src/superlocalmemory/hooks/post_tool_async_hook.py +0 -2
  53. package/src/superlocalmemory/hooks/prewarm_auth.py +0 -2
  54. package/src/superlocalmemory/hooks/user_prompt_hook.py +0 -2
  55. package/src/superlocalmemory/infra/backup.py +44 -8
  56. package/src/superlocalmemory/integrations/bounded_loops_mcp.py +24 -7
  57. package/src/superlocalmemory/learning/arm_catalog.py +0 -2
  58. package/src/superlocalmemory/learning/bandit.py +0 -2
  59. package/src/superlocalmemory/learning/bandit_cache.py +0 -2
  60. package/src/superlocalmemory/learning/dedup_hnsw.py +11 -11
  61. package/src/superlocalmemory/learning/ensemble.py +0 -2
  62. package/src/superlocalmemory/learning/labeler.py +0 -2
  63. package/src/superlocalmemory/learning/legacy_migration.py +0 -2
  64. package/src/superlocalmemory/learning/model_cache.py +0 -2
  65. package/src/superlocalmemory/learning/ranker.py +0 -2
  66. package/src/superlocalmemory/learning/reward_archive.py +6 -1
  67. package/src/superlocalmemory/learning/reward_proxy.py +0 -2
  68. package/src/superlocalmemory/learning/signal_worker.py +0 -2
  69. package/src/superlocalmemory/math/fisher.py +1 -1
  70. package/src/superlocalmemory/math/hopfield.py +4 -1
  71. package/src/superlocalmemory/math/langevin.py +1 -1
  72. package/src/superlocalmemory/math/sheaf.py +7 -3
  73. package/src/superlocalmemory/mcp/cli_fallback.py +1 -1
  74. package/src/superlocalmemory/mcp/profiles.py +1 -1
  75. package/src/superlocalmemory/mcp/server.py +1 -1
  76. package/src/superlocalmemory/mcp/tools_active.py +56 -0
  77. package/src/superlocalmemory/mcp/tools_core.py +1 -1
  78. package/src/superlocalmemory/optimize/cache/manager.py +2 -2
  79. package/src/superlocalmemory/optimize/compress/ccr.py +1 -1
  80. package/src/superlocalmemory/optimize/compress/router.py +1 -1
  81. package/src/superlocalmemory/optimize/proxy/_helpers.py +2 -2
  82. package/src/superlocalmemory/optimize/proxy/server.py +1 -1
  83. package/src/superlocalmemory/optimize/proxy/vertex_surface.py +2 -2
  84. package/src/superlocalmemory/optimize/storage/db.py +2 -2
  85. package/src/superlocalmemory/retrieval/agentic.py +1 -1
  86. package/src/superlocalmemory/retrieval/ann_index.py +9 -2
  87. package/src/superlocalmemory/retrieval/bm25_channel.py +2 -2
  88. package/src/superlocalmemory/retrieval/bridge_discovery.py +2 -2
  89. package/src/superlocalmemory/retrieval/engine.py +272 -43
  90. package/src/superlocalmemory/retrieval/entity_channel.py +1 -1
  91. package/src/superlocalmemory/retrieval/hopfield_channel.py +8 -2
  92. package/src/superlocalmemory/retrieval/profile_channel.py +1 -1
  93. package/src/superlocalmemory/retrieval/quantization_aware_search.py +1 -1
  94. package/src/superlocalmemory/retrieval/remote_reranker.py +2 -2
  95. package/src/superlocalmemory/retrieval/reranker.py +3 -3
  96. package/src/superlocalmemory/retrieval/semantic_channel.py +3 -3
  97. package/src/superlocalmemory/retrieval/spreading_activation.py +8 -8
  98. package/src/superlocalmemory/retrieval/strategy.py +94 -0
  99. package/src/superlocalmemory/retrieval/temporal_channel.py +167 -10
  100. package/src/superlocalmemory/retrieval/temporal_validity_filter.py +1 -1
  101. package/src/superlocalmemory/retrieval/vector_store.py +88 -10
  102. package/src/superlocalmemory/server/recall_serializer.py +10 -0
  103. package/src/superlocalmemory/server/routes/brain.py +2 -2
  104. package/src/superlocalmemory/server/routes/prewarm.py +4 -4
  105. package/src/superlocalmemory/server/unified_daemon.py +459 -7
  106. package/src/superlocalmemory/storage/_schema_version.py +46 -3
  107. package/src/superlocalmemory/storage/backup.py +531 -0
  108. package/src/superlocalmemory/storage/database.py +11 -4
  109. package/src/superlocalmemory/storage/embedding_codec.py +129 -0
  110. package/src/superlocalmemory/storage/embedding_migrator.py +5 -3
  111. package/src/superlocalmemory/storage/migration_runner.py +142 -2
  112. package/src/superlocalmemory/storage/migrations/__init__.py +1 -1
  113. package/src/superlocalmemory/storage/migrations.py +15 -1
  114. package/src/superlocalmemory/storage/models.py +7 -0
  115. package/src/superlocalmemory/storage/quantized_store.py +4 -2
  116. package/src/superlocalmemory/summaries/session_summary.py +1 -1
  117. package/src/superlocalmemory/ui/css/legacy-dashboard.css +1 -1
  118. package/src/superlocalmemory/ui/css/neural-glass.css +1 -1
  119. package/src/superlocalmemory/ui/js/core.js +1 -1
  120. package/src/superlocalmemory/ui/js/od-brain.js +1 -1
@@ -26,6 +26,8 @@ License: AGPL-3.0-or-later
26
26
  from __future__ import annotations
27
27
 
28
28
  import asyncio
29
+ import concurrent.futures
30
+ import functools
29
31
  import hashlib
30
32
  import json
31
33
  import logging
@@ -67,6 +69,126 @@ from superlocalmemory.infra.data_root import (
67
69
  )
68
70
 
69
71
 
72
+ def _notify_migration_applied(
73
+ applied: list,
74
+ elapsed: float,
75
+ backup_dir: "Path | None" = None,
76
+ ) -> None:
77
+ """Print a one-line confirmation after migrations complete successfully.
78
+
79
+ Written to stdout so it is visible in daemon.log and in interactive runs.
80
+ """
81
+ count = len(applied)
82
+ if count == 0:
83
+ return
84
+ parts = [f"[SLM] {count} migration(s) applied in {elapsed:.1f}s."]
85
+ if backup_dir is not None:
86
+ parts.append(f"Backup: {backup_dir}")
87
+ print(" ".join(parts), flush=True)
88
+
89
+
90
+ def _write_migration_error_log(
91
+ failed: list,
92
+ backup_dir: "Path | None",
93
+ slm_home: "Path | None" = None,
94
+ applied: "list | None" = None,
95
+ ) -> "Path":
96
+ """Write a migration-error-{timestamp}.log to the SLM home directory.
97
+
98
+ Returns the Path of the written log file.
99
+ """
100
+ import datetime
101
+
102
+ if slm_home is None:
103
+ slm_home = canonical_data_root()
104
+
105
+ slm_home.mkdir(parents=True, exist_ok=True)
106
+ ts = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
107
+ log_path = slm_home / f"migration-error-{ts}.log"
108
+
109
+ lines = [
110
+ f"Migration error — {ts}",
111
+ f"Failed: {', '.join(str(f) for f in failed)}",
112
+ ]
113
+ # An upgrade is applied step by step and is deliberately non-fatal: some
114
+ # steps can succeed while a later one fails. Saying "your data was not
115
+ # modified" whenever a snapshot exists was therefore untrue in exactly the
116
+ # case that matters — and a user who believes nothing changed may delete the
117
+ # snapshot to reclaim space.
118
+ if applied:
119
+ lines.append(f"Applied before the failure: {', '.join(str(a) for a in applied)}")
120
+ lines.append(
121
+ "YOUR DATABASE WAS PARTIALLY CHANGED. The steps listed above "
122
+ "completed; the ones under 'Failed' did not."
123
+ )
124
+ elif applied is not None:
125
+ lines.append("No steps completed, so your database was not changed.")
126
+ if backup_dir is not None:
127
+ lines.append(f"Snapshot of the state before the upgrade: {backup_dir}")
128
+ lines.append(
129
+ "That snapshot is intact and can be restored. Keep it until you are "
130
+ "satisfied your data is correct."
131
+ )
132
+ else:
133
+ lines.append(
134
+ "NO SNAPSHOT WAS TAKEN for this attempt — do not assume a recovery "
135
+ "copy exists. Check for earlier snapshots before changing anything."
136
+ )
137
+ lines.append("")
138
+ lines.append("To diagnose, run: slm doctor")
139
+
140
+ log_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
141
+ return log_path
142
+
143
+
144
+ def _notify_windows_user(message: str) -> None:
145
+ """Best-effort notification on Windows. Silent no-op on other platforms."""
146
+ import sys
147
+
148
+ if sys.platform != "win32":
149
+ return
150
+
151
+ # Attempt 1 — Windows MessageBox via PowerShell
152
+ try:
153
+ import subprocess
154
+
155
+ safe_msg = message.replace("'", "").replace('"', "")
156
+ subprocess.run(
157
+ [
158
+ "powershell",
159
+ "-Command",
160
+ "[System.Reflection.Assembly]::LoadWithPartialName"
161
+ "('System.Windows.Forms') | Out-Null; "
162
+ f"[System.Windows.Forms.MessageBox]::Show('{safe_msg}', "
163
+ "'SuperLocalMemory')",
164
+ ],
165
+ check=False,
166
+ timeout=5,
167
+ )
168
+ return
169
+ except Exception:
170
+ pass
171
+
172
+ # Attempt 2 — Windows Application EventLog
173
+ try:
174
+ import subprocess
175
+
176
+ subprocess.run(
177
+ [
178
+ "eventcreate",
179
+ "/T", "INFORMATION",
180
+ "/ID", "1001",
181
+ "/L", "APPLICATION",
182
+ "/SO", "SuperLocalMemory",
183
+ "/D", message[:512],
184
+ ],
185
+ check=False,
186
+ timeout=5,
187
+ )
188
+ except Exception:
189
+ pass # best-effort only
190
+
191
+
70
192
  def _learning_db_for_config(config) -> Path:
71
193
  """Single learning.db path for daemon migration + engine (F-05).
72
194
 
@@ -115,11 +237,142 @@ _SOURCE_QUALITY_MAX_BATCH_SIZE = 250
115
237
  _SOURCE_QUALITY_PROFILE_REFRESH_SECONDS = 60.0
116
238
  _FACT_ENTITY_REPAIR_MIN_RETRY_SECONDS = 0.05
117
239
  _FACT_ENTITY_REPAIR_MAX_RETRY_SECONDS = 30.0
118
- # ``wait=true`` is a compatibility affordance, never permission to hold the
119
- # ASGI event loop hostage to a local LLM. Normal clients omit it and receive
120
- # an immediate durable/queryable receipt; explicit waiters get this small
121
- # completion window, then the M018 materializer continues in the background.
122
- _REMEMBER_ENRICHMENT_WAIT_SECONDS = 0.75
240
+ # How long a write may spend making itself findable by meaning before the
241
+ # receipt goes out. Storing a memory has a 1.5 s ceiling, and this is a ceiling
242
+ # rather than a target: the cost of coming in under it is a memory that can only
243
+ # be found by quoting its own wording, which is not how anyone asks. So the
244
+ # window is most of the budget, not a token amount of it. ``wait=true`` is still
245
+ # never permission to hold the event loop open indefinitely — past this the
246
+ # background pass finishes the job.
247
+ # The ceiling for a whole store request, shared by the durable write and the
248
+ # window that makes the memory findable by meaning. Both are sequential, so the
249
+ # second gets what the first left rather than a fresh grant.
250
+ _REMEMBER_TOTAL_CEILING_SECONDS = 1.5
251
+ _REMEMBER_ENRICHMENT_WAIT_SECONDS = 1.2
252
+
253
+ # Enrichment runs off the event loop, but NOT on the loop's default executor.
254
+ # `asyncio.to_thread` uses that default pool, so N simultaneous writers really did
255
+ # create N threads — and every other handler that borrows the same pool queued
256
+ # behind them. One shared, small pool bounds the cost of a burst: past its width
257
+ # the extra callers miss their deadline and defer, which is the intended
258
+ # degradation, instead of multiplying threads.
259
+ _ENRICHMENT_WORKERS = 2
260
+ _enrichment_pool: "concurrent.futures.ThreadPoolExecutor | None" = None
261
+ _enrichment_pool_lock = threading.Lock()
262
+
263
+ # Bounds how much enrichment may be in flight at once. A ThreadPoolExecutor
264
+ # queues without limit, so bounding the pool's WIDTH does not bound what can be
265
+ # handed to it: a burst of writers all submit, the first two run, and the rest sit
266
+ # in a queue nobody is waiting for any more — then wake up and write to the
267
+ # database after their callers have already been answered. A permit is taken
268
+ # before submitting and released by the worker when it has really finished, so at
269
+ # most `_ENRICHMENT_WORKERS` pieces of work exist at once and a caller that
270
+ # cannot get one is told immediately instead of being queued.
271
+ _enrichment_semaphore = threading.Semaphore(_ENRICHMENT_WORKERS)
272
+
273
+ # Set once the pool has been shut down, and never cleared. Without it, a caller
274
+ # that passed the unlocked check below arrives after shutdown, finds None, and
275
+ # builds a replacement pool that the completed shutdown will never reach.
276
+ _enrichment_pool_closed = False
277
+
278
+
279
+ class _EnrichmentAtCapacity(Exception):
280
+ """No permit was free, so nothing was submitted.
281
+
282
+ Its own type, so that "we chose not to start" is never reported through the
283
+ same path as "it started and then failed" — those are different events and
284
+ only one of them is worth a warning about degradation.
285
+ """
286
+
287
+
288
+ def _enrichment_executor():
289
+ """The one pool inline enrichment may use. Created on first need.
290
+
291
+ Raises ``RuntimeError`` once the pool has been shut down. It deliberately
292
+ does not return ``None`` on that path: ``run_in_executor(None, ...)`` means
293
+ *the event loop's default executor*, which is the unbounded-thread behaviour
294
+ this pool exists to replace, so a None here would silently reintroduce it at
295
+ exactly the moment the process is trying to stop.
296
+ """
297
+ global _enrichment_pool
298
+ # Everything happens under the lock, including the read that is returned. An
299
+ # earlier version checked the closed flag only on the create branch and then
300
+ # returned the global, so a caller that found a live pool, lost the CPU, and
301
+ # resumed after shutdown returned None — and None means the event loop's own
302
+ # default executor, unbounded and shared with every other handler. That is
303
+ # the precise failure this function exists to prevent, reintroduced at the
304
+ # one moment it matters.
305
+ with _enrichment_pool_lock:
306
+ if _enrichment_pool_closed:
307
+ raise RuntimeError("enrichment pool is shut down")
308
+ if _enrichment_pool is None:
309
+ _enrichment_pool = concurrent.futures.ThreadPoolExecutor(
310
+ max_workers=_ENRICHMENT_WORKERS,
311
+ thread_name_prefix="slm-enrich",
312
+ )
313
+ pool = _enrichment_pool
314
+ if pool is None: # pragma: no cover — belt and braces around the invariant
315
+ raise RuntimeError("enrichment pool is unavailable")
316
+ return pool
317
+
318
+
319
+ def _open_enrichment_pool() -> None:
320
+ """Allow the pool to be created again, for a fresh run in this process.
321
+
322
+ The closed flag has to survive shutdown — that is its whole purpose — so
323
+ something has to clear it when a new run legitimately begins. Without this a
324
+ second startup in one process (a restart, or a test that builds the app more
325
+ than once) would find enrichment permanently refused and every write would
326
+ come back findable by wording only, with nothing in the logs to explain it.
327
+ """
328
+ global _enrichment_pool_closed, _enrichment_semaphore
329
+ with _enrichment_pool_lock:
330
+ _enrichment_pool_closed = False
331
+ # A fresh run starts with its full capacity. Shutdown cancels work that
332
+ # was queued and never started, and a task that never starts never runs
333
+ # the release in its finally — so permits taken by cancelled work are
334
+ # gone. Without this reset the next run in the same process would run at
335
+ # reduced capacity, or none at all, and the only symptom would be every
336
+ # write coming back findable by wording with nothing in the log.
337
+ _enrichment_semaphore = threading.Semaphore(_ENRICHMENT_WORKERS)
338
+
339
+
340
+ def _enrich_and_release(engine, fact_ids: list[str], budget: float) -> int:
341
+ """Run inline enrichment, releasing the permit only when it is really done.
342
+
343
+ The permit has to be released here rather than by the caller. A caller that
344
+ stops waiting at its own deadline has not finished the work — the worker is
345
+ still holding a pool thread — so releasing on the caller's timeout would let
346
+ the next request submit into a pool that is still fully occupied, which is
347
+ the unbounded queueing this permit exists to prevent.
348
+ """
349
+ try:
350
+ return engine.enrich_new_facts_now(fact_ids, timeout_s=budget)
351
+ finally:
352
+ _enrichment_semaphore.release()
353
+
354
+
355
+ def _shutdown_enrichment_pool_if_created() -> None:
356
+ """Shut down the enrichment pool if it was ever created.
357
+
358
+ Safe to call when the pool was never created (no-op) and safe to call
359
+ twice (second call is a no-op because the pool is set to None under the
360
+ lock before shutdown is called).
361
+
362
+ cancel_futures=True drops any work that is still queued but has not yet
363
+ started. Workers already inside embed() will finish their current call
364
+ and then stop; they do not block the caller because wait=False.
365
+ """
366
+ global _enrichment_pool, _enrichment_pool_closed
367
+ with _enrichment_pool_lock:
368
+ # Marked closed under the same lock that guards creation, so a caller
369
+ # already past the unlocked check cannot build a replacement pool that
370
+ # this shutdown has already walked past.
371
+ _enrichment_pool_closed = True
372
+ pool = _enrichment_pool
373
+ _enrichment_pool = None
374
+ if pool is not None:
375
+ pool.shutdown(wait=False, cancel_futures=True)
123
376
  _SENSITIVE_READ_PREFIXES = (
124
377
  "/api/memories", "/api/facts", "/api/clusters", "/api/graph",
125
378
  "/api/v3/associations", "/api/v3/core-memory",
@@ -1500,6 +1753,11 @@ async def lifespan(application: FastAPI):
1500
1753
  """Initialize engine, workers, and optional services on startup."""
1501
1754
  global _last_activity
1502
1755
 
1756
+ # A previous run in this process left the enrichment pool closed so that a
1757
+ # write racing its shutdown could not resurrect it. This run is entitled to
1758
+ # one.
1759
+ _open_enrichment_pool()
1760
+
1503
1761
  engine = None
1504
1762
  config = None
1505
1763
  deployment = None
@@ -1592,6 +1850,7 @@ async def lifespan(application: FastAPI):
1592
1850
  _path_config = None
1593
1851
 
1594
1852
  try:
1853
+ from superlocalmemory.storage.backup import InsufficientDiskSpaceError
1595
1854
  from superlocalmemory.storage.migration_runner import apply_all
1596
1855
  if _path_config is None:
1597
1856
  # Catastrophic early-load failure: still one root via Mode-A default
@@ -1602,13 +1861,56 @@ async def lifespan(application: FastAPI):
1602
1861
  _path_config = _SLMFallback.for_mode(_ModeFallback.A)
1603
1862
  _learning_db = _learning_db_for_config(_path_config)
1604
1863
  _memory_db = _memory_db_for_config(_path_config)
1864
+ import time as _time_mod
1865
+ _t0 = _time_mod.monotonic()
1605
1866
  _result = apply_all(_learning_db, _memory_db)
1867
+ _elapsed = _time_mod.monotonic() - _t0
1606
1868
  _applied = _result.get("applied", [])
1607
1869
  _failed = _result.get("failed", [])
1870
+ _backup_dir = _result.get("details", {}).get("_backup")
1871
+ _backup_path = Path(_backup_dir) if _backup_dir else None
1608
1872
  if _applied:
1609
1873
  logger.info("migrations applied: %s", _applied)
1874
+ _notify_migration_applied(_applied, _elapsed, _backup_path)
1875
+ _notify_windows_user(
1876
+ f"SuperLocalMemory: {len(_applied)} database migration(s) applied."
1877
+ )
1610
1878
  if _failed:
1611
1879
  logger.warning("migrations failed (non-fatal): %s", _failed)
1880
+ try:
1881
+ _err_log = _write_migration_error_log(
1882
+ _failed, _backup_path,
1883
+ slm_home=_memory_db.parent if _memory_db else None,
1884
+ applied=_applied,
1885
+ )
1886
+ import sys as _sys
1887
+ # "Data is safe" is only true when NOTHING was applied. Migrations
1888
+ # are non-fatal and applied in order, so a later failure can leave
1889
+ # earlier ones committed — a partially changed store. Saying the
1890
+ # data is untouched then tells the user to ignore the one copy
1891
+ # that still holds the original.
1892
+ if _applied:
1893
+ _headline = (
1894
+ f"[SLM] Migration incomplete. YOUR DATABASE WAS PARTIALLY "
1895
+ f"CHANGED — {len(_applied)} step(s) applied, "
1896
+ f"{len(_failed)} failed. The pre-change copy is at "
1897
+ f"{_backup_path} — keep it until this is resolved."
1898
+ )
1899
+ else:
1900
+ _headline = (
1901
+ f"[SLM] Migration failed before any change was made. Your "
1902
+ f"data is as it was; a copy is also at {_backup_path}."
1903
+ )
1904
+ print(
1905
+ f"{_headline} See {_err_log}. Run: slm doctor",
1906
+ file=_sys.stderr,
1907
+ flush=True,
1908
+ )
1909
+ _notify_windows_user(
1910
+ f"SuperLocalMemory migration failed. Run slm doctor. Log: {_err_log}"
1911
+ )
1912
+ except Exception:
1913
+ pass # notification failures are non-fatal
1612
1914
  application.state.migration_result = _result
1613
1915
  # S9-SKEP-15: only commit the new `.last_version` AFTER migrations
1614
1916
  # complete with zero failures. A partial upgrade (schema didn't
@@ -1626,6 +1928,35 @@ async def lifespan(application: FastAPI):
1626
1928
  _version_marker.write_text(_slm_version, encoding="utf-8")
1627
1929
  except OSError:
1628
1930
  pass # non-fatal
1931
+ except InsufficientDiskSpaceError as _disk_exc:
1932
+ # This is not a crash and it must not be reported as "non-fatal". The
1933
+ # migration deliberately refused to start because there was not enough
1934
+ # room to keep a recoverable copy first. The store is intact and
1935
+ # unmigrated; the generic handler below would have logged a warning and
1936
+ # left the daemon serving as though nothing had happened.
1937
+ logger.error(
1938
+ "MIGRATION NOT RUN — not enough free disk to keep a recoverable "
1939
+ "copy first. Your data has NOT been modified. Need %s bytes, have "
1940
+ "%s free. Free some space and restart; run `slm doctor` for details.",
1941
+ f"{_disk_exc.needed_bytes:,}", f"{_disk_exc.free_bytes:,}",
1942
+ )
1943
+ try:
1944
+ _log_path = _write_migration_error_log(
1945
+ ["_insufficient_disk_space"], None,
1946
+ slm_home=locals().get("_memory_db").parent if locals().get("_memory_db") else None,
1947
+ applied=[],
1948
+ )
1949
+ logger.error("Details written to %s", _log_path)
1950
+ except OSError:
1951
+ pass
1952
+ application.state.migration_result = {
1953
+ "applied": [], "skipped": [], "failed": ["_insufficient_disk_space"],
1954
+ "details": {
1955
+ "_needed_bytes": _disk_exc.needed_bytes,
1956
+ "_free_bytes": _disk_exc.free_bytes,
1957
+ "_store_modified": False,
1958
+ },
1959
+ }
1629
1960
  except Exception as _exc:
1630
1961
  logger.warning("migration runner crashed (non-fatal): %s", _exc)
1631
1962
  application.state.migration_result = {
@@ -2822,6 +3153,19 @@ async def lifespan(application: FastAPI):
2822
3153
  except Exception as exc: # pragma: no cover — defensive
2823
3154
  logger.warning("perf_log flush failed: %s", exc)
2824
3155
 
3156
+ # Shut down the enrichment pool before engine.close() so that any worker
3157
+ # still inside embed() receives cancellation before the embed pool is torn
3158
+ # down. shutdown(wait=False, cancel_futures=True) is non-blocking: queued
3159
+ # futures are dropped and running futures complete their current call on
3160
+ # their own. This prevents Python's atexit handler from calling
3161
+ # shutdown(wait=True) on a live embed worker, which would push the daemon
3162
+ # past the graceful-shutdown budget and cause the service manager to
3163
+ # SIGKILL the process.
3164
+ try:
3165
+ _shutdown_enrichment_pool_if_created()
3166
+ except Exception as exc: # pragma: no cover — defensive
3167
+ logger.warning("enrichment pool shutdown failed (non-fatal): %s", exc)
3168
+
2825
3169
  materializer_stopped = _stop_pending_materializer()
2826
3170
  canonical_writer_stopped = _release_canonical_remember_runtime(application)
2827
3171
  _profile_runtime = None
@@ -4175,6 +4519,10 @@ def _register_daemon_routes(application: FastAPI) -> None:
4175
4519
  allowed_profiles=frozenset({engine._profile_id}),
4176
4520
  allowed_scopes=frozenset({scope}),
4177
4521
  )
4522
+ # The whole request has a 1.5 s ceiling, and the two phases below
4523
+ # are sequential, so the ceiling has to be shared between them
4524
+ # rather than granted twice. Measured from here.
4525
+ _store_started = time.monotonic()
4178
4526
  receipt = await asyncio.to_thread(
4179
4527
  runtime.remember,
4180
4528
  admission,
@@ -4183,10 +4531,105 @@ def _register_daemon_routes(application: FastAPI) -> None:
4183
4531
  )
4184
4532
  payload = dict(receipt.payload)
4185
4533
  fact_ids = list(payload.get("fact_ids") or [])
4534
+
4535
+ # The durable receipt is already committed above; nothing below can
4536
+ # fail this write. What remains is the window in which the memory can
4537
+ # only be found by quoting its own wording — a fact with no vector is
4538
+ # invisible to the semantic channel, so the memory describing what is
4539
+ # happening right now is the hardest one to find.
4540
+ #
4541
+ # Every entry point — command line, tool interface, dashboard —
4542
+ # arrives here, so closing the window here closes it for all of them
4543
+ # rather than for whichever door happened to be used.
4544
+ #
4545
+ # Strictly bounded, because this daemon serves many sessions at once:
4546
+ # the work runs off the event loop, on a two-worker pool, and yields
4547
+ # to any recall in flight. Past the deadline the caller gets exactly
4548
+ # the previous behaviour and the background materializer finishes the
4549
+ # job.
4550
+ # A caller that did not ask to wait still gets a real attempt, not a
4551
+ # token one. Almost no client passes wait=true, so a small budget
4552
+ # here meant almost every memory in practice was returned findable by
4553
+ # wording only — the default path deciding the product's behaviour.
4554
+ #
4555
+ # But it gets what is LEFT of the ceiling, not a second full budget.
4556
+ # The durable write above can legitimately consume most of the 1.5 s
4557
+ # under contention, and granting the full enrichment window on top of
4558
+ # that produced receipts well past 3 s — over a ceiling the changelog
4559
+ # states. A caller that asked to wait may exceed the shared ceiling,
4560
+ # because that is what asking to wait means; nobody else may.
4561
+ _elapsed = time.monotonic() - _store_started
4562
+ _remaining = _REMEMBER_TOTAL_CEILING_SECONDS - _elapsed
4563
+ if wait:
4564
+ enrich_budget = _REMEMBER_ENRICHMENT_WAIT_SECONDS
4565
+ else:
4566
+ enrich_budget = min(
4567
+ 1.0, _REMEMBER_ENRICHMENT_WAIT_SECONDS, max(0.0, _remaining),
4568
+ )
4569
+ if enrich_budget <= 0.0:
4570
+ logger.warning(
4571
+ "the durable write used the whole %.1fs budget (%.2fs) — this "
4572
+ "memory is findable by its wording and the background pass "
4573
+ "will attach its vector",
4574
+ _REMEMBER_TOTAL_CEILING_SECONDS, _elapsed,
4575
+ )
4576
+ enriched = 0
4577
+ # Taken before submitting, released by the worker. If none is free
4578
+ # the pool is already saturated, and queueing behind it would mean
4579
+ # writing to the database after this response has been sent — so the
4580
+ # write is simply reported as findable by wording and the background
4581
+ # pass picks it up, which is the intended degradation.
4582
+ _permit = _enrichment_semaphore.acquire(blocking=False)
4583
+ if not _permit:
4584
+ logger.warning(
4585
+ "inline enrichment at capacity for %d fact(s) — deferred to "
4586
+ "the background pass", len(fact_ids),
4587
+ )
4588
+ try:
4589
+ if not _permit:
4590
+ raise _EnrichmentAtCapacity
4591
+ loop = asyncio.get_running_loop()
4592
+ try:
4593
+ _pending = loop.run_in_executor(
4594
+ _enrichment_executor(),
4595
+ functools.partial(
4596
+ _enrich_and_release,
4597
+ engine, fact_ids, enrich_budget,
4598
+ ),
4599
+ )
4600
+ except BaseException:
4601
+ # Never handed to a worker, so nothing will release it.
4602
+ _enrichment_semaphore.release()
4603
+ raise
4604
+ enriched = await asyncio.wait_for(
4605
+ _pending, timeout=enrich_budget + 0.25,
4606
+ )
4607
+ except _EnrichmentAtCapacity:
4608
+ pass
4609
+ except (TimeoutError, asyncio.TimeoutError):
4610
+ # Worth seeing. Sustained timeouts mean writes are coming back
4611
+ # findable by wording only, which is a real degradation and was
4612
+ # previously visible at debug level alone.
4613
+ logger.warning(
4614
+ "inline enrichment exceeded its budget for %d fact(s) — "
4615
+ "deferred to the background pass", len(fact_ids),
4616
+ )
4617
+ except Exception as exc:
4618
+ logger.warning(
4619
+ "inline enrichment unavailable (%s: %s) — deferred to the "
4620
+ "background pass", type(exc).__name__, exc,
4621
+ )
4622
+
4623
+ searchable = "meaning" if enriched == len(fact_ids) and fact_ids else "wording"
4186
4624
  return {
4187
4625
  "ok": True,
4188
4626
  "fact_ids": fact_ids,
4189
4627
  "count": len(fact_ids),
4628
+ # Storing a memory and being able to find it again are different
4629
+ # things, and reporting only the first is how a caller ends up
4630
+ # believing a memory is retrievable when it is not yet.
4631
+ "searchable_by": searchable,
4632
+ "enriched_now": enriched,
4190
4633
  "operation_id": payload["operation_id"],
4191
4634
  # One-release compatibility alias. The durable operation ID is
4192
4635
  # opaque and replaces the integer pending.db row identifier.
@@ -4194,8 +4637,17 @@ def _register_daemon_routes(application: FastAPI) -> None:
4194
4637
  "status": "queryable",
4195
4638
  "materialization_state": payload["materialization_state"],
4196
4639
  "commit_sequence": payload.get("commit_sequence"),
4197
- "note": "queryable now; canonical enrichment continues in the background",
4198
- "wait_ignored": bool(wait),
4640
+ "note": (
4641
+ "stored and searchable by meaning"
4642
+ if searchable == "meaning"
4643
+ else "stored and searchable by wording; "
4644
+ "searchable by meaning shortly"
4645
+ ),
4646
+ # Retained for callers that already read it. It used to be
4647
+ # unconditionally true, because `wait` bought nothing. It now
4648
+ # buys a larger best-effort enrichment budget — still never a
4649
+ # guarantee, and still never a block on durability.
4650
+ "wait_ignored": False if wait else True,
4199
4651
  }
4200
4652
  except Exception as exc:
4201
4653
  from superlocalmemory.core.remember_admission import AdmissionRejected
@@ -84,19 +84,62 @@ def write_schema_version(conn: sqlite3.Connection, version: int) -> None:
84
84
  )
85
85
 
86
86
 
87
+ def _detect_all_installs() -> list:
88
+ """Delegate to core.install_detector; fail-open (returns [] on any error)."""
89
+ try:
90
+ from superlocalmemory.core.install_detector import (
91
+ _detect_all_installs as _real,
92
+ )
93
+ return _real()
94
+ except Exception: # import error, timeout, etc.
95
+ return []
96
+
97
+
98
+ def _format_install_block(installs: list) -> str:
99
+ """Return a formatted multi-line string listing all detected installs."""
100
+ if not installs:
101
+ return ""
102
+
103
+ _upgrade_hints = {
104
+ "pipx": " pipx upgrade superlocalmemory",
105
+ "venv": " pip install -U superlocalmemory (in ~/.slm-venv)",
106
+ "npm": " npm install -g superlocalmemory@latest",
107
+ }
108
+
109
+ lines = ["\nAll detected installs:"]
110
+ seen_types: set[str] = set()
111
+ for entry in installs:
112
+ install_type = entry.get("type", "unknown")
113
+ version = entry.get("version", "unknown")
114
+ path = entry.get("path", "")
115
+ lines.append(f" {install_type:<6} ({version}): {path}")
116
+ seen_types.add(install_type)
117
+
118
+ lines.append("\nUpgrade all installs to the latest version:")
119
+ for install_type, cmd in _upgrade_hints.items():
120
+ if install_type in seen_types:
121
+ lines.append(cmd)
122
+
123
+ return "\n".join(lines)
124
+
125
+
87
126
  def check_version_or_raise(db_path: Path) -> None:
88
127
  """Raise SchemaVersionError if the DB's stored version exceeds supported max.
89
128
 
129
+ When raising, the error message lists every SLM installation detected on
130
+ this machine so the user knows which copies to upgrade.
131
+
90
132
  Non-mutating. Must be called before any write at startup.
91
133
  """
92
134
  stored = read_schema_version(db_path)
93
135
  if stored > SUPPORTED_SCHEMA_VERSION:
136
+ installs = _detect_all_installs()
137
+ install_block = _format_install_block(installs)
94
138
  raise SchemaVersionError(
95
139
  f"Database schema_version={stored} exceeds the maximum "
96
140
  f"supported version={SUPPORTED_SCHEMA_VERSION}. "
97
- "This installation is too old for this database. "
98
- "Upgrade SLM to a version that supports schema_version "
99
- f"{stored} or higher."
141
+ "This installation is too old for this database."
142
+ + install_block
100
143
  )
101
144
 
102
145