superlocalmemory 4.0.8 → 4.0.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (147) hide show
  1. package/CHANGELOG.md +221 -1
  2. package/README.md +6 -6
  3. package/package.json +1 -1
  4. package/plugin/.claude-plugin/plugin.json +1 -1
  5. package/plugin/CLAUDE.md +3 -3
  6. package/plugin/agents/slm-governance-advisor.md +1 -1
  7. package/plugin/agents/slm-loop-runner.md +1 -1
  8. package/plugin/agents/slm-memory-advisor.md +1 -1
  9. package/plugin/agents/slm-optimize-advisor.md +1 -1
  10. package/plugin/requirements.txt +1 -1
  11. package/plugin/scripts/ensure-venv.sh +1 -1
  12. package/plugin/skills/slm-cache/SKILL.md +1 -1
  13. package/plugin/skills/slm-compress/SKILL.md +1 -1
  14. package/plugin/skills/slm-governance/SKILL.md +1 -1
  15. package/plugin/skills/slm-graph/SKILL.md +1 -1
  16. package/plugin/skills/slm-loop/SKILL.md +1 -1
  17. package/plugin/skills/slm-mesh/SKILL.md +1 -1
  18. package/plugin/skills/slm-profile/SKILL.md +1 -1
  19. package/plugin/skills/slm-recall/SKILL.md +1 -1
  20. package/plugin/skills/slm-remember/SKILL.md +1 -1
  21. package/plugin/skills/slm-scope/SKILL.md +1 -1
  22. package/plugin/skills/slm-session/SKILL.md +1 -1
  23. package/plugin/skills/slm-status/SKILL.md +3 -3
  24. package/plugin-src/rules/AGENTS.md +1 -1
  25. package/plugin-src/skills/slm-status/SKILL.md +2 -2
  26. package/pyproject.toml +1 -1
  27. package/scripts/postinstall.js +4 -0
  28. package/src/superlocalmemory/__init__.py +1 -1
  29. package/src/superlocalmemory/cli/_lazy_init.py +1 -1
  30. package/src/superlocalmemory/cli/commands.py +164 -11
  31. package/src/superlocalmemory/cli/db_migrate.py +0 -2
  32. package/src/superlocalmemory/cli/gdpr_io.py +1 -1
  33. package/src/superlocalmemory/cli/main.py +7 -7
  34. package/src/superlocalmemory/cli/service_installer.py +2 -1
  35. package/src/superlocalmemory/cli/setup_wizard.py +1 -1
  36. package/src/superlocalmemory/code_graph/bridge/maintenance.py +8 -0
  37. package/src/superlocalmemory/core/config.py +41 -7
  38. package/src/superlocalmemory/core/context_cache.py +0 -2
  39. package/src/superlocalmemory/core/engine.py +371 -63
  40. package/src/superlocalmemory/core/evidence_bundle.py +3 -1
  41. package/src/superlocalmemory/core/fact_consolidator.py +316 -125
  42. package/src/superlocalmemory/core/install_detector.py +131 -0
  43. package/src/superlocalmemory/core/maintenance.py +44 -6
  44. package/src/superlocalmemory/core/memory_health.py +266 -0
  45. package/src/superlocalmemory/core/operation_policy_registry.py +1 -1
  46. package/src/superlocalmemory/core/operation_request.py +1 -1
  47. package/src/superlocalmemory/core/ops_remediation.py +2 -2
  48. package/src/superlocalmemory/core/progressive_abstraction.py +1 -1
  49. package/src/superlocalmemory/core/security_primitives.py +3 -6
  50. package/src/superlocalmemory/core/store_pipeline.py +172 -29
  51. package/src/superlocalmemory/core/topic_signature.py +0 -2
  52. package/src/superlocalmemory/core/transactions/concrete_owners.py +15 -8
  53. package/src/superlocalmemory/dynamics/eap_scheduler.py +17 -6
  54. package/src/superlocalmemory/encoding/cognitive_consolidator.py +15 -1
  55. package/src/superlocalmemory/encoding/graph_builder.py +2 -2
  56. package/src/superlocalmemory/encoding/scene_builder.py +8 -2
  57. package/src/superlocalmemory/hooks/adapter_base.py +0 -2
  58. package/src/superlocalmemory/hooks/context_payload.py +0 -2
  59. package/src/superlocalmemory/hooks/portable_kit.py +8 -8
  60. package/src/superlocalmemory/hooks/post_tool_async_hook.py +0 -2
  61. package/src/superlocalmemory/hooks/prewarm_auth.py +0 -2
  62. package/src/superlocalmemory/hooks/user_prompt_hook.py +0 -2
  63. package/src/superlocalmemory/infra/backup.py +44 -8
  64. package/src/superlocalmemory/integrations/bounded_loops_mcp.py +24 -7
  65. package/src/superlocalmemory/learning/arm_catalog.py +0 -2
  66. package/src/superlocalmemory/learning/bandit.py +0 -2
  67. package/src/superlocalmemory/learning/bandit_cache.py +0 -2
  68. package/src/superlocalmemory/learning/dedup_hnsw.py +11 -11
  69. package/src/superlocalmemory/learning/ensemble.py +0 -2
  70. package/src/superlocalmemory/learning/labeler.py +0 -2
  71. package/src/superlocalmemory/learning/legacy_migration.py +0 -2
  72. package/src/superlocalmemory/learning/model_cache.py +0 -2
  73. package/src/superlocalmemory/learning/ranker.py +0 -2
  74. package/src/superlocalmemory/learning/reward_archive.py +6 -1
  75. package/src/superlocalmemory/learning/reward_proxy.py +0 -2
  76. package/src/superlocalmemory/learning/signal_worker.py +0 -2
  77. package/src/superlocalmemory/math/fisher.py +1 -1
  78. package/src/superlocalmemory/math/hopfield.py +4 -1
  79. package/src/superlocalmemory/math/langevin.py +1 -1
  80. package/src/superlocalmemory/math/sheaf.py +7 -3
  81. package/src/superlocalmemory/mcp/cli_fallback.py +1 -1
  82. package/src/superlocalmemory/mcp/profiles.py +1 -1
  83. package/src/superlocalmemory/mcp/server.py +2 -2
  84. package/src/superlocalmemory/mcp/session_binding.py +92 -0
  85. package/src/superlocalmemory/mcp/tools_active.py +56 -0
  86. package/src/superlocalmemory/mcp/tools_core.py +41 -40
  87. package/src/superlocalmemory/mcp/tools_ops.py +2 -2
  88. package/src/superlocalmemory/optimize/cache/manager.py +2 -2
  89. package/src/superlocalmemory/optimize/compress/ccr.py +1 -1
  90. package/src/superlocalmemory/optimize/compress/router.py +1 -1
  91. package/src/superlocalmemory/optimize/proxy/_helpers.py +2 -2
  92. package/src/superlocalmemory/optimize/proxy/server.py +1 -1
  93. package/src/superlocalmemory/optimize/proxy/vertex_surface.py +2 -2
  94. package/src/superlocalmemory/optimize/storage/db.py +2 -2
  95. package/src/superlocalmemory/retrieval/agentic.py +1 -1
  96. package/src/superlocalmemory/retrieval/ann_index.py +9 -2
  97. package/src/superlocalmemory/retrieval/bm25_channel.py +6 -10
  98. package/src/superlocalmemory/retrieval/bridge_discovery.py +2 -2
  99. package/src/superlocalmemory/retrieval/engine.py +272 -43
  100. package/src/superlocalmemory/retrieval/entity_channel.py +8 -2
  101. package/src/superlocalmemory/retrieval/hopfield_channel.py +8 -2
  102. package/src/superlocalmemory/retrieval/profile_channel.py +1 -1
  103. package/src/superlocalmemory/retrieval/quantization_aware_search.py +1 -1
  104. package/src/superlocalmemory/retrieval/remote_reranker.py +2 -2
  105. package/src/superlocalmemory/retrieval/reranker.py +3 -3
  106. package/src/superlocalmemory/retrieval/scope_policy.py +22 -1
  107. package/src/superlocalmemory/retrieval/semantic_channel.py +3 -3
  108. package/src/superlocalmemory/retrieval/spreading_activation.py +8 -8
  109. package/src/superlocalmemory/retrieval/strategy.py +94 -0
  110. package/src/superlocalmemory/retrieval/temporal_channel.py +179 -10
  111. package/src/superlocalmemory/retrieval/temporal_validity_filter.py +1 -1
  112. package/src/superlocalmemory/retrieval/vector_store.py +151 -10
  113. package/src/superlocalmemory/server/api.py +6 -1
  114. package/src/superlocalmemory/server/asset_versions.py +171 -0
  115. package/src/superlocalmemory/server/recall_serializer.py +10 -0
  116. package/src/superlocalmemory/server/routes/abstraction.py +201 -0
  117. package/src/superlocalmemory/server/routes/brain.py +2 -2
  118. package/src/superlocalmemory/server/routes/data_io.py +29 -1
  119. package/src/superlocalmemory/server/routes/entity.py +13 -1
  120. package/src/superlocalmemory/server/routes/mesh.py +1 -1
  121. package/src/superlocalmemory/server/routes/prewarm.py +4 -4
  122. package/src/superlocalmemory/server/routes/v3_api.py +2 -2
  123. package/src/superlocalmemory/server/ui.py +8 -1
  124. package/src/superlocalmemory/server/unified_daemon.py +570 -16
  125. package/src/superlocalmemory/storage/_migration_internals.py +4 -0
  126. package/src/superlocalmemory/storage/_schema_version.py +46 -3
  127. package/src/superlocalmemory/storage/backup.py +531 -0
  128. package/src/superlocalmemory/storage/database.py +139 -34
  129. package/src/superlocalmemory/storage/embedding_codec.py +129 -0
  130. package/src/superlocalmemory/storage/embedding_migrator.py +5 -3
  131. package/src/superlocalmemory/storage/migration_runner.py +153 -2
  132. package/src/superlocalmemory/storage/migrations/M043_quarantine_display_summaries.py +488 -0
  133. package/src/superlocalmemory/storage/migrations/__init__.py +1 -1
  134. package/src/superlocalmemory/storage/migrations.py +15 -1
  135. package/src/superlocalmemory/storage/models.py +7 -0
  136. package/src/superlocalmemory/storage/quantized_store.py +4 -2
  137. package/src/superlocalmemory/storage/schema.py +98 -0
  138. package/src/superlocalmemory/summaries/base.py +1 -1
  139. package/src/superlocalmemory/summaries/non_answer.py +223 -0
  140. package/src/superlocalmemory/summaries/session_summary.py +1 -1
  141. package/src/superlocalmemory/ui/css/legacy-dashboard.css +1 -1
  142. package/src/superlocalmemory/ui/css/neural-glass.css +1 -1
  143. package/src/superlocalmemory/ui/index.html +1 -1
  144. package/src/superlocalmemory/ui/js/core.js +1 -1
  145. package/src/superlocalmemory/ui/js/od-brain.js +1 -1
  146. package/src/superlocalmemory/ui/js/od-memories.js +190 -1
  147. package/src/superlocalmemory/ui/js/od-ops-health.js +1 -1
@@ -26,6 +26,8 @@ License: AGPL-3.0-or-later
26
26
  from __future__ import annotations
27
27
 
28
28
  import asyncio
29
+ import concurrent.futures
30
+ import functools
29
31
  import hashlib
30
32
  import json
31
33
  import logging
@@ -50,7 +52,7 @@ os.environ.setdefault("SLM_MCP_EMBEDDED", "1")
50
52
  from fastapi import FastAPI, HTTPException, Request
51
53
  from fastapi.middleware.cors import CORSMiddleware
52
54
  from fastapi.middleware.gzip import GZipMiddleware
53
- from pydantic import BaseModel
55
+ from pydantic import BaseModel, field_validator
54
56
 
55
57
  from superlocalmemory.core.config import CANONICAL_RECALL_LIMIT
56
58
  from superlocalmemory.infra.daemon_identity import (
@@ -67,6 +69,126 @@ from superlocalmemory.infra.data_root import (
67
69
  )
68
70
 
69
71
 
72
+ def _notify_migration_applied(
73
+ applied: list,
74
+ elapsed: float,
75
+ backup_dir: "Path | None" = None,
76
+ ) -> None:
77
+ """Print a one-line confirmation after migrations complete successfully.
78
+
79
+ Written to stdout so it is visible in daemon.log and in interactive runs.
80
+ """
81
+ count = len(applied)
82
+ if count == 0:
83
+ return
84
+ parts = [f"[SLM] {count} migration(s) applied in {elapsed:.1f}s."]
85
+ if backup_dir is not None:
86
+ parts.append(f"Backup: {backup_dir}")
87
+ print(" ".join(parts), flush=True)
88
+
89
+
90
+ def _write_migration_error_log(
91
+ failed: list,
92
+ backup_dir: "Path | None",
93
+ slm_home: "Path | None" = None,
94
+ applied: "list | None" = None,
95
+ ) -> "Path":
96
+ """Write a migration-error-{timestamp}.log to the SLM home directory.
97
+
98
+ Returns the Path of the written log file.
99
+ """
100
+ import datetime
101
+
102
+ if slm_home is None:
103
+ slm_home = canonical_data_root()
104
+
105
+ slm_home.mkdir(parents=True, exist_ok=True)
106
+ ts = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
107
+ log_path = slm_home / f"migration-error-{ts}.log"
108
+
109
+ lines = [
110
+ f"Migration error — {ts}",
111
+ f"Failed: {', '.join(str(f) for f in failed)}",
112
+ ]
113
+ # An upgrade is applied step by step and is deliberately non-fatal: some
114
+ # steps can succeed while a later one fails. Saying "your data was not
115
+ # modified" whenever a snapshot exists was therefore untrue in exactly the
116
+ # case that matters — and a user who believes nothing changed may delete the
117
+ # snapshot to reclaim space.
118
+ if applied:
119
+ lines.append(f"Applied before the failure: {', '.join(str(a) for a in applied)}")
120
+ lines.append(
121
+ "YOUR DATABASE WAS PARTIALLY CHANGED. The steps listed above "
122
+ "completed; the ones under 'Failed' did not."
123
+ )
124
+ elif applied is not None:
125
+ lines.append("No steps completed, so your database was not changed.")
126
+ if backup_dir is not None:
127
+ lines.append(f"Snapshot of the state before the upgrade: {backup_dir}")
128
+ lines.append(
129
+ "That snapshot is intact and can be restored. Keep it until you are "
130
+ "satisfied your data is correct."
131
+ )
132
+ else:
133
+ lines.append(
134
+ "NO SNAPSHOT WAS TAKEN for this attempt — do not assume a recovery "
135
+ "copy exists. Check for earlier snapshots before changing anything."
136
+ )
137
+ lines.append("")
138
+ lines.append("To diagnose, run: slm doctor")
139
+
140
+ log_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
141
+ return log_path
142
+
143
+
144
+ def _notify_windows_user(message: str) -> None:
145
+ """Best-effort notification on Windows. Silent no-op on other platforms."""
146
+ import sys
147
+
148
+ if sys.platform != "win32":
149
+ return
150
+
151
+ # Attempt 1 — Windows MessageBox via PowerShell
152
+ try:
153
+ import subprocess
154
+
155
+ safe_msg = message.replace("'", "").replace('"', "")
156
+ subprocess.run(
157
+ [
158
+ "powershell",
159
+ "-Command",
160
+ "[System.Reflection.Assembly]::LoadWithPartialName"
161
+ "('System.Windows.Forms') | Out-Null; "
162
+ f"[System.Windows.Forms.MessageBox]::Show('{safe_msg}', "
163
+ "'SuperLocalMemory')",
164
+ ],
165
+ check=False,
166
+ timeout=5,
167
+ )
168
+ return
169
+ except Exception:
170
+ pass
171
+
172
+ # Attempt 2 — Windows Application EventLog
173
+ try:
174
+ import subprocess
175
+
176
+ subprocess.run(
177
+ [
178
+ "eventcreate",
179
+ "/T", "INFORMATION",
180
+ "/ID", "1001",
181
+ "/L", "APPLICATION",
182
+ "/SO", "SuperLocalMemory",
183
+ "/D", message[:512],
184
+ ],
185
+ check=False,
186
+ timeout=5,
187
+ )
188
+ except Exception:
189
+ pass # best-effort only
190
+
191
+
70
192
  def _learning_db_for_config(config) -> Path:
71
193
  """Single learning.db path for daemon migration + engine (F-05).
72
194
 
@@ -115,11 +237,142 @@ _SOURCE_QUALITY_MAX_BATCH_SIZE = 250
115
237
  _SOURCE_QUALITY_PROFILE_REFRESH_SECONDS = 60.0
116
238
  _FACT_ENTITY_REPAIR_MIN_RETRY_SECONDS = 0.05
117
239
  _FACT_ENTITY_REPAIR_MAX_RETRY_SECONDS = 30.0
118
- # ``wait=true`` is a compatibility affordance, never permission to hold the
119
- # ASGI event loop hostage to a local LLM. Normal clients omit it and receive
120
- # an immediate durable/queryable receipt; explicit waiters get this small
121
- # completion window, then the M018 materializer continues in the background.
122
- _REMEMBER_ENRICHMENT_WAIT_SECONDS = 0.75
240
+ # How long a write may spend making itself findable by meaning before the
241
+ # receipt goes out. Storing a memory has a 1.5 s ceiling, and this is a ceiling
242
+ # rather than a target: the cost of coming in under it is a memory that can only
243
+ # be found by quoting its own wording, which is not how anyone asks. So the
244
+ # window is most of the budget, not a token amount of it. ``wait=true`` is still
245
+ # never permission to hold the event loop open indefinitely — past this the
246
+ # background pass finishes the job.
247
+ # The ceiling for a whole store request, shared by the durable write and the
248
+ # window that makes the memory findable by meaning. Both are sequential, so the
249
+ # second gets what the first left rather than a fresh grant.
250
+ _REMEMBER_TOTAL_CEILING_SECONDS = 1.5
251
+ _REMEMBER_ENRICHMENT_WAIT_SECONDS = 1.2
252
+
253
+ # Enrichment runs off the event loop, but NOT on the loop's default executor.
254
+ # `asyncio.to_thread` uses that default pool, so N simultaneous writers really did
255
+ # create N threads — and every other handler that borrows the same pool queued
256
+ # behind them. One shared, small pool bounds the cost of a burst: past its width
257
+ # the extra callers miss their deadline and defer, which is the intended
258
+ # degradation, instead of multiplying threads.
259
+ _ENRICHMENT_WORKERS = 2
260
+ _enrichment_pool: "concurrent.futures.ThreadPoolExecutor | None" = None
261
+ _enrichment_pool_lock = threading.Lock()
262
+
263
+ # Bounds how much enrichment may be in flight at once. A ThreadPoolExecutor
264
+ # queues without limit, so bounding the pool's WIDTH does not bound what can be
265
+ # handed to it: a burst of writers all submit, the first two run, and the rest sit
266
+ # in a queue nobody is waiting for any more — then wake up and write to the
267
+ # database after their callers have already been answered. A permit is taken
268
+ # before submitting and released by the worker when it has really finished, so at
269
+ # most `_ENRICHMENT_WORKERS` pieces of work exist at once and a caller that
270
+ # cannot get one is told immediately instead of being queued.
271
+ _enrichment_semaphore = threading.Semaphore(_ENRICHMENT_WORKERS)
272
+
273
+ # Set once the pool has been shut down, and never cleared. Without it, a caller
274
+ # that passed the unlocked check below arrives after shutdown, finds None, and
275
+ # builds a replacement pool that the completed shutdown will never reach.
276
+ _enrichment_pool_closed = False
277
+
278
+
279
+ class _EnrichmentAtCapacity(Exception):
280
+ """No permit was free, so nothing was submitted.
281
+
282
+ Its own type, so that "we chose not to start" is never reported through the
283
+ same path as "it started and then failed" — those are different events and
284
+ only one of them is worth a warning about degradation.
285
+ """
286
+
287
+
288
+ def _enrichment_executor():
289
+ """The one pool inline enrichment may use. Created on first need.
290
+
291
+ Raises ``RuntimeError`` once the pool has been shut down. It deliberately
292
+ does not return ``None`` on that path: ``run_in_executor(None, ...)`` means
293
+ *the event loop's default executor*, which is the unbounded-thread behaviour
294
+ this pool exists to replace, so a None here would silently reintroduce it at
295
+ exactly the moment the process is trying to stop.
296
+ """
297
+ global _enrichment_pool
298
+ # Everything happens under the lock, including the read that is returned. An
299
+ # earlier version checked the closed flag only on the create branch and then
300
+ # returned the global, so a caller that found a live pool, lost the CPU, and
301
+ # resumed after shutdown returned None — and None means the event loop's own
302
+ # default executor, unbounded and shared with every other handler. That is
303
+ # the precise failure this function exists to prevent, reintroduced at the
304
+ # one moment it matters.
305
+ with _enrichment_pool_lock:
306
+ if _enrichment_pool_closed:
307
+ raise RuntimeError("enrichment pool is shut down")
308
+ if _enrichment_pool is None:
309
+ _enrichment_pool = concurrent.futures.ThreadPoolExecutor(
310
+ max_workers=_ENRICHMENT_WORKERS,
311
+ thread_name_prefix="slm-enrich",
312
+ )
313
+ pool = _enrichment_pool
314
+ if pool is None: # pragma: no cover — belt and braces around the invariant
315
+ raise RuntimeError("enrichment pool is unavailable")
316
+ return pool
317
+
318
+
319
+ def _open_enrichment_pool() -> None:
320
+ """Allow the pool to be created again, for a fresh run in this process.
321
+
322
+ The closed flag has to survive shutdown — that is its whole purpose — so
323
+ something has to clear it when a new run legitimately begins. Without this a
324
+ second startup in one process (a restart, or a test that builds the app more
325
+ than once) would find enrichment permanently refused and every write would
326
+ come back findable by wording only, with nothing in the logs to explain it.
327
+ """
328
+ global _enrichment_pool_closed, _enrichment_semaphore
329
+ with _enrichment_pool_lock:
330
+ _enrichment_pool_closed = False
331
+ # A fresh run starts with its full capacity. Shutdown cancels work that
332
+ # was queued and never started, and a task that never starts never runs
333
+ # the release in its finally — so permits taken by cancelled work are
334
+ # gone. Without this reset the next run in the same process would run at
335
+ # reduced capacity, or none at all, and the only symptom would be every
336
+ # write coming back findable by wording with nothing in the log.
337
+ _enrichment_semaphore = threading.Semaphore(_ENRICHMENT_WORKERS)
338
+
339
+
340
+ def _enrich_and_release(engine, fact_ids: list[str], budget: float) -> int:
341
+ """Run inline enrichment, releasing the permit only when it is really done.
342
+
343
+ The permit has to be released here rather than by the caller. A caller that
344
+ stops waiting at its own deadline has not finished the work — the worker is
345
+ still holding a pool thread — so releasing on the caller's timeout would let
346
+ the next request submit into a pool that is still fully occupied, which is
347
+ the unbounded queueing this permit exists to prevent.
348
+ """
349
+ try:
350
+ return engine.enrich_new_facts_now(fact_ids, timeout_s=budget)
351
+ finally:
352
+ _enrichment_semaphore.release()
353
+
354
+
355
+ def _shutdown_enrichment_pool_if_created() -> None:
356
+ """Shut down the enrichment pool if it was ever created.
357
+
358
+ Safe to call when the pool was never created (no-op) and safe to call
359
+ twice (second call is a no-op because the pool is set to None under the
360
+ lock before shutdown is called).
361
+
362
+ cancel_futures=True drops any work that is still queued but has not yet
363
+ started. Workers already inside embed() will finish their current call
364
+ and then stop; they do not block the caller because wait=False.
365
+ """
366
+ global _enrichment_pool, _enrichment_pool_closed
367
+ with _enrichment_pool_lock:
368
+ # Marked closed under the same lock that guards creation, so a caller
369
+ # already past the unlocked check cannot build a replacement pool that
370
+ # this shutdown has already walked past.
371
+ _enrichment_pool_closed = True
372
+ pool = _enrichment_pool
373
+ _enrichment_pool = None
374
+ if pool is not None:
375
+ pool.shutdown(wait=False, cancel_futures=True)
123
376
  _SENSITIVE_READ_PREFIXES = (
124
377
  "/api/memories", "/api/facts", "/api/clusters", "/api/graph",
125
378
  "/api/v3/associations", "/api/v3/core-memory",
@@ -334,6 +587,62 @@ class RememberRequest(BaseModel):
334
587
  metadata: dict | None = None # v3.4.26: pass-through from MCP pool_store
335
588
  idempotency_key: str | None = None
336
589
  session_id: str = ""
590
+ #: WHEN this memory is about, as distinct from when it was written.
591
+ #:
592
+ #: The internal admission record has carried this field all along and this
593
+ #: model never had it, so every memory arriving over HTTP — which is every
594
+ #: memory, from the CLI, the tool interface and the dashboard alike — was
595
+ #: stamped with its ingestion date. Measured on the author's store: 200 of
596
+ #: the 200 most recent facts have an observation_date, and 196 of them are
597
+ #: the day they were written. A store that cannot be told "this happened in
598
+ #: March" cannot answer a question about March.
599
+ #:
600
+ #: Empty means "today", the previous behaviour. Format is YYYY-MM-DD or a
601
+ #: full ISO 8601 timestamp.
602
+ session_date: str = ""
603
+
604
+ @field_validator("session_date")
605
+ @classmethod
606
+ def _session_date_is_a_date(cls, value: str) -> str:
607
+ """Reject a malformed date rather than ignore it.
608
+
609
+ Dropping it silently would leave the caller believing the date was
610
+ recorded while the memory quietly filed itself under today — which is
611
+ the exact failure this field exists to fix, reintroduced one layer up.
612
+ Rejecting is recoverable: the caller sees the error and resends.
613
+
614
+ ISO ONLY, DELIBERATELY, even though this system ships a parser that is
615
+ far more permissive. ``encoding/temporal_parser.parse_session_date``
616
+ accepts "May 8, 2026", "1:56 pm on 8 May, 2026", "14/03/2026", and also
617
+ "last tuesday" and "March 2026" — and those last two are why it is not
618
+ used here. Asked on 2026-08-21 it resolves "last tuesday" to
619
+ **2026-08-25**, four days into the future, and "March 2026" to the 21st,
620
+ a day it invents. Accepting either at this boundary would file a memory
621
+ under a confidently wrong date, silently, which is precisely the class
622
+ of defect this field was added to remove.
623
+
624
+ The caller here is a program — the command line, the tool interface, the
625
+ dashboard — not a person typing. ISO is the right contract for a
626
+ program, and a caller holding a human-typed date can run it through the
627
+ parser itself and send the result.
628
+ """
629
+ if not value:
630
+ return value
631
+ from datetime import datetime as _dt
632
+
633
+ text = value.strip()
634
+ try:
635
+ _dt.fromisoformat(text.replace("Z", "+00:00"))
636
+ except ValueError:
637
+ raise ValueError(
638
+ "session_date must be YYYY-MM-DD or a full ISO 8601 timestamp; "
639
+ f"got {value!r}. To accept looser human phrasing, parse it "
640
+ "first with TemporalParser.parse_session_date and send the ISO "
641
+ "result — but check what it returns, because it resolves "
642
+ "relative phrases against today and can answer with a future "
643
+ "date."
644
+ ) from None
645
+ return text
337
646
  # v3.6.15 multi-scope: visibility of the new memory. ``None`` scope means
338
647
  # "use the configured default_scope" (personal). shared_with is the list of
339
648
  # profile_ids for scope='shared'.
@@ -1500,6 +1809,11 @@ async def lifespan(application: FastAPI):
1500
1809
  """Initialize engine, workers, and optional services on startup."""
1501
1810
  global _last_activity
1502
1811
 
1812
+ # A previous run in this process left the enrichment pool closed so that a
1813
+ # write racing its shutdown could not resurrect it. This run is entitled to
1814
+ # one.
1815
+ _open_enrichment_pool()
1816
+
1503
1817
  engine = None
1504
1818
  config = None
1505
1819
  deployment = None
@@ -1592,6 +1906,7 @@ async def lifespan(application: FastAPI):
1592
1906
  _path_config = None
1593
1907
 
1594
1908
  try:
1909
+ from superlocalmemory.storage.backup import InsufficientDiskSpaceError
1595
1910
  from superlocalmemory.storage.migration_runner import apply_all
1596
1911
  if _path_config is None:
1597
1912
  # Catastrophic early-load failure: still one root via Mode-A default
@@ -1602,13 +1917,56 @@ async def lifespan(application: FastAPI):
1602
1917
  _path_config = _SLMFallback.for_mode(_ModeFallback.A)
1603
1918
  _learning_db = _learning_db_for_config(_path_config)
1604
1919
  _memory_db = _memory_db_for_config(_path_config)
1920
+ import time as _time_mod
1921
+ _t0 = _time_mod.monotonic()
1605
1922
  _result = apply_all(_learning_db, _memory_db)
1923
+ _elapsed = _time_mod.monotonic() - _t0
1606
1924
  _applied = _result.get("applied", [])
1607
1925
  _failed = _result.get("failed", [])
1926
+ _backup_dir = _result.get("details", {}).get("_backup")
1927
+ _backup_path = Path(_backup_dir) if _backup_dir else None
1608
1928
  if _applied:
1609
1929
  logger.info("migrations applied: %s", _applied)
1930
+ _notify_migration_applied(_applied, _elapsed, _backup_path)
1931
+ _notify_windows_user(
1932
+ f"SuperLocalMemory: {len(_applied)} database migration(s) applied."
1933
+ )
1610
1934
  if _failed:
1611
1935
  logger.warning("migrations failed (non-fatal): %s", _failed)
1936
+ try:
1937
+ _err_log = _write_migration_error_log(
1938
+ _failed, _backup_path,
1939
+ slm_home=_memory_db.parent if _memory_db else None,
1940
+ applied=_applied,
1941
+ )
1942
+ import sys as _sys
1943
+ # "Data is safe" is only true when NOTHING was applied. Migrations
1944
+ # are non-fatal and applied in order, so a later failure can leave
1945
+ # earlier ones committed — a partially changed store. Saying the
1946
+ # data is untouched then tells the user to ignore the one copy
1947
+ # that still holds the original.
1948
+ if _applied:
1949
+ _headline = (
1950
+ f"[SLM] Migration incomplete. YOUR DATABASE WAS PARTIALLY "
1951
+ f"CHANGED — {len(_applied)} step(s) applied, "
1952
+ f"{len(_failed)} failed. The pre-change copy is at "
1953
+ f"{_backup_path} — keep it until this is resolved."
1954
+ )
1955
+ else:
1956
+ _headline = (
1957
+ f"[SLM] Migration failed before any change was made. Your "
1958
+ f"data is as it was; a copy is also at {_backup_path}."
1959
+ )
1960
+ print(
1961
+ f"{_headline} See {_err_log}. Run: slm doctor",
1962
+ file=_sys.stderr,
1963
+ flush=True,
1964
+ )
1965
+ _notify_windows_user(
1966
+ f"SuperLocalMemory migration failed. Run slm doctor. Log: {_err_log}"
1967
+ )
1968
+ except Exception:
1969
+ pass # notification failures are non-fatal
1612
1970
  application.state.migration_result = _result
1613
1971
  # S9-SKEP-15: only commit the new `.last_version` AFTER migrations
1614
1972
  # complete with zero failures. A partial upgrade (schema didn't
@@ -1626,6 +1984,35 @@ async def lifespan(application: FastAPI):
1626
1984
  _version_marker.write_text(_slm_version, encoding="utf-8")
1627
1985
  except OSError:
1628
1986
  pass # non-fatal
1987
+ except InsufficientDiskSpaceError as _disk_exc:
1988
+ # This is not a crash and it must not be reported as "non-fatal". The
1989
+ # migration deliberately refused to start because there was not enough
1990
+ # room to keep a recoverable copy first. The store is intact and
1991
+ # unmigrated; the generic handler below would have logged a warning and
1992
+ # left the daemon serving as though nothing had happened.
1993
+ logger.error(
1994
+ "MIGRATION NOT RUN — not enough free disk to keep a recoverable "
1995
+ "copy first. Your data has NOT been modified. Need %s bytes, have "
1996
+ "%s free. Free some space and restart; run `slm doctor` for details.",
1997
+ f"{_disk_exc.needed_bytes:,}", f"{_disk_exc.free_bytes:,}",
1998
+ )
1999
+ try:
2000
+ _log_path = _write_migration_error_log(
2001
+ ["_insufficient_disk_space"], None,
2002
+ slm_home=locals().get("_memory_db").parent if locals().get("_memory_db") else None,
2003
+ applied=[],
2004
+ )
2005
+ logger.error("Details written to %s", _log_path)
2006
+ except OSError:
2007
+ pass
2008
+ application.state.migration_result = {
2009
+ "applied": [], "skipped": [], "failed": ["_insufficient_disk_space"],
2010
+ "details": {
2011
+ "_needed_bytes": _disk_exc.needed_bytes,
2012
+ "_free_bytes": _disk_exc.free_bytes,
2013
+ "_store_modified": False,
2014
+ },
2015
+ }
1629
2016
  except Exception as _exc:
1630
2017
  logger.warning("migration runner crashed (non-fatal): %s", _exc)
1631
2018
  application.state.migration_result = {
@@ -2068,6 +2455,10 @@ async def lifespan(application: FastAPI):
2068
2455
  "state": "checking_components", "embeddings_backfilled": 0,
2069
2456
  "expansion_backfilled": 0, "null_remaining": None,
2070
2457
  "components": None,
2458
+ # Declared here so /status carries the same keys whatever
2459
+ # happens; a field that only appears on failure is a field
2460
+ # nobody's dashboard renders.
2461
+ "incomplete_reason": None,
2071
2462
  "started_at": _t.time(), "finished_at": None,
2072
2463
  }
2073
2464
  # Step 0 (v3.8.2 "whole self-healer"): repair components that
@@ -2176,9 +2567,42 @@ async def lifespan(application: FastAPI):
2176
2567
  _backfill_vector_store()
2177
2568
  except Exception as exc:
2178
2569
  logger.warning("Self-heal vector index failed (non-fatal): %s", exc)
2179
- _SELF_HEAL_STATUS["state"] = "complete"
2570
+
2571
+ # "complete" has to mean complete.
2572
+ #
2573
+ # This line used to set "complete" unconditionally, so a
2574
+ # backfill that gave up after five no-progress attempts, or ran
2575
+ # out its 500-iteration budget, reported success with facts
2576
+ # still unembedded — and an unembedded fact cannot be found by
2577
+ # meaning at all. That is how a machine sat at 56.3% of its
2578
+ # memory reachable while its own status endpoint said the heal
2579
+ # had finished. Nobody was going to look past a green light.
2580
+ #
2581
+ # Now the state is derived from the remaining count, and the
2582
+ # gap is named in plain language for the dashboard, so an
2583
+ # incomplete heal is visible and gets retried on next start
2584
+ # instead of being declared done forever.
2585
+ _remaining = _SELF_HEAL_STATUS.get("null_remaining")
2180
2586
  _SELF_HEAL_STATUS["finished_at"] = _t.time()
2181
- logger.info("Self-heal complete: %s", _SELF_HEAL_STATUS)
2587
+ if _remaining is None:
2588
+ _SELF_HEAL_STATUS["state"] = "complete"
2589
+ elif int(_remaining) > 0:
2590
+ _SELF_HEAL_STATUS["state"] = "incomplete"
2591
+ _SELF_HEAL_STATUS["incomplete_reason"] = (
2592
+ f"{int(_remaining)} memories still have no meaning "
2593
+ f"vector, so they cannot be found by asking a question. "
2594
+ f"This retries automatically next time the service "
2595
+ f"starts. It usually means the embedding model was "
2596
+ f"unavailable."
2597
+ )
2598
+ logger.warning(
2599
+ "Self-heal INCOMPLETE: %d facts still unembedded and "
2600
+ "therefore unreachable by meaning; will retry on next "
2601
+ "start. %s", int(_remaining), _SELF_HEAL_STATUS,
2602
+ )
2603
+ else:
2604
+ _SELF_HEAL_STATUS["state"] = "complete"
2605
+ logger.info("Self-heal complete: %s", _SELF_HEAL_STATUS)
2182
2606
  except Exception as exc:
2183
2607
  _SELF_HEAL_STATUS["state"] = "error"
2184
2608
  logger.warning("Self-heal failed (non-fatal): %s", exc)
@@ -2822,6 +3246,19 @@ async def lifespan(application: FastAPI):
2822
3246
  except Exception as exc: # pragma: no cover — defensive
2823
3247
  logger.warning("perf_log flush failed: %s", exc)
2824
3248
 
3249
+ # Shut down the enrichment pool before engine.close() so that any worker
3250
+ # still inside embed() receives cancellation before the embed pool is torn
3251
+ # down. shutdown(wait=False, cancel_futures=True) is non-blocking: queued
3252
+ # futures are dropped and running futures complete their current call on
3253
+ # their own. This prevents Python's atexit handler from calling
3254
+ # shutdown(wait=True) on a live embed worker, which would push the daemon
3255
+ # past the graceful-shutdown budget and cause the service manager to
3256
+ # SIGKILL the process.
3257
+ try:
3258
+ _shutdown_enrichment_pool_if_created()
3259
+ except Exception as exc: # pragma: no cover — defensive
3260
+ logger.warning("enrichment pool shutdown failed (non-fatal): %s", exc)
3261
+
2825
3262
  materializer_stopped = _stop_pending_materializer()
2826
3263
  canonical_writer_stopped = _release_canonical_remember_runtime(application)
2827
3264
  _profile_runtime = None
@@ -3631,8 +4068,16 @@ def _register_dashboard_routes(application: FastAPI) -> None:
3631
4068
  # v3.4.23: substitute version placeholder so the dashboard can detect
3632
4069
  # upgrades and auto-reload. Read fresh each request (daemon uptime is
3633
4070
  # days, but we want zero caching surprises during development).
3634
- html = index_path.read_text()
3635
- return html.replace("__SLM_VERSION__", _SLM_VERSION)
4071
+ #
4072
+ # 4.0.10: asset ?v= strings are now derived from file content instead of
4073
+ # being hand-written literals that tracked nothing. See
4074
+ # server/asset_versions.py — including what that does and does not fix.
4075
+ from superlocalmemory.server.asset_versions import render_index
4076
+
4077
+ return render_index(
4078
+ index_path, UI_DIR,
4079
+ substitutions={"__SLM_VERSION__": _SLM_VERSION},
4080
+ )
3636
4081
 
3637
4082
  @application.get("/favicon.ico", include_in_schema=False)
3638
4083
  async def favicon():
@@ -3802,7 +4247,7 @@ def _register_daemon_routes(application: FastAPI) -> None:
3802
4247
  "runtime_state": runtime_state,
3803
4248
  "active_profile": profile_snapshot.profile_id,
3804
4249
  "profile_generation": profile_snapshot.generation,
3805
- # Wave-3: operational failure counts (visible to all team members)
4250
+ # operational failure counts (visible to all team members)
3806
4251
  **_ops_failure_counts(engine, application),
3807
4252
  # issue #107: does this daemon's *imported* code still match the
3808
4253
  # installed distribution? ``version`` above reports what this
@@ -4169,12 +4614,17 @@ def _register_daemon_routes(application: FastAPI) -> None:
4169
4614
  shared_with=tuple(shared_with or ()),
4170
4615
  trusted_actor_id=trusted_actor_id,
4171
4616
  session_id=req.session_id,
4617
+ session_date=req.session_date,
4172
4618
  )
4173
4619
  actor = Actor(
4174
4620
  principal_id=trusted_actor_id,
4175
4621
  allowed_profiles=frozenset({engine._profile_id}),
4176
4622
  allowed_scopes=frozenset({scope}),
4177
4623
  )
4624
+ # The whole request has a 1.5 s ceiling, and the two phases below
4625
+ # are sequential, so the ceiling has to be shared between them
4626
+ # rather than granted twice. Measured from here.
4627
+ _store_started = time.monotonic()
4178
4628
  receipt = await asyncio.to_thread(
4179
4629
  runtime.remember,
4180
4630
  admission,
@@ -4183,10 +4633,105 @@ def _register_daemon_routes(application: FastAPI) -> None:
4183
4633
  )
4184
4634
  payload = dict(receipt.payload)
4185
4635
  fact_ids = list(payload.get("fact_ids") or [])
4636
+
4637
+ # The durable receipt is already committed above; nothing below can
4638
+ # fail this write. What remains is the window in which the memory can
4639
+ # only be found by quoting its own wording — a fact with no vector is
4640
+ # invisible to the semantic channel, so the memory describing what is
4641
+ # happening right now is the hardest one to find.
4642
+ #
4643
+ # Every entry point — command line, tool interface, dashboard —
4644
+ # arrives here, so closing the window here closes it for all of them
4645
+ # rather than for whichever door happened to be used.
4646
+ #
4647
+ # Strictly bounded, because this daemon serves many sessions at once:
4648
+ # the work runs off the event loop, on a two-worker pool, and yields
4649
+ # to any recall in flight. Past the deadline the caller gets exactly
4650
+ # the previous behaviour and the background materializer finishes the
4651
+ # job.
4652
+ # A caller that did not ask to wait still gets a real attempt, not a
4653
+ # token one. Almost no client passes wait=true, so a small budget
4654
+ # here meant almost every memory in practice was returned findable by
4655
+ # wording only — the default path deciding the product's behaviour.
4656
+ #
4657
+ # But it gets what is LEFT of the ceiling, not a second full budget.
4658
+ # The durable write above can legitimately consume most of the 1.5 s
4659
+ # under contention, and granting the full enrichment window on top of
4660
+ # that produced receipts well past 3 s — over a ceiling the changelog
4661
+ # states. A caller that asked to wait may exceed the shared ceiling,
4662
+ # because that is what asking to wait means; nobody else may.
4663
+ _elapsed = time.monotonic() - _store_started
4664
+ _remaining = _REMEMBER_TOTAL_CEILING_SECONDS - _elapsed
4665
+ if wait:
4666
+ enrich_budget = _REMEMBER_ENRICHMENT_WAIT_SECONDS
4667
+ else:
4668
+ enrich_budget = min(
4669
+ 1.0, _REMEMBER_ENRICHMENT_WAIT_SECONDS, max(0.0, _remaining),
4670
+ )
4671
+ if enrich_budget <= 0.0:
4672
+ logger.warning(
4673
+ "the durable write used the whole %.1fs budget (%.2fs) — this "
4674
+ "memory is findable by its wording and the background pass "
4675
+ "will attach its vector",
4676
+ _REMEMBER_TOTAL_CEILING_SECONDS, _elapsed,
4677
+ )
4678
+ enriched = 0
4679
+ # Taken before submitting, released by the worker. If none is free
4680
+ # the pool is already saturated, and queueing behind it would mean
4681
+ # writing to the database after this response has been sent — so the
4682
+ # write is simply reported as findable by wording and the background
4683
+ # pass picks it up, which is the intended degradation.
4684
+ _permit = _enrichment_semaphore.acquire(blocking=False)
4685
+ if not _permit:
4686
+ logger.warning(
4687
+ "inline enrichment at capacity for %d fact(s) — deferred to "
4688
+ "the background pass", len(fact_ids),
4689
+ )
4690
+ try:
4691
+ if not _permit:
4692
+ raise _EnrichmentAtCapacity
4693
+ loop = asyncio.get_running_loop()
4694
+ try:
4695
+ _pending = loop.run_in_executor(
4696
+ _enrichment_executor(),
4697
+ functools.partial(
4698
+ _enrich_and_release,
4699
+ engine, fact_ids, enrich_budget,
4700
+ ),
4701
+ )
4702
+ except BaseException:
4703
+ # Never handed to a worker, so nothing will release it.
4704
+ _enrichment_semaphore.release()
4705
+ raise
4706
+ enriched = await asyncio.wait_for(
4707
+ _pending, timeout=enrich_budget + 0.25,
4708
+ )
4709
+ except _EnrichmentAtCapacity:
4710
+ pass
4711
+ except (TimeoutError, asyncio.TimeoutError):
4712
+ # Worth seeing. Sustained timeouts mean writes are coming back
4713
+ # findable by wording only, which is a real degradation and was
4714
+ # previously visible at debug level alone.
4715
+ logger.warning(
4716
+ "inline enrichment exceeded its budget for %d fact(s) — "
4717
+ "deferred to the background pass", len(fact_ids),
4718
+ )
4719
+ except Exception as exc:
4720
+ logger.warning(
4721
+ "inline enrichment unavailable (%s: %s) — deferred to the "
4722
+ "background pass", type(exc).__name__, exc,
4723
+ )
4724
+
4725
+ searchable = "meaning" if enriched == len(fact_ids) and fact_ids else "wording"
4186
4726
  return {
4187
4727
  "ok": True,
4188
4728
  "fact_ids": fact_ids,
4189
4729
  "count": len(fact_ids),
4730
+ # Storing a memory and being able to find it again are different
4731
+ # things, and reporting only the first is how a caller ends up
4732
+ # believing a memory is retrievable when it is not yet.
4733
+ "searchable_by": searchable,
4734
+ "enriched_now": enriched,
4190
4735
  "operation_id": payload["operation_id"],
4191
4736
  # One-release compatibility alias. The durable operation ID is
4192
4737
  # opaque and replaces the integer pending.db row identifier.
@@ -4194,8 +4739,17 @@ def _register_daemon_routes(application: FastAPI) -> None:
4194
4739
  "status": "queryable",
4195
4740
  "materialization_state": payload["materialization_state"],
4196
4741
  "commit_sequence": payload.get("commit_sequence"),
4197
- "note": "queryable now; canonical enrichment continues in the background",
4198
- "wait_ignored": bool(wait),
4742
+ "note": (
4743
+ "stored and searchable by meaning"
4744
+ if searchable == "meaning"
4745
+ else "stored and searchable by wording; "
4746
+ "searchable by meaning shortly"
4747
+ ),
4748
+ # Retained for callers that already read it. It used to be
4749
+ # unconditionally true, because `wait` bought nothing. It now
4750
+ # buys a larger best-effort enrichment budget — still never a
4751
+ # guarantee, and still never a block on durability.
4752
+ "wait_ignored": False if wait else True,
4199
4753
  }
4200
4754
  except Exception as exc:
4201
4755
  from superlocalmemory.core.remember_admission import AdmissionRejected
@@ -4407,7 +4961,7 @@ def _register_daemon_routes(application: FastAPI) -> None:
4407
4961
  # index backfill after an upgrade). Dashboard renders a plain
4408
4962
  # "Optimizing memory…" line from this. Defaults to idle before start.
4409
4963
  "self_heal": globals().get("_SELF_HEAL_STATUS", {"state": "idle"}),
4410
- # Wave-3: operational failure counts (dead-letter, degraded, stalled)
4964
+ # operational failure counts (dead-letter, degraded, stalled)
4411
4965
  **_ops_failure_counts(engine, application),
4412
4966
  }
4413
4967
 
@@ -4461,7 +5015,7 @@ def _register_daemon_routes(application: FastAPI) -> None:
4461
5015
  return {"status": "started"}
4462
5016
 
4463
5017
  # ------------------------------------------------------------------
4464
- # Wave-3: Operational Recovery & Admin Remediation (V4 resilience slice)
5018
+ # Operational Recovery & Admin Remediation (V4 resilience slice)
4465
5019
  # ------------------------------------------------------------------
4466
5020
 
4467
5021
  @application.get("/operations/failed")
@@ -5008,7 +5562,7 @@ def _terminalize_orphan_operation(engine, operation_id: str) -> None:
5008
5562
 
5009
5563
 
5010
5564
  def _ops_failure_counts(engine, application) -> dict:
5011
- """Return Wave-3 operational failure counts for /status and /health.
5565
+ """Return operational failure counts for /status and /health.
5012
5566
 
5013
5567
  Always returns a dict (never raises). Counts default to 0 on any error.
5014
5568
  Includes: dead_letter_count, degraded_operations, exhausted_obligations,