superlocalmemory 4.0.8 → 4.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +146 -1
- package/README.md +6 -6
- package/package.json +1 -1
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/CLAUDE.md +3 -3
- package/plugin/agents/slm-governance-advisor.md +1 -1
- package/plugin/agents/slm-loop-runner.md +1 -1
- package/plugin/agents/slm-memory-advisor.md +1 -1
- package/plugin/agents/slm-optimize-advisor.md +1 -1
- package/plugin/requirements.txt +1 -1
- package/plugin/scripts/ensure-venv.sh +1 -1
- package/plugin/skills/slm-cache/SKILL.md +1 -1
- package/plugin/skills/slm-compress/SKILL.md +1 -1
- package/plugin/skills/slm-governance/SKILL.md +1 -1
- package/plugin/skills/slm-graph/SKILL.md +1 -1
- package/plugin/skills/slm-loop/SKILL.md +1 -1
- package/plugin/skills/slm-mesh/SKILL.md +1 -1
- package/plugin/skills/slm-profile/SKILL.md +1 -1
- package/plugin/skills/slm-recall/SKILL.md +1 -1
- package/plugin/skills/slm-remember/SKILL.md +1 -1
- package/plugin/skills/slm-scope/SKILL.md +1 -1
- package/plugin/skills/slm-session/SKILL.md +1 -1
- package/plugin/skills/slm-status/SKILL.md +3 -3
- package/plugin-src/rules/AGENTS.md +1 -1
- package/plugin-src/skills/slm-status/SKILL.md +2 -2
- package/pyproject.toml +1 -1
- package/scripts/postinstall.js +4 -0
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/cli/_lazy_init.py +1 -1
- package/src/superlocalmemory/cli/commands.py +119 -9
- package/src/superlocalmemory/cli/db_migrate.py +0 -2
- package/src/superlocalmemory/cli/gdpr_io.py +1 -1
- package/src/superlocalmemory/cli/main.py +5 -5
- package/src/superlocalmemory/cli/service_installer.py +2 -1
- package/src/superlocalmemory/cli/setup_wizard.py +1 -1
- package/src/superlocalmemory/core/config.py +41 -7
- package/src/superlocalmemory/core/context_cache.py +0 -2
- package/src/superlocalmemory/core/engine.py +371 -63
- package/src/superlocalmemory/core/evidence_bundle.py +3 -1
- package/src/superlocalmemory/core/install_detector.py +131 -0
- package/src/superlocalmemory/core/progressive_abstraction.py +1 -1
- package/src/superlocalmemory/core/security_primitives.py +3 -6
- package/src/superlocalmemory/core/store_pipeline.py +94 -26
- package/src/superlocalmemory/core/topic_signature.py +0 -2
- package/src/superlocalmemory/core/transactions/concrete_owners.py +15 -8
- package/src/superlocalmemory/dynamics/eap_scheduler.py +17 -6
- package/src/superlocalmemory/encoding/graph_builder.py +2 -2
- package/src/superlocalmemory/encoding/scene_builder.py +8 -2
- package/src/superlocalmemory/hooks/adapter_base.py +0 -2
- package/src/superlocalmemory/hooks/context_payload.py +0 -2
- package/src/superlocalmemory/hooks/portable_kit.py +8 -8
- package/src/superlocalmemory/hooks/post_tool_async_hook.py +0 -2
- package/src/superlocalmemory/hooks/prewarm_auth.py +0 -2
- package/src/superlocalmemory/hooks/user_prompt_hook.py +0 -2
- package/src/superlocalmemory/infra/backup.py +44 -8
- package/src/superlocalmemory/integrations/bounded_loops_mcp.py +24 -7
- package/src/superlocalmemory/learning/arm_catalog.py +0 -2
- package/src/superlocalmemory/learning/bandit.py +0 -2
- package/src/superlocalmemory/learning/bandit_cache.py +0 -2
- package/src/superlocalmemory/learning/dedup_hnsw.py +11 -11
- package/src/superlocalmemory/learning/ensemble.py +0 -2
- package/src/superlocalmemory/learning/labeler.py +0 -2
- package/src/superlocalmemory/learning/legacy_migration.py +0 -2
- package/src/superlocalmemory/learning/model_cache.py +0 -2
- package/src/superlocalmemory/learning/ranker.py +0 -2
- package/src/superlocalmemory/learning/reward_archive.py +6 -1
- package/src/superlocalmemory/learning/reward_proxy.py +0 -2
- package/src/superlocalmemory/learning/signal_worker.py +0 -2
- package/src/superlocalmemory/math/fisher.py +1 -1
- package/src/superlocalmemory/math/hopfield.py +4 -1
- package/src/superlocalmemory/math/langevin.py +1 -1
- package/src/superlocalmemory/math/sheaf.py +7 -3
- package/src/superlocalmemory/mcp/cli_fallback.py +1 -1
- package/src/superlocalmemory/mcp/profiles.py +1 -1
- package/src/superlocalmemory/mcp/server.py +1 -1
- package/src/superlocalmemory/mcp/tools_active.py +56 -0
- package/src/superlocalmemory/mcp/tools_core.py +1 -1
- package/src/superlocalmemory/optimize/cache/manager.py +2 -2
- package/src/superlocalmemory/optimize/compress/ccr.py +1 -1
- package/src/superlocalmemory/optimize/compress/router.py +1 -1
- package/src/superlocalmemory/optimize/proxy/_helpers.py +2 -2
- package/src/superlocalmemory/optimize/proxy/server.py +1 -1
- package/src/superlocalmemory/optimize/proxy/vertex_surface.py +2 -2
- package/src/superlocalmemory/optimize/storage/db.py +2 -2
- package/src/superlocalmemory/retrieval/agentic.py +1 -1
- package/src/superlocalmemory/retrieval/ann_index.py +9 -2
- package/src/superlocalmemory/retrieval/bm25_channel.py +2 -2
- package/src/superlocalmemory/retrieval/bridge_discovery.py +2 -2
- package/src/superlocalmemory/retrieval/engine.py +272 -43
- package/src/superlocalmemory/retrieval/entity_channel.py +1 -1
- package/src/superlocalmemory/retrieval/hopfield_channel.py +8 -2
- package/src/superlocalmemory/retrieval/profile_channel.py +1 -1
- package/src/superlocalmemory/retrieval/quantization_aware_search.py +1 -1
- package/src/superlocalmemory/retrieval/remote_reranker.py +2 -2
- package/src/superlocalmemory/retrieval/reranker.py +3 -3
- package/src/superlocalmemory/retrieval/semantic_channel.py +3 -3
- package/src/superlocalmemory/retrieval/spreading_activation.py +8 -8
- package/src/superlocalmemory/retrieval/strategy.py +94 -0
- package/src/superlocalmemory/retrieval/temporal_channel.py +167 -10
- package/src/superlocalmemory/retrieval/temporal_validity_filter.py +1 -1
- package/src/superlocalmemory/retrieval/vector_store.py +88 -10
- package/src/superlocalmemory/server/recall_serializer.py +10 -0
- package/src/superlocalmemory/server/routes/brain.py +2 -2
- package/src/superlocalmemory/server/routes/prewarm.py +4 -4
- package/src/superlocalmemory/server/unified_daemon.py +459 -7
- package/src/superlocalmemory/storage/_schema_version.py +46 -3
- package/src/superlocalmemory/storage/backup.py +531 -0
- package/src/superlocalmemory/storage/database.py +11 -4
- package/src/superlocalmemory/storage/embedding_codec.py +129 -0
- package/src/superlocalmemory/storage/embedding_migrator.py +5 -3
- package/src/superlocalmemory/storage/migration_runner.py +142 -2
- package/src/superlocalmemory/storage/migrations/__init__.py +1 -1
- package/src/superlocalmemory/storage/migrations.py +15 -1
- package/src/superlocalmemory/storage/models.py +7 -0
- package/src/superlocalmemory/storage/quantized_store.py +4 -2
- package/src/superlocalmemory/summaries/session_summary.py +1 -1
- package/src/superlocalmemory/ui/css/legacy-dashboard.css +1 -1
- package/src/superlocalmemory/ui/css/neural-glass.css +1 -1
- package/src/superlocalmemory/ui/js/core.js +1 -1
- package/src/superlocalmemory/ui/js/od-brain.js +1 -1
|
@@ -26,6 +26,8 @@ License: AGPL-3.0-or-later
|
|
|
26
26
|
from __future__ import annotations
|
|
27
27
|
|
|
28
28
|
import asyncio
|
|
29
|
+
import concurrent.futures
|
|
30
|
+
import functools
|
|
29
31
|
import hashlib
|
|
30
32
|
import json
|
|
31
33
|
import logging
|
|
@@ -67,6 +69,126 @@ from superlocalmemory.infra.data_root import (
|
|
|
67
69
|
)
|
|
68
70
|
|
|
69
71
|
|
|
72
|
+
def _notify_migration_applied(
|
|
73
|
+
applied: list,
|
|
74
|
+
elapsed: float,
|
|
75
|
+
backup_dir: "Path | None" = None,
|
|
76
|
+
) -> None:
|
|
77
|
+
"""Print a one-line confirmation after migrations complete successfully.
|
|
78
|
+
|
|
79
|
+
Written to stdout so it is visible in daemon.log and in interactive runs.
|
|
80
|
+
"""
|
|
81
|
+
count = len(applied)
|
|
82
|
+
if count == 0:
|
|
83
|
+
return
|
|
84
|
+
parts = [f"[SLM] {count} migration(s) applied in {elapsed:.1f}s."]
|
|
85
|
+
if backup_dir is not None:
|
|
86
|
+
parts.append(f"Backup: {backup_dir}")
|
|
87
|
+
print(" ".join(parts), flush=True)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _write_migration_error_log(
|
|
91
|
+
failed: list,
|
|
92
|
+
backup_dir: "Path | None",
|
|
93
|
+
slm_home: "Path | None" = None,
|
|
94
|
+
applied: "list | None" = None,
|
|
95
|
+
) -> "Path":
|
|
96
|
+
"""Write a migration-error-{timestamp}.log to the SLM home directory.
|
|
97
|
+
|
|
98
|
+
Returns the Path of the written log file.
|
|
99
|
+
"""
|
|
100
|
+
import datetime
|
|
101
|
+
|
|
102
|
+
if slm_home is None:
|
|
103
|
+
slm_home = canonical_data_root()
|
|
104
|
+
|
|
105
|
+
slm_home.mkdir(parents=True, exist_ok=True)
|
|
106
|
+
ts = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
|
|
107
|
+
log_path = slm_home / f"migration-error-{ts}.log"
|
|
108
|
+
|
|
109
|
+
lines = [
|
|
110
|
+
f"Migration error — {ts}",
|
|
111
|
+
f"Failed: {', '.join(str(f) for f in failed)}",
|
|
112
|
+
]
|
|
113
|
+
# An upgrade is applied step by step and is deliberately non-fatal: some
|
|
114
|
+
# steps can succeed while a later one fails. Saying "your data was not
|
|
115
|
+
# modified" whenever a snapshot exists was therefore untrue in exactly the
|
|
116
|
+
# case that matters — and a user who believes nothing changed may delete the
|
|
117
|
+
# snapshot to reclaim space.
|
|
118
|
+
if applied:
|
|
119
|
+
lines.append(f"Applied before the failure: {', '.join(str(a) for a in applied)}")
|
|
120
|
+
lines.append(
|
|
121
|
+
"YOUR DATABASE WAS PARTIALLY CHANGED. The steps listed above "
|
|
122
|
+
"completed; the ones under 'Failed' did not."
|
|
123
|
+
)
|
|
124
|
+
elif applied is not None:
|
|
125
|
+
lines.append("No steps completed, so your database was not changed.")
|
|
126
|
+
if backup_dir is not None:
|
|
127
|
+
lines.append(f"Snapshot of the state before the upgrade: {backup_dir}")
|
|
128
|
+
lines.append(
|
|
129
|
+
"That snapshot is intact and can be restored. Keep it until you are "
|
|
130
|
+
"satisfied your data is correct."
|
|
131
|
+
)
|
|
132
|
+
else:
|
|
133
|
+
lines.append(
|
|
134
|
+
"NO SNAPSHOT WAS TAKEN for this attempt — do not assume a recovery "
|
|
135
|
+
"copy exists. Check for earlier snapshots before changing anything."
|
|
136
|
+
)
|
|
137
|
+
lines.append("")
|
|
138
|
+
lines.append("To diagnose, run: slm doctor")
|
|
139
|
+
|
|
140
|
+
log_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
141
|
+
return log_path
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _notify_windows_user(message: str) -> None:
|
|
145
|
+
"""Best-effort notification on Windows. Silent no-op on other platforms."""
|
|
146
|
+
import sys
|
|
147
|
+
|
|
148
|
+
if sys.platform != "win32":
|
|
149
|
+
return
|
|
150
|
+
|
|
151
|
+
# Attempt 1 — Windows MessageBox via PowerShell
|
|
152
|
+
try:
|
|
153
|
+
import subprocess
|
|
154
|
+
|
|
155
|
+
safe_msg = message.replace("'", "").replace('"', "")
|
|
156
|
+
subprocess.run(
|
|
157
|
+
[
|
|
158
|
+
"powershell",
|
|
159
|
+
"-Command",
|
|
160
|
+
"[System.Reflection.Assembly]::LoadWithPartialName"
|
|
161
|
+
"('System.Windows.Forms') | Out-Null; "
|
|
162
|
+
f"[System.Windows.Forms.MessageBox]::Show('{safe_msg}', "
|
|
163
|
+
"'SuperLocalMemory')",
|
|
164
|
+
],
|
|
165
|
+
check=False,
|
|
166
|
+
timeout=5,
|
|
167
|
+
)
|
|
168
|
+
return
|
|
169
|
+
except Exception:
|
|
170
|
+
pass
|
|
171
|
+
|
|
172
|
+
# Attempt 2 — Windows Application EventLog
|
|
173
|
+
try:
|
|
174
|
+
import subprocess
|
|
175
|
+
|
|
176
|
+
subprocess.run(
|
|
177
|
+
[
|
|
178
|
+
"eventcreate",
|
|
179
|
+
"/T", "INFORMATION",
|
|
180
|
+
"/ID", "1001",
|
|
181
|
+
"/L", "APPLICATION",
|
|
182
|
+
"/SO", "SuperLocalMemory",
|
|
183
|
+
"/D", message[:512],
|
|
184
|
+
],
|
|
185
|
+
check=False,
|
|
186
|
+
timeout=5,
|
|
187
|
+
)
|
|
188
|
+
except Exception:
|
|
189
|
+
pass # best-effort only
|
|
190
|
+
|
|
191
|
+
|
|
70
192
|
def _learning_db_for_config(config) -> Path:
|
|
71
193
|
"""Single learning.db path for daemon migration + engine (F-05).
|
|
72
194
|
|
|
@@ -115,11 +237,142 @@ _SOURCE_QUALITY_MAX_BATCH_SIZE = 250
|
|
|
115
237
|
_SOURCE_QUALITY_PROFILE_REFRESH_SECONDS = 60.0
|
|
116
238
|
_FACT_ENTITY_REPAIR_MIN_RETRY_SECONDS = 0.05
|
|
117
239
|
_FACT_ENTITY_REPAIR_MAX_RETRY_SECONDS = 30.0
|
|
118
|
-
#
|
|
119
|
-
#
|
|
120
|
-
#
|
|
121
|
-
#
|
|
122
|
-
|
|
240
|
+
# How long a write may spend making itself findable by meaning before the
|
|
241
|
+
# receipt goes out. Storing a memory has a 1.5 s ceiling, and this is a ceiling
|
|
242
|
+
# rather than a target: the cost of coming in under it is a memory that can only
|
|
243
|
+
# be found by quoting its own wording, which is not how anyone asks. So the
|
|
244
|
+
# window is most of the budget, not a token amount of it. ``wait=true`` is still
|
|
245
|
+
# never permission to hold the event loop open indefinitely — past this the
|
|
246
|
+
# background pass finishes the job.
|
|
247
|
+
# The ceiling for a whole store request, shared by the durable write and the
|
|
248
|
+
# window that makes the memory findable by meaning. Both are sequential, so the
|
|
249
|
+
# second gets what the first left rather than a fresh grant.
|
|
250
|
+
_REMEMBER_TOTAL_CEILING_SECONDS = 1.5
|
|
251
|
+
_REMEMBER_ENRICHMENT_WAIT_SECONDS = 1.2
|
|
252
|
+
|
|
253
|
+
# Enrichment runs off the event loop, but NOT on the loop's default executor.
|
|
254
|
+
# `asyncio.to_thread` uses that default pool, so N simultaneous writers really did
|
|
255
|
+
# create N threads — and every other handler that borrows the same pool queued
|
|
256
|
+
# behind them. One shared, small pool bounds the cost of a burst: past its width
|
|
257
|
+
# the extra callers miss their deadline and defer, which is the intended
|
|
258
|
+
# degradation, instead of multiplying threads.
|
|
259
|
+
_ENRICHMENT_WORKERS = 2
|
|
260
|
+
_enrichment_pool: "concurrent.futures.ThreadPoolExecutor | None" = None
|
|
261
|
+
_enrichment_pool_lock = threading.Lock()
|
|
262
|
+
|
|
263
|
+
# Bounds how much enrichment may be in flight at once. A ThreadPoolExecutor
|
|
264
|
+
# queues without limit, so bounding the pool's WIDTH does not bound what can be
|
|
265
|
+
# handed to it: a burst of writers all submit, the first two run, and the rest sit
|
|
266
|
+
# in a queue nobody is waiting for any more — then wake up and write to the
|
|
267
|
+
# database after their callers have already been answered. A permit is taken
|
|
268
|
+
# before submitting and released by the worker when it has really finished, so at
|
|
269
|
+
# most `_ENRICHMENT_WORKERS` pieces of work exist at once and a caller that
|
|
270
|
+
# cannot get one is told immediately instead of being queued.
|
|
271
|
+
_enrichment_semaphore = threading.Semaphore(_ENRICHMENT_WORKERS)
|
|
272
|
+
|
|
273
|
+
# Set once the pool has been shut down, and never cleared. Without it, a caller
|
|
274
|
+
# that passed the unlocked check below arrives after shutdown, finds None, and
|
|
275
|
+
# builds a replacement pool that the completed shutdown will never reach.
|
|
276
|
+
_enrichment_pool_closed = False
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
class _EnrichmentAtCapacity(Exception):
|
|
280
|
+
"""No permit was free, so nothing was submitted.
|
|
281
|
+
|
|
282
|
+
Its own type, so that "we chose not to start" is never reported through the
|
|
283
|
+
same path as "it started and then failed" — those are different events and
|
|
284
|
+
only one of them is worth a warning about degradation.
|
|
285
|
+
"""
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def _enrichment_executor():
|
|
289
|
+
"""The one pool inline enrichment may use. Created on first need.
|
|
290
|
+
|
|
291
|
+
Raises ``RuntimeError`` once the pool has been shut down. It deliberately
|
|
292
|
+
does not return ``None`` on that path: ``run_in_executor(None, ...)`` means
|
|
293
|
+
*the event loop's default executor*, which is the unbounded-thread behaviour
|
|
294
|
+
this pool exists to replace, so a None here would silently reintroduce it at
|
|
295
|
+
exactly the moment the process is trying to stop.
|
|
296
|
+
"""
|
|
297
|
+
global _enrichment_pool
|
|
298
|
+
# Everything happens under the lock, including the read that is returned. An
|
|
299
|
+
# earlier version checked the closed flag only on the create branch and then
|
|
300
|
+
# returned the global, so a caller that found a live pool, lost the CPU, and
|
|
301
|
+
# resumed after shutdown returned None — and None means the event loop's own
|
|
302
|
+
# default executor, unbounded and shared with every other handler. That is
|
|
303
|
+
# the precise failure this function exists to prevent, reintroduced at the
|
|
304
|
+
# one moment it matters.
|
|
305
|
+
with _enrichment_pool_lock:
|
|
306
|
+
if _enrichment_pool_closed:
|
|
307
|
+
raise RuntimeError("enrichment pool is shut down")
|
|
308
|
+
if _enrichment_pool is None:
|
|
309
|
+
_enrichment_pool = concurrent.futures.ThreadPoolExecutor(
|
|
310
|
+
max_workers=_ENRICHMENT_WORKERS,
|
|
311
|
+
thread_name_prefix="slm-enrich",
|
|
312
|
+
)
|
|
313
|
+
pool = _enrichment_pool
|
|
314
|
+
if pool is None: # pragma: no cover — belt and braces around the invariant
|
|
315
|
+
raise RuntimeError("enrichment pool is unavailable")
|
|
316
|
+
return pool
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def _open_enrichment_pool() -> None:
|
|
320
|
+
"""Allow the pool to be created again, for a fresh run in this process.
|
|
321
|
+
|
|
322
|
+
The closed flag has to survive shutdown — that is its whole purpose — so
|
|
323
|
+
something has to clear it when a new run legitimately begins. Without this a
|
|
324
|
+
second startup in one process (a restart, or a test that builds the app more
|
|
325
|
+
than once) would find enrichment permanently refused and every write would
|
|
326
|
+
come back findable by wording only, with nothing in the logs to explain it.
|
|
327
|
+
"""
|
|
328
|
+
global _enrichment_pool_closed, _enrichment_semaphore
|
|
329
|
+
with _enrichment_pool_lock:
|
|
330
|
+
_enrichment_pool_closed = False
|
|
331
|
+
# A fresh run starts with its full capacity. Shutdown cancels work that
|
|
332
|
+
# was queued and never started, and a task that never starts never runs
|
|
333
|
+
# the release in its finally — so permits taken by cancelled work are
|
|
334
|
+
# gone. Without this reset the next run in the same process would run at
|
|
335
|
+
# reduced capacity, or none at all, and the only symptom would be every
|
|
336
|
+
# write coming back findable by wording with nothing in the log.
|
|
337
|
+
_enrichment_semaphore = threading.Semaphore(_ENRICHMENT_WORKERS)
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def _enrich_and_release(engine, fact_ids: list[str], budget: float) -> int:
|
|
341
|
+
"""Run inline enrichment, releasing the permit only when it is really done.
|
|
342
|
+
|
|
343
|
+
The permit has to be released here rather than by the caller. A caller that
|
|
344
|
+
stops waiting at its own deadline has not finished the work — the worker is
|
|
345
|
+
still holding a pool thread — so releasing on the caller's timeout would let
|
|
346
|
+
the next request submit into a pool that is still fully occupied, which is
|
|
347
|
+
the unbounded queueing this permit exists to prevent.
|
|
348
|
+
"""
|
|
349
|
+
try:
|
|
350
|
+
return engine.enrich_new_facts_now(fact_ids, timeout_s=budget)
|
|
351
|
+
finally:
|
|
352
|
+
_enrichment_semaphore.release()
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
def _shutdown_enrichment_pool_if_created() -> None:
|
|
356
|
+
"""Shut down the enrichment pool if it was ever created.
|
|
357
|
+
|
|
358
|
+
Safe to call when the pool was never created (no-op) and safe to call
|
|
359
|
+
twice (second call is a no-op because the pool is set to None under the
|
|
360
|
+
lock before shutdown is called).
|
|
361
|
+
|
|
362
|
+
cancel_futures=True drops any work that is still queued but has not yet
|
|
363
|
+
started. Workers already inside embed() will finish their current call
|
|
364
|
+
and then stop; they do not block the caller because wait=False.
|
|
365
|
+
"""
|
|
366
|
+
global _enrichment_pool, _enrichment_pool_closed
|
|
367
|
+
with _enrichment_pool_lock:
|
|
368
|
+
# Marked closed under the same lock that guards creation, so a caller
|
|
369
|
+
# already past the unlocked check cannot build a replacement pool that
|
|
370
|
+
# this shutdown has already walked past.
|
|
371
|
+
_enrichment_pool_closed = True
|
|
372
|
+
pool = _enrichment_pool
|
|
373
|
+
_enrichment_pool = None
|
|
374
|
+
if pool is not None:
|
|
375
|
+
pool.shutdown(wait=False, cancel_futures=True)
|
|
123
376
|
_SENSITIVE_READ_PREFIXES = (
|
|
124
377
|
"/api/memories", "/api/facts", "/api/clusters", "/api/graph",
|
|
125
378
|
"/api/v3/associations", "/api/v3/core-memory",
|
|
@@ -1500,6 +1753,11 @@ async def lifespan(application: FastAPI):
|
|
|
1500
1753
|
"""Initialize engine, workers, and optional services on startup."""
|
|
1501
1754
|
global _last_activity
|
|
1502
1755
|
|
|
1756
|
+
# A previous run in this process left the enrichment pool closed so that a
|
|
1757
|
+
# write racing its shutdown could not resurrect it. This run is entitled to
|
|
1758
|
+
# one.
|
|
1759
|
+
_open_enrichment_pool()
|
|
1760
|
+
|
|
1503
1761
|
engine = None
|
|
1504
1762
|
config = None
|
|
1505
1763
|
deployment = None
|
|
@@ -1592,6 +1850,7 @@ async def lifespan(application: FastAPI):
|
|
|
1592
1850
|
_path_config = None
|
|
1593
1851
|
|
|
1594
1852
|
try:
|
|
1853
|
+
from superlocalmemory.storage.backup import InsufficientDiskSpaceError
|
|
1595
1854
|
from superlocalmemory.storage.migration_runner import apply_all
|
|
1596
1855
|
if _path_config is None:
|
|
1597
1856
|
# Catastrophic early-load failure: still one root via Mode-A default
|
|
@@ -1602,13 +1861,56 @@ async def lifespan(application: FastAPI):
|
|
|
1602
1861
|
_path_config = _SLMFallback.for_mode(_ModeFallback.A)
|
|
1603
1862
|
_learning_db = _learning_db_for_config(_path_config)
|
|
1604
1863
|
_memory_db = _memory_db_for_config(_path_config)
|
|
1864
|
+
import time as _time_mod
|
|
1865
|
+
_t0 = _time_mod.monotonic()
|
|
1605
1866
|
_result = apply_all(_learning_db, _memory_db)
|
|
1867
|
+
_elapsed = _time_mod.monotonic() - _t0
|
|
1606
1868
|
_applied = _result.get("applied", [])
|
|
1607
1869
|
_failed = _result.get("failed", [])
|
|
1870
|
+
_backup_dir = _result.get("details", {}).get("_backup")
|
|
1871
|
+
_backup_path = Path(_backup_dir) if _backup_dir else None
|
|
1608
1872
|
if _applied:
|
|
1609
1873
|
logger.info("migrations applied: %s", _applied)
|
|
1874
|
+
_notify_migration_applied(_applied, _elapsed, _backup_path)
|
|
1875
|
+
_notify_windows_user(
|
|
1876
|
+
f"SuperLocalMemory: {len(_applied)} database migration(s) applied."
|
|
1877
|
+
)
|
|
1610
1878
|
if _failed:
|
|
1611
1879
|
logger.warning("migrations failed (non-fatal): %s", _failed)
|
|
1880
|
+
try:
|
|
1881
|
+
_err_log = _write_migration_error_log(
|
|
1882
|
+
_failed, _backup_path,
|
|
1883
|
+
slm_home=_memory_db.parent if _memory_db else None,
|
|
1884
|
+
applied=_applied,
|
|
1885
|
+
)
|
|
1886
|
+
import sys as _sys
|
|
1887
|
+
# "Data is safe" is only true when NOTHING was applied. Migrations
|
|
1888
|
+
# are non-fatal and applied in order, so a later failure can leave
|
|
1889
|
+
# earlier ones committed — a partially changed store. Saying the
|
|
1890
|
+
# data is untouched then tells the user to ignore the one copy
|
|
1891
|
+
# that still holds the original.
|
|
1892
|
+
if _applied:
|
|
1893
|
+
_headline = (
|
|
1894
|
+
f"[SLM] Migration incomplete. YOUR DATABASE WAS PARTIALLY "
|
|
1895
|
+
f"CHANGED — {len(_applied)} step(s) applied, "
|
|
1896
|
+
f"{len(_failed)} failed. The pre-change copy is at "
|
|
1897
|
+
f"{_backup_path} — keep it until this is resolved."
|
|
1898
|
+
)
|
|
1899
|
+
else:
|
|
1900
|
+
_headline = (
|
|
1901
|
+
f"[SLM] Migration failed before any change was made. Your "
|
|
1902
|
+
f"data is as it was; a copy is also at {_backup_path}."
|
|
1903
|
+
)
|
|
1904
|
+
print(
|
|
1905
|
+
f"{_headline} See {_err_log}. Run: slm doctor",
|
|
1906
|
+
file=_sys.stderr,
|
|
1907
|
+
flush=True,
|
|
1908
|
+
)
|
|
1909
|
+
_notify_windows_user(
|
|
1910
|
+
f"SuperLocalMemory migration failed. Run slm doctor. Log: {_err_log}"
|
|
1911
|
+
)
|
|
1912
|
+
except Exception:
|
|
1913
|
+
pass # notification failures are non-fatal
|
|
1612
1914
|
application.state.migration_result = _result
|
|
1613
1915
|
# S9-SKEP-15: only commit the new `.last_version` AFTER migrations
|
|
1614
1916
|
# complete with zero failures. A partial upgrade (schema didn't
|
|
@@ -1626,6 +1928,35 @@ async def lifespan(application: FastAPI):
|
|
|
1626
1928
|
_version_marker.write_text(_slm_version, encoding="utf-8")
|
|
1627
1929
|
except OSError:
|
|
1628
1930
|
pass # non-fatal
|
|
1931
|
+
except InsufficientDiskSpaceError as _disk_exc:
|
|
1932
|
+
# This is not a crash and it must not be reported as "non-fatal". The
|
|
1933
|
+
# migration deliberately refused to start because there was not enough
|
|
1934
|
+
# room to keep a recoverable copy first. The store is intact and
|
|
1935
|
+
# unmigrated; the generic handler below would have logged a warning and
|
|
1936
|
+
# left the daemon serving as though nothing had happened.
|
|
1937
|
+
logger.error(
|
|
1938
|
+
"MIGRATION NOT RUN — not enough free disk to keep a recoverable "
|
|
1939
|
+
"copy first. Your data has NOT been modified. Need %s bytes, have "
|
|
1940
|
+
"%s free. Free some space and restart; run `slm doctor` for details.",
|
|
1941
|
+
f"{_disk_exc.needed_bytes:,}", f"{_disk_exc.free_bytes:,}",
|
|
1942
|
+
)
|
|
1943
|
+
try:
|
|
1944
|
+
_log_path = _write_migration_error_log(
|
|
1945
|
+
["_insufficient_disk_space"], None,
|
|
1946
|
+
slm_home=locals().get("_memory_db").parent if locals().get("_memory_db") else None,
|
|
1947
|
+
applied=[],
|
|
1948
|
+
)
|
|
1949
|
+
logger.error("Details written to %s", _log_path)
|
|
1950
|
+
except OSError:
|
|
1951
|
+
pass
|
|
1952
|
+
application.state.migration_result = {
|
|
1953
|
+
"applied": [], "skipped": [], "failed": ["_insufficient_disk_space"],
|
|
1954
|
+
"details": {
|
|
1955
|
+
"_needed_bytes": _disk_exc.needed_bytes,
|
|
1956
|
+
"_free_bytes": _disk_exc.free_bytes,
|
|
1957
|
+
"_store_modified": False,
|
|
1958
|
+
},
|
|
1959
|
+
}
|
|
1629
1960
|
except Exception as _exc:
|
|
1630
1961
|
logger.warning("migration runner crashed (non-fatal): %s", _exc)
|
|
1631
1962
|
application.state.migration_result = {
|
|
@@ -2822,6 +3153,19 @@ async def lifespan(application: FastAPI):
|
|
|
2822
3153
|
except Exception as exc: # pragma: no cover — defensive
|
|
2823
3154
|
logger.warning("perf_log flush failed: %s", exc)
|
|
2824
3155
|
|
|
3156
|
+
# Shut down the enrichment pool before engine.close() so that any worker
|
|
3157
|
+
# still inside embed() receives cancellation before the embed pool is torn
|
|
3158
|
+
# down. shutdown(wait=False, cancel_futures=True) is non-blocking: queued
|
|
3159
|
+
# futures are dropped and running futures complete their current call on
|
|
3160
|
+
# their own. This prevents Python's atexit handler from calling
|
|
3161
|
+
# shutdown(wait=True) on a live embed worker, which would push the daemon
|
|
3162
|
+
# past the graceful-shutdown budget and cause the service manager to
|
|
3163
|
+
# SIGKILL the process.
|
|
3164
|
+
try:
|
|
3165
|
+
_shutdown_enrichment_pool_if_created()
|
|
3166
|
+
except Exception as exc: # pragma: no cover — defensive
|
|
3167
|
+
logger.warning("enrichment pool shutdown failed (non-fatal): %s", exc)
|
|
3168
|
+
|
|
2825
3169
|
materializer_stopped = _stop_pending_materializer()
|
|
2826
3170
|
canonical_writer_stopped = _release_canonical_remember_runtime(application)
|
|
2827
3171
|
_profile_runtime = None
|
|
@@ -4175,6 +4519,10 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
4175
4519
|
allowed_profiles=frozenset({engine._profile_id}),
|
|
4176
4520
|
allowed_scopes=frozenset({scope}),
|
|
4177
4521
|
)
|
|
4522
|
+
# The whole request has a 1.5 s ceiling, and the two phases below
|
|
4523
|
+
# are sequential, so the ceiling has to be shared between them
|
|
4524
|
+
# rather than granted twice. Measured from here.
|
|
4525
|
+
_store_started = time.monotonic()
|
|
4178
4526
|
receipt = await asyncio.to_thread(
|
|
4179
4527
|
runtime.remember,
|
|
4180
4528
|
admission,
|
|
@@ -4183,10 +4531,105 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
4183
4531
|
)
|
|
4184
4532
|
payload = dict(receipt.payload)
|
|
4185
4533
|
fact_ids = list(payload.get("fact_ids") or [])
|
|
4534
|
+
|
|
4535
|
+
# The durable receipt is already committed above; nothing below can
|
|
4536
|
+
# fail this write. What remains is the window in which the memory can
|
|
4537
|
+
# only be found by quoting its own wording — a fact with no vector is
|
|
4538
|
+
# invisible to the semantic channel, so the memory describing what is
|
|
4539
|
+
# happening right now is the hardest one to find.
|
|
4540
|
+
#
|
|
4541
|
+
# Every entry point — command line, tool interface, dashboard —
|
|
4542
|
+
# arrives here, so closing the window here closes it for all of them
|
|
4543
|
+
# rather than for whichever door happened to be used.
|
|
4544
|
+
#
|
|
4545
|
+
# Strictly bounded, because this daemon serves many sessions at once:
|
|
4546
|
+
# the work runs off the event loop, on a two-worker pool, and yields
|
|
4547
|
+
# to any recall in flight. Past the deadline the caller gets exactly
|
|
4548
|
+
# the previous behaviour and the background materializer finishes the
|
|
4549
|
+
# job.
|
|
4550
|
+
# A caller that did not ask to wait still gets a real attempt, not a
|
|
4551
|
+
# token one. Almost no client passes wait=true, so a small budget
|
|
4552
|
+
# here meant almost every memory in practice was returned findable by
|
|
4553
|
+
# wording only — the default path deciding the product's behaviour.
|
|
4554
|
+
#
|
|
4555
|
+
# But it gets what is LEFT of the ceiling, not a second full budget.
|
|
4556
|
+
# The durable write above can legitimately consume most of the 1.5 s
|
|
4557
|
+
# under contention, and granting the full enrichment window on top of
|
|
4558
|
+
# that produced receipts well past 3 s — over a ceiling the changelog
|
|
4559
|
+
# states. A caller that asked to wait may exceed the shared ceiling,
|
|
4560
|
+
# because that is what asking to wait means; nobody else may.
|
|
4561
|
+
_elapsed = time.monotonic() - _store_started
|
|
4562
|
+
_remaining = _REMEMBER_TOTAL_CEILING_SECONDS - _elapsed
|
|
4563
|
+
if wait:
|
|
4564
|
+
enrich_budget = _REMEMBER_ENRICHMENT_WAIT_SECONDS
|
|
4565
|
+
else:
|
|
4566
|
+
enrich_budget = min(
|
|
4567
|
+
1.0, _REMEMBER_ENRICHMENT_WAIT_SECONDS, max(0.0, _remaining),
|
|
4568
|
+
)
|
|
4569
|
+
if enrich_budget <= 0.0:
|
|
4570
|
+
logger.warning(
|
|
4571
|
+
"the durable write used the whole %.1fs budget (%.2fs) — this "
|
|
4572
|
+
"memory is findable by its wording and the background pass "
|
|
4573
|
+
"will attach its vector",
|
|
4574
|
+
_REMEMBER_TOTAL_CEILING_SECONDS, _elapsed,
|
|
4575
|
+
)
|
|
4576
|
+
enriched = 0
|
|
4577
|
+
# Taken before submitting, released by the worker. If none is free
|
|
4578
|
+
# the pool is already saturated, and queueing behind it would mean
|
|
4579
|
+
# writing to the database after this response has been sent — so the
|
|
4580
|
+
# write is simply reported as findable by wording and the background
|
|
4581
|
+
# pass picks it up, which is the intended degradation.
|
|
4582
|
+
_permit = _enrichment_semaphore.acquire(blocking=False)
|
|
4583
|
+
if not _permit:
|
|
4584
|
+
logger.warning(
|
|
4585
|
+
"inline enrichment at capacity for %d fact(s) — deferred to "
|
|
4586
|
+
"the background pass", len(fact_ids),
|
|
4587
|
+
)
|
|
4588
|
+
try:
|
|
4589
|
+
if not _permit:
|
|
4590
|
+
raise _EnrichmentAtCapacity
|
|
4591
|
+
loop = asyncio.get_running_loop()
|
|
4592
|
+
try:
|
|
4593
|
+
_pending = loop.run_in_executor(
|
|
4594
|
+
_enrichment_executor(),
|
|
4595
|
+
functools.partial(
|
|
4596
|
+
_enrich_and_release,
|
|
4597
|
+
engine, fact_ids, enrich_budget,
|
|
4598
|
+
),
|
|
4599
|
+
)
|
|
4600
|
+
except BaseException:
|
|
4601
|
+
# Never handed to a worker, so nothing will release it.
|
|
4602
|
+
_enrichment_semaphore.release()
|
|
4603
|
+
raise
|
|
4604
|
+
enriched = await asyncio.wait_for(
|
|
4605
|
+
_pending, timeout=enrich_budget + 0.25,
|
|
4606
|
+
)
|
|
4607
|
+
except _EnrichmentAtCapacity:
|
|
4608
|
+
pass
|
|
4609
|
+
except (TimeoutError, asyncio.TimeoutError):
|
|
4610
|
+
# Worth seeing. Sustained timeouts mean writes are coming back
|
|
4611
|
+
# findable by wording only, which is a real degradation and was
|
|
4612
|
+
# previously visible at debug level alone.
|
|
4613
|
+
logger.warning(
|
|
4614
|
+
"inline enrichment exceeded its budget for %d fact(s) — "
|
|
4615
|
+
"deferred to the background pass", len(fact_ids),
|
|
4616
|
+
)
|
|
4617
|
+
except Exception as exc:
|
|
4618
|
+
logger.warning(
|
|
4619
|
+
"inline enrichment unavailable (%s: %s) — deferred to the "
|
|
4620
|
+
"background pass", type(exc).__name__, exc,
|
|
4621
|
+
)
|
|
4622
|
+
|
|
4623
|
+
searchable = "meaning" if enriched == len(fact_ids) and fact_ids else "wording"
|
|
4186
4624
|
return {
|
|
4187
4625
|
"ok": True,
|
|
4188
4626
|
"fact_ids": fact_ids,
|
|
4189
4627
|
"count": len(fact_ids),
|
|
4628
|
+
# Storing a memory and being able to find it again are different
|
|
4629
|
+
# things, and reporting only the first is how a caller ends up
|
|
4630
|
+
# believing a memory is retrievable when it is not yet.
|
|
4631
|
+
"searchable_by": searchable,
|
|
4632
|
+
"enriched_now": enriched,
|
|
4190
4633
|
"operation_id": payload["operation_id"],
|
|
4191
4634
|
# One-release compatibility alias. The durable operation ID is
|
|
4192
4635
|
# opaque and replaces the integer pending.db row identifier.
|
|
@@ -4194,8 +4637,17 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
4194
4637
|
"status": "queryable",
|
|
4195
4638
|
"materialization_state": payload["materialization_state"],
|
|
4196
4639
|
"commit_sequence": payload.get("commit_sequence"),
|
|
4197
|
-
"note":
|
|
4198
|
-
|
|
4640
|
+
"note": (
|
|
4641
|
+
"stored and searchable by meaning"
|
|
4642
|
+
if searchable == "meaning"
|
|
4643
|
+
else "stored and searchable by wording; "
|
|
4644
|
+
"searchable by meaning shortly"
|
|
4645
|
+
),
|
|
4646
|
+
# Retained for callers that already read it. It used to be
|
|
4647
|
+
# unconditionally true, because `wait` bought nothing. It now
|
|
4648
|
+
# buys a larger best-effort enrichment budget — still never a
|
|
4649
|
+
# guarantee, and still never a block on durability.
|
|
4650
|
+
"wait_ignored": False if wait else True,
|
|
4199
4651
|
}
|
|
4200
4652
|
except Exception as exc:
|
|
4201
4653
|
from superlocalmemory.core.remember_admission import AdmissionRejected
|
|
@@ -84,19 +84,62 @@ def write_schema_version(conn: sqlite3.Connection, version: int) -> None:
|
|
|
84
84
|
)
|
|
85
85
|
|
|
86
86
|
|
|
87
|
+
def _detect_all_installs() -> list:
|
|
88
|
+
"""Delegate to core.install_detector; fail-open (returns [] on any error)."""
|
|
89
|
+
try:
|
|
90
|
+
from superlocalmemory.core.install_detector import (
|
|
91
|
+
_detect_all_installs as _real,
|
|
92
|
+
)
|
|
93
|
+
return _real()
|
|
94
|
+
except Exception: # import error, timeout, etc.
|
|
95
|
+
return []
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _format_install_block(installs: list) -> str:
|
|
99
|
+
"""Return a formatted multi-line string listing all detected installs."""
|
|
100
|
+
if not installs:
|
|
101
|
+
return ""
|
|
102
|
+
|
|
103
|
+
_upgrade_hints = {
|
|
104
|
+
"pipx": " pipx upgrade superlocalmemory",
|
|
105
|
+
"venv": " pip install -U superlocalmemory (in ~/.slm-venv)",
|
|
106
|
+
"npm": " npm install -g superlocalmemory@latest",
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
lines = ["\nAll detected installs:"]
|
|
110
|
+
seen_types: set[str] = set()
|
|
111
|
+
for entry in installs:
|
|
112
|
+
install_type = entry.get("type", "unknown")
|
|
113
|
+
version = entry.get("version", "unknown")
|
|
114
|
+
path = entry.get("path", "")
|
|
115
|
+
lines.append(f" {install_type:<6} ({version}): {path}")
|
|
116
|
+
seen_types.add(install_type)
|
|
117
|
+
|
|
118
|
+
lines.append("\nUpgrade all installs to the latest version:")
|
|
119
|
+
for install_type, cmd in _upgrade_hints.items():
|
|
120
|
+
if install_type in seen_types:
|
|
121
|
+
lines.append(cmd)
|
|
122
|
+
|
|
123
|
+
return "\n".join(lines)
|
|
124
|
+
|
|
125
|
+
|
|
87
126
|
def check_version_or_raise(db_path: Path) -> None:
|
|
88
127
|
"""Raise SchemaVersionError if the DB's stored version exceeds supported max.
|
|
89
128
|
|
|
129
|
+
When raising, the error message lists every SLM installation detected on
|
|
130
|
+
this machine so the user knows which copies to upgrade.
|
|
131
|
+
|
|
90
132
|
Non-mutating. Must be called before any write at startup.
|
|
91
133
|
"""
|
|
92
134
|
stored = read_schema_version(db_path)
|
|
93
135
|
if stored > SUPPORTED_SCHEMA_VERSION:
|
|
136
|
+
installs = _detect_all_installs()
|
|
137
|
+
install_block = _format_install_block(installs)
|
|
94
138
|
raise SchemaVersionError(
|
|
95
139
|
f"Database schema_version={stored} exceeds the maximum "
|
|
96
140
|
f"supported version={SUPPORTED_SCHEMA_VERSION}. "
|
|
97
|
-
"This installation is too old for this database.
|
|
98
|
-
|
|
99
|
-
f"{stored} or higher."
|
|
141
|
+
"This installation is too old for this database."
|
|
142
|
+
+ install_block
|
|
100
143
|
)
|
|
101
144
|
|
|
102
145
|
|